1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (c) 2006, Intel Corporation.
4	*
5	* Copyright (C) 2006-2008 Intel Corporation
6	* Author: Ashok Raj <ashok.raj@intel.com>
7	* Author: Shaohua Li <shaohua.li@intel.com>
8	* Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
9	*
10	* This file implements early detection/parsing of Remapping Devices
11	* reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
12	* tables.
13	*
14	* These routines are used by both DMA-remapping and Interrupt-remapping
15	*/
16
17	#define pr_fmt(fmt) "DMAR: " fmt
18
19	#include <linux/pci.h>
20	#include <linux/dmar.h>
21	#include <linux/iova.h>
22	#include <linux/timer.h>
23	#include <linux/irq.h>
24	#include <linux/interrupt.h>
25	#include <linux/tboot.h>
26	#include <linux/dmi.h>
27	#include <linux/slab.h>
28	#include <linux/iommu.h>
29	#include <linux/numa.h>
30	#include <linux/limits.h>
31	#include <asm/irq_remapping.h>
32
33	#include "iommu.h"
34	#include "../irq_remapping.h"
35	#include "perf.h"
36	#include "trace.h"
37	#include "perfmon.h"
38
39	typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
40	struct dmar_res_callback {
41	dmar_res_handler_t cb[ACPI_DMAR_TYPE_RESERVED];
42	void *arg[ACPI_DMAR_TYPE_RESERVED];
43	bool ignore_unhandled;
44	bool print_entry;
45	};
46
47	/*
48	* Assumptions:
49	* 1) The hotplug framework guarentees that DMAR unit will be hot-added
50	* before IO devices managed by that unit.
51	* 2) The hotplug framework guarantees that DMAR unit will be hot-removed
52	* after IO devices managed by that unit.
53	* 3) Hotplug events are rare.
54	*
55	* Locking rules for DMA and interrupt remapping related global data structures:
56	* 1) Use dmar_global_lock in process context
57	* 2) Use RCU in interrupt context
58	*/
59	DECLARE_RWSEM(dmar_global_lock);
60	LIST_HEAD(dmar_drhd_units);
61
62	struct acpi_table_header * __initdata dmar_tbl;
63	static int dmar_dev_scope_status = 1;
64	static DEFINE_IDA(dmar_seq_ids);
65
66	static int alloc_iommu(struct dmar_drhd_unit *drhd);
67	static void free_iommu(struct intel_iommu *iommu);
68
69	static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
70	{
71	/*
72	* add INCLUDE_ALL at the tail, so scan the list will find it at
73	* the very end.
74	*/
75	if (drhd->include_all)
76	list_add_tail_rcu(new: &drhd->list, head: &dmar_drhd_units);
77	else
78	list_add_rcu(new: &drhd->list, head: &dmar_drhd_units);
79	}
80
81	void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
82	{
83	struct acpi_dmar_device_scope *scope;
84
85	*cnt = 0;
86	while (start < end) {
87	scope = start;
88	if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
89	scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
90	scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
91	(*cnt)++;
92	else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
93	scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
94	pr_warn("Unsupported device scope\n");
95	}
96	start += scope->length;
97	}
98	if (*cnt == 0)
99	return NULL;
100
101	return kcalloc(n: *cnt, size: sizeof(struct dmar_dev_scope), GFP_KERNEL);
102	}
103
104	void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
105	{
106	int i;
107	struct device *tmp_dev;
108
109	if (*devices && *cnt) {
110	for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
111	put_device(dev: tmp_dev);
112	kfree(objp: *devices);
113	}
114
115	*devices = NULL;
116	*cnt = 0;
117	}
118
119	/* Optimize out kzalloc()/kfree() for normal cases */
120	static char dmar_pci_notify_info_buf[64];
121
122	static struct dmar_pci_notify_info *
123	dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
124	{
125	int level = 0;
126	size_t size;
127	struct pci_dev *tmp;
128	struct dmar_pci_notify_info *info;
129
130	/*
131	* Ignore devices that have a domain number higher than what can
132	* be looked up in DMAR, e.g. VMD subdevices with domain 0x10000
133	*/
134	if (pci_domain_nr(bus: dev->bus) > U16_MAX)
135	return NULL;
136
137	/* Only generate path[] for device addition event */
138	if (event == BUS_NOTIFY_ADD_DEVICE)
139	for (tmp = dev; tmp; tmp = tmp->bus->self)
140	level++;
141
142	size = struct_size(info, path, level);
143	if (size <= sizeof(dmar_pci_notify_info_buf)) {
144	info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
145	} else {
146	info = kzalloc(size, GFP_KERNEL);
147	if (!info) {
148	if (dmar_dev_scope_status == 0)
149	dmar_dev_scope_status = -ENOMEM;
150	return NULL;
151	}
152	}
153
154	info->event = event;
155	info->dev = dev;
156	info->seg = pci_domain_nr(bus: dev->bus);
157	info->level = level;
158	if (event == BUS_NOTIFY_ADD_DEVICE) {
159	for (tmp = dev; tmp; tmp = tmp->bus->self) {
160	level--;
161	info->path[level].bus = tmp->bus->number;
162	info->path[level].device = PCI_SLOT(tmp->devfn);
163	info->path[level].function = PCI_FUNC(tmp->devfn);
164	if (pci_is_root_bus(pbus: tmp->bus))
165	info->bus = tmp->bus->number;
166	}
167	}
168
169	return info;
170	}
171
172	static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
173	{
174	if ((void *)info != dmar_pci_notify_info_buf)
175	kfree(objp: info);
176	}
177
178	static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
179	struct acpi_dmar_pci_path *path, int count)
180	{
181	int i;
182
183	if (info->bus != bus)
184	goto fallback;
185	if (info->level != count)
186	goto fallback;
187
188	for (i = 0; i < count; i++) {
189	if (path[i].device != info->path[i].device ||
190	path[i].function != info->path[i].function)
191	goto fallback;
192	}
193
194	return true;
195
196	fallback:
197
198	if (count != 1)
199	return false;
200
201	i = info->level - 1;
202	if (bus == info->path[i].bus &&
203	path[0].device == info->path[i].device &&
204	path[0].function == info->path[i].function) {
205	pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
206	bus, path[0].device, path[0].function);
207	return true;
208	}
209
210	return false;
211	}
212
213	/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
214	int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
215	void *start, void*end, u16 segment,
216	struct dmar_dev_scope *devices,
217	int devices_cnt)
218	{
219	int i, level;
220	struct device *tmp, *dev = &info->dev->dev;
221	struct acpi_dmar_device_scope *scope;
222	struct acpi_dmar_pci_path *path;
223
224	if (segment != info->seg)
225	return 0;
226
227	for (; start < end; start += scope->length) {
228	scope = start;
229	if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
230	scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
231	continue;
232
233	path = (struct acpi_dmar_pci_path *)(scope + 1);
234	level = (scope->length - sizeof(*scope)) / sizeof(*path);
235	if (!dmar_match_pci_path(info, bus: scope->bus, path, count: level))
236	continue;
237
238	/*
239	* We expect devices with endpoint scope to have normal PCI
240	* headers, and devices with bridge scope to have bridge PCI
241	* headers. However PCI NTB devices may be listed in the
242	* DMAR table with bridge scope, even though they have a
243	* normal PCI header. NTB devices are identified by class
244	* "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
245	* for this special case.
246	*/
247	if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
248	info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
249	(scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
250	(info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
251	info->dev->class >> 16 != PCI_BASE_CLASS_BRIDGE))) {
252	pr_warn("Device scope type does not match for %s\n",
253	pci_name(info->dev));
254	return -EINVAL;
255	}
256
257	for_each_dev_scope(devices, devices_cnt, i, tmp)
258	if (tmp == NULL) {
259	devices[i].bus = info->dev->bus->number;
260	devices[i].devfn = info->dev->devfn;
261	rcu_assign_pointer(devices[i].dev,
262	get_device(dev));
263	return 1;
264	}
265	if (WARN_ON(i >= devices_cnt))
266	return -EINVAL;
267	}
268
269	return 0;
270	}
271
272	int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
273	struct dmar_dev_scope *devices, int count)
274	{
275	int index;
276	struct device *tmp;
277
278	if (info->seg != segment)
279	return 0;
280
281	for_each_active_dev_scope(devices, count, index, tmp)
282	if (tmp == &info->dev->dev) {
283	RCU_INIT_POINTER(devices[index].dev, NULL);
284	synchronize_rcu();
285	put_device(dev: tmp);
286	return 1;
287	}
288
289	return 0;
290	}
291
292	static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
293	{
294	int ret = 0;
295	struct dmar_drhd_unit *dmaru;
296	struct acpi_dmar_hardware_unit *drhd;
297
298	for_each_drhd_unit(dmaru) {
299	if (dmaru->include_all)
300	continue;
301
302	drhd = container_of(dmaru->hdr,
303	struct acpi_dmar_hardware_unit, header);
304	ret = dmar_insert_dev_scope(info, start: (void *)(drhd + 1),
305	end: ((void *)drhd) + drhd->header.length,
306	segment: dmaru->segment,
307	devices: dmaru->devices, devices_cnt: dmaru->devices_cnt);
308	if (ret)
309	break;
310	}
311	if (ret >= 0)
312	ret = dmar_iommu_notify_scope_dev(info);
313	if (ret < 0 && dmar_dev_scope_status == 0)
314	dmar_dev_scope_status = ret;
315
316	if (ret >= 0)
317	intel_irq_remap_add_device(info);
318
319	return ret;
320	}
321
322	static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
323	{
324	struct dmar_drhd_unit *dmaru;
325
326	for_each_drhd_unit(dmaru)
327	if (dmar_remove_dev_scope(info, segment: dmaru->segment,
328	devices: dmaru->devices, count: dmaru->devices_cnt))
329	break;
330	dmar_iommu_notify_scope_dev(info);
331	}
332
333	static inline void vf_inherit_msi_domain(struct pci_dev *pdev)
334	{
335	struct pci_dev *physfn = pci_physfn(dev: pdev);
336
337	dev_set_msi_domain(dev: &pdev->dev, d: dev_get_msi_domain(dev: &physfn->dev));
338	}
339
340	static int dmar_pci_bus_notifier(struct notifier_block *nb,
341	unsigned long action, void *data)
342	{
343	struct pci_dev *pdev = to_pci_dev(data);
344	struct dmar_pci_notify_info *info;
345
346	/* Only care about add/remove events for physical functions.
347	* For VFs we actually do the lookup based on the corresponding
348	* PF in device_to_iommu() anyway. */
349	if (pdev->is_virtfn) {
350	/*
351	* Ensure that the VF device inherits the irq domain of the
352	* PF device. Ideally the device would inherit the domain
353	* from the bus, but DMAR can have multiple units per bus
354	* which makes this impossible. The VF 'bus' could inherit
355	* from the PF device, but that's yet another x86'sism to
356	* inflict on everybody else.
357	*/
358	if (action == BUS_NOTIFY_ADD_DEVICE)
359	vf_inherit_msi_domain(pdev);
360	return NOTIFY_DONE;
361	}
362
363	if (action != BUS_NOTIFY_ADD_DEVICE &&
364	action != BUS_NOTIFY_REMOVED_DEVICE)
365	return NOTIFY_DONE;
366
367	info = dmar_alloc_pci_notify_info(dev: pdev, event: action);
368	if (!info)
369	return NOTIFY_DONE;
370
371	down_write(sem: &dmar_global_lock);
372	if (action == BUS_NOTIFY_ADD_DEVICE)
373	dmar_pci_bus_add_dev(info);
374	else if (action == BUS_NOTIFY_REMOVED_DEVICE)
375	dmar_pci_bus_del_dev(info);
376	up_write(sem: &dmar_global_lock);
377
378	dmar_free_pci_notify_info(info);
379
380	return NOTIFY_OK;
381	}
382
383	static struct notifier_block dmar_pci_bus_nb = {
384	.notifier_call = dmar_pci_bus_notifier,
385	.priority = 1,
386	};
387
388	static struct dmar_drhd_unit *
389	dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
390	{
391	struct dmar_drhd_unit *dmaru;
392
393	list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list,
394	dmar_rcu_check())
395	if (dmaru->segment == drhd->segment &&
396	dmaru->reg_base_addr == drhd->address)
397	return dmaru;
398
399	return NULL;
400	}
401
402	/*
403	* dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
404	* structure which uniquely represent one DMA remapping hardware unit
405	* present in the platform
406	*/
407	static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
408	{
409	struct acpi_dmar_hardware_unit *drhd;
410	struct dmar_drhd_unit *dmaru;
411	int ret;
412
413	drhd = (struct acpi_dmar_hardware_unit *)header;
414	dmaru = dmar_find_dmaru(drhd);
415	if (dmaru)
416	goto out;
417
418	dmaru = kzalloc(size: sizeof(*dmaru) + header->length, GFP_KERNEL);
419	if (!dmaru)
420	return -ENOMEM;
421
422	/*
423	* If header is allocated from slab by ACPI _DSM method, we need to
424	* copy the content because the memory buffer will be freed on return.
425	*/
426	dmaru->hdr = (void *)(dmaru + 1);
427	memcpy(dmaru->hdr, header, header->length);
428	dmaru->reg_base_addr = drhd->address;
429	dmaru->segment = drhd->segment;
430	/* The size of the register set is 2 ^ N 4 KB pages. */
431	dmaru->reg_size = 1UL << (drhd->size + 12);
432	dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
433	dmaru->devices = dmar_alloc_dev_scope(start: (void *)(drhd + 1),
434	end: ((void *)drhd) + drhd->header.length,
435	cnt: &dmaru->devices_cnt);
436	if (dmaru->devices_cnt && dmaru->devices == NULL) {
437	kfree(objp: dmaru);
438	return -ENOMEM;
439	}
440
441	ret = alloc_iommu(drhd: dmaru);
442	if (ret) {
443	dmar_free_dev_scope(devices: &dmaru->devices,
444	cnt: &dmaru->devices_cnt);
445	kfree(objp: dmaru);
446	return ret;
447	}
448	dmar_register_drhd_unit(drhd: dmaru);
449
450	out:
451	if (arg)
452	(*(int *)arg)++;
453
454	return 0;
455	}
456
457	static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
458	{
459	if (dmaru->devices && dmaru->devices_cnt)
460	dmar_free_dev_scope(devices: &dmaru->devices, cnt: &dmaru->devices_cnt);
461	if (dmaru->iommu)
462	free_iommu(iommu: dmaru->iommu);
463	kfree(objp: dmaru);
464	}
465
466	static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
467	void *arg)
468	{
469	struct acpi_dmar_andd *andd = (void *)header;
470
471	/* Check for NUL termination within the designated length */
472	if (strnlen(p: andd->device_name, maxlen: header->length - 8) == header->length - 8) {
473	pr_warn(FW_BUG
474	"Your BIOS is broken; ANDD object name is not NUL-terminated\n"
475	"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
476	dmi_get_system_info(DMI_BIOS_VENDOR),
477	dmi_get_system_info(DMI_BIOS_VERSION),
478	dmi_get_system_info(DMI_PRODUCT_VERSION));
479	add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
480	return -EINVAL;
481	}
482	pr_info("ANDD device: %x name: %s\n", andd->device_number,
483	andd->device_name);
484
485	return 0;
486	}
487
488	#ifdef CONFIG_ACPI_NUMA
489	static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
490	{
491	struct acpi_dmar_rhsa *rhsa;
492	struct dmar_drhd_unit *drhd;
493
494	rhsa = (struct acpi_dmar_rhsa *)header;
495	for_each_drhd_unit(drhd) {
496	if (drhd->reg_base_addr == rhsa->base_address) {
497	int node = pxm_to_node(rhsa->proximity_domain);
498
499	if (node != NUMA_NO_NODE && !node_online(node))
500	node = NUMA_NO_NODE;
501	drhd->iommu->node = node;
502	return 0;
503	}
504	}
505	pr_warn(FW_BUG
506	"Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
507	"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
508	rhsa->base_address,
509	dmi_get_system_info(DMI_BIOS_VENDOR),
510	dmi_get_system_info(DMI_BIOS_VERSION),
511	dmi_get_system_info(DMI_PRODUCT_VERSION));
512	add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
513
514	return 0;
515	}
516	#else
517	#define dmar_parse_one_rhsa dmar_res_noop
518	#endif
519
520	static void
521	dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
522	{
523	struct acpi_dmar_hardware_unit *drhd;
524	struct acpi_dmar_reserved_memory *rmrr;
525	struct acpi_dmar_atsr *atsr;
526	struct acpi_dmar_rhsa *rhsa;
527	struct acpi_dmar_satc *satc;
528
529	switch (header->type) {
530	case ACPI_DMAR_TYPE_HARDWARE_UNIT:
531	drhd = container_of(header, struct acpi_dmar_hardware_unit,
532	header);
533	pr_info("DRHD base: %#016Lx flags: %#x\n",
534	(unsigned long long)drhd->address, drhd->flags);
535	break;
536	case ACPI_DMAR_TYPE_RESERVED_MEMORY:
537	rmrr = container_of(header, struct acpi_dmar_reserved_memory,
538	header);
539	pr_info("RMRR base: %#016Lx end: %#016Lx\n",
540	(unsigned long long)rmrr->base_address,
541	(unsigned long long)rmrr->end_address);
542	break;
543	case ACPI_DMAR_TYPE_ROOT_ATS:
544	atsr = container_of(header, struct acpi_dmar_atsr, header);
545	pr_info("ATSR flags: %#x\n", atsr->flags);
546	break;
547	case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
548	rhsa = container_of(header, struct acpi_dmar_rhsa, header);
549	pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
550	(unsigned long long)rhsa->base_address,
551	rhsa->proximity_domain);
552	break;
553	case ACPI_DMAR_TYPE_NAMESPACE:
554	/* We don't print this here because we need to sanity-check
555	it first. So print it in dmar_parse_one_andd() instead. */
556	break;
557	case ACPI_DMAR_TYPE_SATC:
558	satc = container_of(header, struct acpi_dmar_satc, header);
559	pr_info("SATC flags: 0x%x\n", satc->flags);
560	break;
561	}
562	}
563
564	/**
565	* dmar_table_detect - checks to see if the platform supports DMAR devices
566	*/
567	static int __init dmar_table_detect(void)
568	{
569	acpi_status status = AE_OK;
570
571	/* if we could find DMAR table, then there are DMAR devices */
572	status = acpi_get_table(ACPI_SIG_DMAR, instance: 0, out_table: &dmar_tbl);
573
574	if (ACPI_SUCCESS(status) && !dmar_tbl) {
575	pr_warn("Unable to map DMAR\n");
576	status = AE_NOT_FOUND;
577	}
578
579	return ACPI_SUCCESS(status) ? 0 : -ENOENT;
580	}
581
582	static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
583	size_t len, struct dmar_res_callback *cb)
584	{
585	struct acpi_dmar_header *iter, *next;
586	struct acpi_dmar_header *end = ((void *)start) + len;
587
588	for (iter = start; iter < end; iter = next) {
589	next = (void *)iter + iter->length;
590	if (iter->length == 0) {
591	/* Avoid looping forever on bad ACPI tables */
592	pr_debug(FW_BUG "Invalid 0-length structure\n");
593	break;
594	} else if (next > end) {
595	/* Avoid passing table end */
596	pr_warn(FW_BUG "Record passes table end\n");
597	return -EINVAL;
598	}
599
600	if (cb->print_entry)
601	dmar_table_print_dmar_entry(header: iter);
602
603	if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
604	/* continue for forward compatibility */
605	pr_debug("Unknown DMAR structure type %d\n",
606	iter->type);
607	} else if (cb->cb[iter->type]) {
608	int ret;
609
610	ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
611	if (ret)
612	return ret;
613	} else if (!cb->ignore_unhandled) {
614	pr_warn("No handler for DMAR structure type %d\n",
615	iter->type);
616	return -EINVAL;
617	}
618	}
619
620	return 0;
621	}
622
623	static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
624	struct dmar_res_callback *cb)
625	{
626	return dmar_walk_remapping_entries(start: (void *)(dmar + 1),
627	len: dmar->header.length - sizeof(*dmar), cb);
628	}
629
630	/**
631	* parse_dmar_table - parses the DMA reporting table
632	*/
633	static int __init
634	parse_dmar_table(void)
635	{
636	struct acpi_table_dmar *dmar;
637	int drhd_count = 0;
638	int ret;
639	struct dmar_res_callback cb = {
640	.print_entry = true,
641	.ignore_unhandled = true,
642	.arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
643	.cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
644	.cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
645	.cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
646	.cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
647	.cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
648	.cb[ACPI_DMAR_TYPE_SATC] = &dmar_parse_one_satc,
649	};
650
651	/*
652	* Do it again, earlier dmar_tbl mapping could be mapped with
653	* fixed map.
654	*/
655	dmar_table_detect();
656
657	/*
658	* ACPI tables may not be DMA protected by tboot, so use DMAR copy
659	* SINIT saved in SinitMleData in TXT heap (which is DMA protected)
660	*/
661	dmar_tbl = tboot_get_dmar_table(dmar_tbl);
662
663	dmar = (struct acpi_table_dmar *)dmar_tbl;
664	if (!dmar)
665	return -ENODEV;
666
667	if (dmar->width < PAGE_SHIFT - 1) {
668	pr_warn("Invalid DMAR haw\n");
669	return -EINVAL;
670	}
671
672	pr_info("Host address width %d\n", dmar->width + 1);
673	ret = dmar_walk_dmar_table(dmar, cb: &cb);
674	if (ret == 0 && drhd_count == 0)
675	pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
676
677	return ret;
678	}
679
680	static int dmar_pci_device_match(struct dmar_dev_scope devices[],
681	int cnt, struct pci_dev *dev)
682	{
683	int index;
684	struct device *tmp;
685
686	while (dev) {
687	for_each_active_dev_scope(devices, cnt, index, tmp)
688	if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
689	return 1;
690
691	/* Check our parent */
692	dev = dev->bus->self;
693	}
694
695	return 0;
696	}
697
698	struct dmar_drhd_unit *
699	dmar_find_matched_drhd_unit(struct pci_dev *dev)
700	{
701	struct dmar_drhd_unit *dmaru;
702	struct acpi_dmar_hardware_unit *drhd;
703
704	dev = pci_physfn(dev);
705
706	rcu_read_lock();
707	for_each_drhd_unit(dmaru) {
708	drhd = container_of(dmaru->hdr,
709	struct acpi_dmar_hardware_unit,
710	header);
711
712	if (dmaru->include_all &&
713	drhd->segment == pci_domain_nr(bus: dev->bus))
714	goto out;
715
716	if (dmar_pci_device_match(devices: dmaru->devices,
717	cnt: dmaru->devices_cnt, dev))
718	goto out;
719	}
720	dmaru = NULL;
721	out:
722	rcu_read_unlock();
723
724	return dmaru;
725	}
726
727	static void __init dmar_acpi_insert_dev_scope(u8 device_number,
728	struct acpi_device *adev)
729	{
730	struct dmar_drhd_unit *dmaru;
731	struct acpi_dmar_hardware_unit *drhd;
732	struct acpi_dmar_device_scope *scope;
733	struct device *tmp;
734	int i;
735	struct acpi_dmar_pci_path *path;
736
737	for_each_drhd_unit(dmaru) {
738	drhd = container_of(dmaru->hdr,
739	struct acpi_dmar_hardware_unit,
740	header);
741
742	for (scope = (void *)(drhd + 1);
743	(unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
744	scope = ((void *)scope) + scope->length) {
745	if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
746	continue;
747	if (scope->enumeration_id != device_number)
748	continue;
749
750	path = (void *)(scope + 1);
751	pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
752	dev_name(&adev->dev), dmaru->reg_base_addr,
753	scope->bus, path->device, path->function);
754	for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
755	if (tmp == NULL) {
756	dmaru->devices[i].bus = scope->bus;
757	dmaru->devices[i].devfn = PCI_DEVFN(path->device,
758	path->function);
759	rcu_assign_pointer(dmaru->devices[i].dev,
760	get_device(&adev->dev));
761	return;
762	}
763	BUG_ON(i >= dmaru->devices_cnt);
764	}
765	}
766	pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
767	device_number, dev_name(&adev->dev));
768	}
769
770	static int __init dmar_acpi_dev_scope_init(void)
771	{
772	struct acpi_dmar_andd *andd;
773
774	if (dmar_tbl == NULL)
775	return -ENODEV;
776
777	for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
778	((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
779	andd = ((void *)andd) + andd->header.length) {
780	if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
781	acpi_handle h;
782	struct acpi_device *adev;
783
784	if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
785	andd->device_name,
786	&h))) {
787	pr_err("Failed to find handle for ACPI object %s\n",
788	andd->device_name);
789	continue;
790	}
791	adev = acpi_fetch_acpi_dev(handle: h);
792	if (!adev) {
793	pr_err("Failed to get device for ACPI object %s\n",
794	andd->device_name);
795	continue;
796	}
797	dmar_acpi_insert_dev_scope(device_number: andd->device_number, adev);
798	}
799	}
800	return 0;
801	}
802
803	int __init dmar_dev_scope_init(void)
804	{
805	struct pci_dev *dev = NULL;
806	struct dmar_pci_notify_info *info;
807
808	if (dmar_dev_scope_status != 1)
809	return dmar_dev_scope_status;
810
811	if (list_empty(head: &dmar_drhd_units)) {
812	dmar_dev_scope_status = -ENODEV;
813	} else {
814	dmar_dev_scope_status = 0;
815
816	dmar_acpi_dev_scope_init();
817
818	for_each_pci_dev(dev) {
819	if (dev->is_virtfn)
820	continue;
821
822	info = dmar_alloc_pci_notify_info(dev,
823	event: BUS_NOTIFY_ADD_DEVICE);
824	if (!info) {
825	pci_dev_put(dev);
826	return dmar_dev_scope_status;
827	} else {
828	dmar_pci_bus_add_dev(info);
829	dmar_free_pci_notify_info(info);
830	}
831	}
832	}
833
834	return dmar_dev_scope_status;
835	}
836
837	void __init dmar_register_bus_notifier(void)
838	{
839	bus_register_notifier(bus: &pci_bus_type, nb: &dmar_pci_bus_nb);
840	}
841
842
843	int __init dmar_table_init(void)
844	{
845	static int dmar_table_initialized;
846	int ret;
847
848	if (dmar_table_initialized == 0) {
849	ret = parse_dmar_table();
850	if (ret < 0) {
851	if (ret != -ENODEV)
852	pr_info("Parse DMAR table failure.\n");
853	} else if (list_empty(head: &dmar_drhd_units)) {
854	pr_info("No DMAR devices found\n");
855	ret = -ENODEV;
856	}
857
858	if (ret < 0)
859	dmar_table_initialized = ret;
860	else
861	dmar_table_initialized = 1;
862	}
863
864	return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
865	}
866
867	static void warn_invalid_dmar(u64 addr, const char *message)
868	{
869	pr_warn_once(FW_BUG
870	"Your BIOS is broken; DMAR reported at address %llx%s!\n"
871	"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
872	addr, message,
873	dmi_get_system_info(DMI_BIOS_VENDOR),
874	dmi_get_system_info(DMI_BIOS_VERSION),
875	dmi_get_system_info(DMI_PRODUCT_VERSION));
876	add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
877	}
878
879	static int __ref
880	dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
881	{
882	struct acpi_dmar_hardware_unit *drhd;
883	void __iomem *addr;
884	u64 cap, ecap;
885
886	drhd = (void *)entry;
887	if (!drhd->address) {
888	warn_invalid_dmar(addr: 0, message: "");
889	return -EINVAL;
890	}
891
892	if (arg)
893	addr = ioremap(offset: drhd->address, VTD_PAGE_SIZE);
894	else
895	addr = early_ioremap(phys_addr: drhd->address, VTD_PAGE_SIZE);
896	if (!addr) {
897	pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
898	return -EINVAL;
899	}
900
901	cap = dmar_readq(addr + DMAR_CAP_REG);
902	ecap = dmar_readq(addr + DMAR_ECAP_REG);
903
904	if (arg)
905	iounmap(addr);
906	else
907	early_iounmap(addr, VTD_PAGE_SIZE);
908
909	if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
910	warn_invalid_dmar(addr: drhd->address, message: " returns all ones");
911	return -EINVAL;
912	}
913
914	return 0;
915	}
916
917	void __init detect_intel_iommu(void)
918	{
919	int ret;
920	struct dmar_res_callback validate_drhd_cb = {
921	.cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
922	.ignore_unhandled = true,
923	};
924
925	down_write(sem: &dmar_global_lock);
926	ret = dmar_table_detect();
927	if (!ret)
928	ret = dmar_walk_dmar_table(dmar: (struct acpi_table_dmar *)dmar_tbl,
929	cb: &validate_drhd_cb);
930	if (!ret && !no_iommu && !iommu_detected &&
931	(!dmar_disabled || dmar_platform_optin())) {
932	iommu_detected = 1;
933	/* Make sure ACS will be enabled */
934	pci_request_acs();
935	}
936
937	#ifdef CONFIG_X86
938	if (!ret) {
939	x86_init.iommu.iommu_init = intel_iommu_init;
940	x86_platform.iommu_shutdown = intel_iommu_shutdown;
941	}
942
943	#endif
944
945	if (dmar_tbl) {
946	acpi_put_table(table: dmar_tbl);
947	dmar_tbl = NULL;
948	}
949	up_write(sem: &dmar_global_lock);
950	}
951
952	static void unmap_iommu(struct intel_iommu *iommu)
953	{
954	iounmap(addr: iommu->reg);
955	release_mem_region(iommu->reg_phys, iommu->reg_size);
956	}
957
958	/**
959	* map_iommu: map the iommu's registers
960	* @iommu: the iommu to map
961	* @drhd: DMA remapping hardware definition structure
962	*
963	* Memory map the iommu's registers. Start w/ a single page, and
964	* possibly expand if that turns out to be insufficent.
965	*/
966	static int map_iommu(struct intel_iommu *iommu, struct dmar_drhd_unit *drhd)
967	{
968	u64 phys_addr = drhd->reg_base_addr;
969	int map_size, err=0;
970
971	iommu->reg_phys = phys_addr;
972	iommu->reg_size = drhd->reg_size;
973
974	if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
975	pr_err("Can't reserve memory\n");
976	err = -EBUSY;
977	goto out;
978	}
979
980	iommu->reg = ioremap(offset: iommu->reg_phys, size: iommu->reg_size);
981	if (!iommu->reg) {
982	pr_err("Can't map the region\n");
983	err = -ENOMEM;
984	goto release;
985	}
986
987	iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
988	iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
989
990	if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
991	err = -EINVAL;
992	warn_invalid_dmar(addr: phys_addr, message: " returns all ones");
993	goto unmap;
994	}
995
996	/* the registers might be more than one page */
997	map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
998	cap_max_fault_reg_offset(iommu->cap));
999	map_size = VTD_PAGE_ALIGN(map_size);
1000	if (map_size > iommu->reg_size) {
1001	iounmap(addr: iommu->reg);
1002	release_mem_region(iommu->reg_phys, iommu->reg_size);
1003	iommu->reg_size = map_size;
1004	if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
1005	iommu->name)) {
1006	pr_err("Can't reserve memory\n");
1007	err = -EBUSY;
1008	goto out;
1009	}
1010	iommu->reg = ioremap(offset: iommu->reg_phys, size: iommu->reg_size);
1011	if (!iommu->reg) {
1012	pr_err("Can't map the region\n");
1013	err = -ENOMEM;
1014	goto release;
1015	}
1016	}
1017
1018	if (cap_ecmds(iommu->cap)) {
1019	int i;
1020
1021	for (i = 0; i < DMA_MAX_NUM_ECMDCAP; i++) {
1022	iommu->ecmdcap[i] = dmar_readq(iommu->reg + DMAR_ECCAP_REG +
1023	i * DMA_ECMD_REG_STEP);
1024	}
1025	}
1026
1027	err = 0;
1028	goto out;
1029
1030	unmap:
1031	iounmap(addr: iommu->reg);
1032	release:
1033	release_mem_region(iommu->reg_phys, iommu->reg_size);
1034	out:
1035	return err;
1036	}
1037
1038	static int alloc_iommu(struct dmar_drhd_unit *drhd)
1039	{
1040	struct intel_iommu *iommu;
1041	u32 ver, sts;
1042	int agaw = -1;
1043	int msagaw = -1;
1044	int err;
1045
1046	if (!drhd->reg_base_addr) {
1047	warn_invalid_dmar(addr: 0, message: "");
1048	return -EINVAL;
1049	}
1050
1051	iommu = kzalloc(size: sizeof(*iommu), GFP_KERNEL);
1052	if (!iommu)
1053	return -ENOMEM;
1054
1055	iommu->seq_id = ida_alloc_range(&dmar_seq_ids, min: 0,
1056	DMAR_UNITS_SUPPORTED - 1, GFP_KERNEL);
1057	if (iommu->seq_id < 0) {
1058	pr_err("Failed to allocate seq_id\n");
1059	err = iommu->seq_id;
1060	goto error;
1061	}
1062	sprintf(buf: iommu->name, fmt: "dmar%d", iommu->seq_id);
1063
1064	err = map_iommu(iommu, drhd);
1065	if (err) {
1066	pr_err("Failed to map %s\n", iommu->name);
1067	goto error_free_seq_id;
1068	}
1069
1070	err = -EINVAL;
1071	if (!cap_sagaw(iommu->cap) &&
1072	(!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) {
1073	pr_info("%s: No supported address widths. Not attempting DMA translation.\n",
1074	iommu->name);
1075	drhd->ignored = 1;
1076	}
1077
1078	if (!drhd->ignored) {
1079	agaw = iommu_calculate_agaw(iommu);
1080	if (agaw < 0) {
1081	pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1082	iommu->seq_id);
1083	drhd->ignored = 1;
1084	}
1085	}
1086	if (!drhd->ignored) {
1087	msagaw = iommu_calculate_max_sagaw(iommu);
1088	if (msagaw < 0) {
1089	pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1090	iommu->seq_id);
1091	drhd->ignored = 1;
1092	agaw = -1;
1093	}
1094	}
1095	iommu->agaw = agaw;
1096	iommu->msagaw = msagaw;
1097	iommu->segment = drhd->segment;
1098	iommu->device_rbtree = RB_ROOT;
1099	spin_lock_init(&iommu->device_rbtree_lock);
1100	mutex_init(&iommu->iopf_lock);
1101	iommu->node = NUMA_NO_NODE;
1102
1103	ver = readl(addr: iommu->reg + DMAR_VER_REG);
1104	pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1105	iommu->name,
1106	(unsigned long long)drhd->reg_base_addr,
1107	DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1108	(unsigned long long)iommu->cap,
1109	(unsigned long long)iommu->ecap);
1110
1111	/* Reflect status in gcmd */
1112	sts = readl(addr: iommu->reg + DMAR_GSTS_REG);
1113	if (sts & DMA_GSTS_IRES)
1114	iommu->gcmd |= DMA_GCMD_IRE;
1115	if (sts & DMA_GSTS_TES)
1116	iommu->gcmd |= DMA_GCMD_TE;
1117	if (sts & DMA_GSTS_QIES)
1118	iommu->gcmd |= DMA_GCMD_QIE;
1119
1120	if (alloc_iommu_pmu(iommu))
1121	pr_debug("Cannot alloc PMU for iommu (seq_id = %d)\n", iommu->seq_id);
1122
1123	raw_spin_lock_init(&iommu->register_lock);
1124
1125	/*
1126	* A value of N in PSS field of eCap register indicates hardware
1127	* supports PASID field of N+1 bits.
1128	*/
1129	if (pasid_supported(iommu))
1130	iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);
1131
1132	/*
1133	* This is only for hotplug; at boot time intel_iommu_enabled won't
1134	* be set yet. When intel_iommu_init() runs, it registers the units
1135	* present at boot time, then sets intel_iommu_enabled.
1136	*/
1137	if (intel_iommu_enabled && !drhd->ignored) {
1138	err = iommu_device_sysfs_add(iommu: &iommu->iommu, NULL,
1139	groups: intel_iommu_groups,
1140	fmt: "%s", iommu->name);
1141	if (err)
1142	goto err_unmap;
1143
1144	err = iommu_device_register(iommu: &iommu->iommu, ops: &intel_iommu_ops, NULL);
1145	if (err)
1146	goto err_sysfs;
1147
1148	iommu_pmu_register(iommu);
1149	}
1150
1151	drhd->iommu = iommu;
1152	iommu->drhd = drhd;
1153
1154	return 0;
1155
1156	err_sysfs:
1157	iommu_device_sysfs_remove(iommu: &iommu->iommu);
1158	err_unmap:
1159	free_iommu_pmu(iommu);
1160	unmap_iommu(iommu);
1161	error_free_seq_id:
1162	ida_free(&dmar_seq_ids, id: iommu->seq_id);
1163	error:
1164	kfree(objp: iommu);
1165	return err;
1166	}
1167
1168	static void free_iommu(struct intel_iommu *iommu)
1169	{
1170	if (intel_iommu_enabled && !iommu->drhd->ignored) {
1171	iommu_pmu_unregister(iommu);
1172	iommu_device_unregister(iommu: &iommu->iommu);
1173	iommu_device_sysfs_remove(iommu: &iommu->iommu);
1174	}
1175
1176	free_iommu_pmu(iommu);
1177
1178	if (iommu->irq) {
1179	if (iommu->pr_irq) {
1180	free_irq(iommu->pr_irq, iommu);
1181	dmar_free_hwirq(irq: iommu->pr_irq);
1182	iommu->pr_irq = 0;
1183	}
1184	free_irq(iommu->irq, iommu);
1185	dmar_free_hwirq(irq: iommu->irq);
1186	iommu->irq = 0;
1187	}
1188
1189	if (iommu->qi) {
1190	free_page((unsigned long)iommu->qi->desc);
1191	kfree(objp: iommu->qi->desc_status);
1192	kfree(objp: iommu->qi);
1193	}
1194
1195	if (iommu->reg)
1196	unmap_iommu(iommu);
1197
1198	ida_free(&dmar_seq_ids, id: iommu->seq_id);
1199	kfree(objp: iommu);
1200	}
1201
1202	/*
1203	* Reclaim all the submitted descriptors which have completed its work.
1204	*/
1205	static inline void reclaim_free_desc(struct q_inval *qi)
1206	{
1207	while (qi->desc_status[qi->free_tail] == QI_DONE ||
1208	qi->desc_status[qi->free_tail] == QI_ABORT) {
1209	qi->desc_status[qi->free_tail] = QI_FREE;
1210	qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1211	qi->free_cnt++;
1212	}
1213	}
1214
1215	static const char *qi_type_string(u8 type)
1216	{
1217	switch (type) {
1218	case QI_CC_TYPE:
1219	return "Context-cache Invalidation";
1220	case QI_IOTLB_TYPE:
1221	return "IOTLB Invalidation";
1222	case QI_DIOTLB_TYPE:
1223	return "Device-TLB Invalidation";
1224	case QI_IEC_TYPE:
1225	return "Interrupt Entry Cache Invalidation";
1226	case QI_IWD_TYPE:
1227	return "Invalidation Wait";
1228	case QI_EIOTLB_TYPE:
1229	return "PASID-based IOTLB Invalidation";
1230	case QI_PC_TYPE:
1231	return "PASID-cache Invalidation";
1232	case QI_DEIOTLB_TYPE:
1233	return "PASID-based Device-TLB Invalidation";
1234	case QI_PGRP_RESP_TYPE:
1235	return "Page Group Response";
1236	default:
1237	return "UNKNOWN";
1238	}
1239	}
1240
1241	static void qi_dump_fault(struct intel_iommu *iommu, u32 fault)
1242	{
1243	unsigned int head = dmar_readl(iommu->reg + DMAR_IQH_REG);
1244	u64 iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG);
1245	struct qi_desc *desc = iommu->qi->desc + head;
1246
1247	if (fault & DMA_FSTS_IQE)
1248	pr_err("VT-d detected Invalidation Queue Error: Reason %llx",
1249	DMAR_IQER_REG_IQEI(iqe_err));
1250	if (fault & DMA_FSTS_ITE)
1251	pr_err("VT-d detected Invalidation Time-out Error: SID %llx",
1252	DMAR_IQER_REG_ITESID(iqe_err));
1253	if (fault & DMA_FSTS_ICE)
1254	pr_err("VT-d detected Invalidation Completion Error: SID %llx",
1255	DMAR_IQER_REG_ICESID(iqe_err));
1256
1257	pr_err("QI HEAD: %s qw0 = 0x%llx, qw1 = 0x%llx\n",
1258	qi_type_string(desc->qw0 & 0xf),
1259	(unsigned long long)desc->qw0,
1260	(unsigned long long)desc->qw1);
1261
1262	head = ((head >> qi_shift(iommu)) + QI_LENGTH - 1) % QI_LENGTH;
1263	head <<= qi_shift(iommu);
1264	desc = iommu->qi->desc + head;
1265
1266	pr_err("QI PRIOR: %s qw0 = 0x%llx, qw1 = 0x%llx\n",
1267	qi_type_string(desc->qw0 & 0xf),
1268	(unsigned long long)desc->qw0,
1269	(unsigned long long)desc->qw1);
1270	}
1271
1272	static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
1273	{
1274	u32 fault;
1275	int head, tail;
1276	struct device *dev;
1277	u64 iqe_err, ite_sid;
1278	struct q_inval *qi = iommu->qi;
1279	int shift = qi_shift(iommu);
1280
1281	if (qi->desc_status[wait_index] == QI_ABORT)
1282	return -EAGAIN;
1283
1284	fault = readl(addr: iommu->reg + DMAR_FSTS_REG);
1285	if (fault & (DMA_FSTS_IQE | DMA_FSTS_ITE | DMA_FSTS_ICE))
1286	qi_dump_fault(iommu, fault);
1287
1288	/*
1289	* If IQE happens, the head points to the descriptor associated
1290	* with the error. No new descriptors are fetched until the IQE
1291	* is cleared.
1292	*/
1293	if (fault & DMA_FSTS_IQE) {
1294	head = readl(addr: iommu->reg + DMAR_IQH_REG);
1295	if ((head >> shift) == index) {
1296	struct qi_desc *desc = qi->desc + head;
1297
1298	/*
1299	* desc->qw2 and desc->qw3 are either reserved or
1300	* used by software as private data. We won't print
1301	* out these two qw's for security consideration.
1302	*/
1303	memcpy(desc, qi->desc + (wait_index << shift),
1304	1 << shift);
1305	writel(DMA_FSTS_IQE, addr: iommu->reg + DMAR_FSTS_REG);
1306	pr_info("Invalidation Queue Error (IQE) cleared\n");
1307	return -EINVAL;
1308	}
1309	}
1310
1311	/*
1312	* If ITE happens, all pending wait_desc commands are aborted.
1313	* No new descriptors are fetched until the ITE is cleared.
1314	*/
1315	if (fault & DMA_FSTS_ITE) {
1316	head = readl(addr: iommu->reg + DMAR_IQH_REG);
1317	head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1318	head |= 1;
1319	tail = readl(addr: iommu->reg + DMAR_IQT_REG);
1320	tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1321
1322	/*
1323	* SID field is valid only when the ITE field is Set in FSTS_REG
1324	* see Intel VT-d spec r4.1, section 11.4.9.9
1325	*/
1326	iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG);
1327	ite_sid = DMAR_IQER_REG_ITESID(iqe_err);
1328
1329	writel(DMA_FSTS_ITE, addr: iommu->reg + DMAR_FSTS_REG);
1330	pr_info("Invalidation Time-out Error (ITE) cleared\n");
1331
1332	do {
1333	if (qi->desc_status[head] == QI_IN_USE)
1334	qi->desc_status[head] = QI_ABORT;
1335	head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1336	} while (head != tail);
1337
1338	/*
1339	* If device was released or isn't present, no need to retry
1340	* the ATS invalidate request anymore.
1341	*
1342	* 0 value of ite_sid means old VT-d device, no ite_sid value.
1343	* see Intel VT-d spec r4.1, section 11.4.9.9
1344	*/
1345	if (ite_sid) {
1346	dev = device_rbtree_find(iommu, rid: ite_sid);
1347	if (!dev || !dev_is_pci(dev) ||
1348	!pci_device_is_present(to_pci_dev(dev)))
1349	return -ETIMEDOUT;
1350	}
1351	if (qi->desc_status[wait_index] == QI_ABORT)
1352	return -EAGAIN;
1353	}
1354
1355	if (fault & DMA_FSTS_ICE) {
1356	writel(DMA_FSTS_ICE, addr: iommu->reg + DMAR_FSTS_REG);
1357	pr_info("Invalidation Completion Error (ICE) cleared\n");
1358	}
1359
1360	return 0;
1361	}
1362
1363	/*
1364	* Function to submit invalidation descriptors of all types to the queued
1365	* invalidation interface(QI). Multiple descriptors can be submitted at a
1366	* time, a wait descriptor will be appended to each submission to ensure
1367	* hardware has completed the invalidation before return. Wait descriptors
1368	* can be part of the submission but it will not be polled for completion.
1369	*/
1370	int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
1371	unsigned int count, unsigned long options)
1372	{
1373	struct q_inval *qi = iommu->qi;
1374	s64 devtlb_start_ktime = 0;
1375	s64 iotlb_start_ktime = 0;
1376	s64 iec_start_ktime = 0;
1377	struct qi_desc wait_desc;
1378	int wait_index, index;
1379	unsigned long flags;
1380	int offset, shift;
1381	int rc, i;
1382	u64 type;
1383
1384	if (!qi)
1385	return 0;
1386
1387	type = desc->qw0 & GENMASK_ULL(3, 0);
1388
1389	if ((type == QI_IOTLB_TYPE || type == QI_EIOTLB_TYPE) &&
1390	dmar_latency_enabled(iommu, type: DMAR_LATENCY_INV_IOTLB))
1391	iotlb_start_ktime = ktime_to_ns(kt: ktime_get());
1392
1393	if ((type == QI_DIOTLB_TYPE || type == QI_DEIOTLB_TYPE) &&
1394	dmar_latency_enabled(iommu, type: DMAR_LATENCY_INV_DEVTLB))
1395	devtlb_start_ktime = ktime_to_ns(kt: ktime_get());
1396
1397	if (type == QI_IEC_TYPE &&
1398	dmar_latency_enabled(iommu, type: DMAR_LATENCY_INV_IEC))
1399	iec_start_ktime = ktime_to_ns(kt: ktime_get());
1400
1401	restart:
1402	rc = 0;
1403
1404	raw_spin_lock_irqsave(&qi->q_lock, flags);
1405	/*
1406	* Check if we have enough empty slots in the queue to submit,
1407	* the calculation is based on:
1408	* # of desc + 1 wait desc + 1 space between head and tail
1409	*/
1410	while (qi->free_cnt < count + 2) {
1411	raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1412	cpu_relax();
1413	raw_spin_lock_irqsave(&qi->q_lock, flags);
1414	}
1415
1416	index = qi->free_head;
1417	wait_index = (index + count) % QI_LENGTH;
1418	shift = qi_shift(iommu);
1419
1420	for (i = 0; i < count; i++) {
1421	offset = ((index + i) % QI_LENGTH) << shift;
1422	memcpy(qi->desc + offset, &desc[i], 1 << shift);
1423	qi->desc_status[(index + i) % QI_LENGTH] = QI_IN_USE;
1424	trace_qi_submit(iommu, qw0: desc[i].qw0, qw1: desc[i].qw1,
1425	qw2: desc[i].qw2, qw3: desc[i].qw3);
1426	}
1427	qi->desc_status[wait_index] = QI_IN_USE;
1428
1429	wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
1430	QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1431	if (options & QI_OPT_WAIT_DRAIN)
1432	wait_desc.qw0 |= QI_IWD_PRQ_DRAIN;
1433	wait_desc.qw1 = virt_to_phys(address: &qi->desc_status[wait_index]);
1434	wait_desc.qw2 = 0;
1435	wait_desc.qw3 = 0;
1436
1437	offset = wait_index << shift;
1438	memcpy(qi->desc + offset, &wait_desc, 1 << shift);
1439
1440	qi->free_head = (qi->free_head + count + 1) % QI_LENGTH;
1441	qi->free_cnt -= count + 1;
1442
1443	/*
1444	* update the HW tail register indicating the presence of
1445	* new descriptors.
1446	*/
1447	writel(val: qi->free_head << shift, addr: iommu->reg + DMAR_IQT_REG);
1448
1449	while (qi->desc_status[wait_index] != QI_DONE) {
1450	/*
1451	* We will leave the interrupts disabled, to prevent interrupt
1452	* context to queue another cmd while a cmd is already submitted
1453	* and waiting for completion on this cpu. This is to avoid
1454	* a deadlock where the interrupt context can wait indefinitely
1455	* for free slots in the queue.
1456	*/
1457	rc = qi_check_fault(iommu, index, wait_index);
1458	if (rc)
1459	break;
1460
1461	raw_spin_unlock(&qi->q_lock);
1462	cpu_relax();
1463	raw_spin_lock(&qi->q_lock);
1464	}
1465
1466	for (i = 0; i < count; i++)
1467	qi->desc_status[(index + i) % QI_LENGTH] = QI_DONE;
1468
1469	reclaim_free_desc(qi);
1470	raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1471
1472	if (rc == -EAGAIN)
1473	goto restart;
1474
1475	if (iotlb_start_ktime)
1476	dmar_latency_update(iommu, type: DMAR_LATENCY_INV_IOTLB,
1477	latency: ktime_to_ns(kt: ktime_get()) - iotlb_start_ktime);
1478
1479	if (devtlb_start_ktime)
1480	dmar_latency_update(iommu, type: DMAR_LATENCY_INV_DEVTLB,
1481	latency: ktime_to_ns(kt: ktime_get()) - devtlb_start_ktime);
1482
1483	if (iec_start_ktime)
1484	dmar_latency_update(iommu, type: DMAR_LATENCY_INV_IEC,
1485	latency: ktime_to_ns(kt: ktime_get()) - iec_start_ktime);
1486
1487	return rc;
1488	}
1489
1490	/*
1491	* Flush the global interrupt entry cache.
1492	*/
1493	void qi_global_iec(struct intel_iommu *iommu)
1494	{
1495	struct qi_desc desc;
1496
1497	desc.qw0 = QI_IEC_TYPE;
1498	desc.qw1 = 0;
1499	desc.qw2 = 0;
1500	desc.qw3 = 0;
1501
1502	/* should never fail */
1503	qi_submit_sync(iommu, desc: &desc, count: 1, options: 0);
1504	}
1505
1506	void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1507	u64 type)
1508	{
1509	struct qi_desc desc;
1510
1511	desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1512	| QI_CC_GRAN(type) | QI_CC_TYPE;
1513	desc.qw1 = 0;
1514	desc.qw2 = 0;
1515	desc.qw3 = 0;
1516
1517	qi_submit_sync(iommu, desc: &desc, count: 1, options: 0);
1518	}
1519
1520	void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1521	unsigned int size_order, u64 type)
1522	{
1523	u8 dw = 0, dr = 0;
1524
1525	struct qi_desc desc;
1526	int ih = 0;
1527
1528	if (cap_write_drain(iommu->cap))
1529	dw = 1;
1530
1531	if (cap_read_drain(iommu->cap))
1532	dr = 1;
1533
1534	desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1535	| QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1536	desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1537	| QI_IOTLB_AM(size_order);
1538	desc.qw2 = 0;
1539	desc.qw3 = 0;
1540
1541	qi_submit_sync(iommu, desc: &desc, count: 1, options: 0);
1542	}
1543
1544	void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1545	u16 qdep, u64 addr, unsigned mask)
1546	{
1547	struct qi_desc desc;
1548
1549	/*
1550	* VT-d spec, section 4.3:
1551	*
1552	* Software is recommended to not submit any Device-TLB invalidation
1553	* requests while address remapping hardware is disabled.
1554	*/
1555	if (!(iommu->gcmd & DMA_GCMD_TE))
1556	return;
1557
1558	if (mask) {
1559	addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1560	desc.qw1 = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1561	} else
1562	desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
1563
1564	if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1565	qdep = 0;
1566
1567	desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1568	QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
1569	desc.qw2 = 0;
1570	desc.qw3 = 0;
1571
1572	qi_submit_sync(iommu, desc: &desc, count: 1, options: 0);
1573	}
1574
1575	/* PASID-based IOTLB invalidation */
1576	void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
1577	unsigned long npages, bool ih)
1578	{
1579	struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
1580
1581	/*
1582	* npages == -1 means a PASID-selective invalidation, otherwise,
1583	* a positive value for Page-selective-within-PASID invalidation.
1584	* 0 is not a valid input.
1585	*/
1586	if (WARN_ON(!npages)) {
1587	pr_err("Invalid input npages = %ld\n", npages);
1588	return;
1589	}
1590
1591	if (npages == -1) {
1592	desc.qw0 = QI_EIOTLB_PASID(pasid) |
1593	QI_EIOTLB_DID(did) |
1594	QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |
1595	QI_EIOTLB_TYPE;
1596	desc.qw1 = 0;
1597	} else {
1598	int mask = ilog2(__roundup_pow_of_two(npages));
1599	unsigned long align = (1ULL << (VTD_PAGE_SHIFT + mask));
1600
1601	if (WARN_ON_ONCE(!IS_ALIGNED(addr, align)))
1602	addr = ALIGN_DOWN(addr, align);
1603
1604	desc.qw0 = QI_EIOTLB_PASID(pasid) |
1605	QI_EIOTLB_DID(did) |
1606	QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
1607	QI_EIOTLB_TYPE;
1608	desc.qw1 = QI_EIOTLB_ADDR(addr) |
1609	QI_EIOTLB_IH(ih) |
1610	QI_EIOTLB_AM(mask);
1611	}
1612
1613	qi_submit_sync(iommu, desc: &desc, count: 1, options: 0);
1614	}
1615
1616	/* PASID-based device IOTLB Invalidate */
1617	void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1618	u32 pasid, u16 qdep, u64 addr, unsigned int size_order)
1619	{
1620	unsigned long mask = 1UL << (VTD_PAGE_SHIFT + size_order - 1);
1621	struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1622
1623	/*
1624	* VT-d spec, section 4.3:
1625	*
1626	* Software is recommended to not submit any Device-TLB invalidation
1627	* requests while address remapping hardware is disabled.
1628	*/
1629	if (!(iommu->gcmd & DMA_GCMD_TE))
1630	return;
1631
1632	desc.qw0 = QI_DEV_EIOTLB_PASID(pasid) | QI_DEV_EIOTLB_SID(sid) |
1633	QI_DEV_EIOTLB_QDEP(qdep) | QI_DEIOTLB_TYPE |
1634	QI_DEV_IOTLB_PFSID(pfsid);
1635
1636	/*
1637	* If S bit is 0, we only flush a single page. If S bit is set,
1638	* The least significant zero bit indicates the invalidation address
1639	* range. VT-d spec 6.5.2.6.
1640	* e.g. address bit 12[0] indicates 8KB, 13[0] indicates 16KB.
1641	* size order = 0 is PAGE_SIZE 4KB
1642	* Max Invs Pending (MIP) is set to 0 for now until we have DIT in
1643	* ECAP.
1644	*/
1645	if (!IS_ALIGNED(addr, VTD_PAGE_SIZE << size_order))
1646	pr_warn_ratelimited("Invalidate non-aligned address %llx, order %d\n",
1647	addr, size_order);
1648
1649	/* Take page address */
1650	desc.qw1 = QI_DEV_EIOTLB_ADDR(addr);
1651
1652	if (size_order) {
1653	/*
1654	* Existing 0s in address below size_order may be the least
1655	* significant bit, we must set them to 1s to avoid having
1656	* smaller size than desired.
1657	*/
1658	desc.qw1 |= GENMASK_ULL(size_order + VTD_PAGE_SHIFT - 1,
1659	VTD_PAGE_SHIFT);
1660	/* Clear size_order bit to indicate size */
1661	desc.qw1 &= ~mask;
1662	/* Set the S bit to indicate flushing more than 1 page */
1663	desc.qw1 |= QI_DEV_EIOTLB_SIZE;
1664	}
1665
1666	qi_submit_sync(iommu, desc: &desc, count: 1, options: 0);
1667	}
1668
1669	void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,
1670	u64 granu, u32 pasid)
1671	{
1672	struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1673
1674	desc.qw0 = QI_PC_PASID(pasid) | QI_PC_DID(did) |
1675	QI_PC_GRAN(granu) | QI_PC_TYPE;
1676	qi_submit_sync(iommu, desc: &desc, count: 1, options: 0);
1677	}
1678
1679	/*
1680	* Disable Queued Invalidation interface.
1681	*/
1682	void dmar_disable_qi(struct intel_iommu *iommu)
1683	{
1684	unsigned long flags;
1685	u32 sts;
1686	cycles_t start_time = get_cycles();
1687
1688	if (!ecap_qis(iommu->ecap))
1689	return;
1690
1691	raw_spin_lock_irqsave(&iommu->register_lock, flags);
1692
1693	sts = readl(addr: iommu->reg + DMAR_GSTS_REG);
1694	if (!(sts & DMA_GSTS_QIES))
1695	goto end;
1696
1697	/*
1698	* Give a chance to HW to complete the pending invalidation requests.
1699	*/
1700	while ((readl(addr: iommu->reg + DMAR_IQT_REG) !=
1701	readl(addr: iommu->reg + DMAR_IQH_REG)) &&
1702	(DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1703	cpu_relax();
1704
1705	iommu->gcmd &= ~DMA_GCMD_QIE;
1706	writel(val: iommu->gcmd, addr: iommu->reg + DMAR_GCMD_REG);
1707
1708	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1709	!(sts & DMA_GSTS_QIES), sts);
1710	end:
1711	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1712	}
1713
1714	/*
1715	* Enable queued invalidation.
1716	*/
1717	static void __dmar_enable_qi(struct intel_iommu *iommu)
1718	{
1719	u32 sts;
1720	unsigned long flags;
1721	struct q_inval *qi = iommu->qi;
1722	u64 val = virt_to_phys(address: qi->desc);
1723
1724	qi->free_head = qi->free_tail = 0;
1725	qi->free_cnt = QI_LENGTH;
1726
1727	/*
1728	* Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
1729	* is present.
1730	*/
1731	if (ecap_smts(iommu->ecap))
1732	val |= BIT_ULL(11) | BIT_ULL(0);
1733
1734	raw_spin_lock_irqsave(&iommu->register_lock, flags);
1735
1736	/* write zero to the tail reg */
1737	writel(val: 0, addr: iommu->reg + DMAR_IQT_REG);
1738
1739	dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
1740
1741	iommu->gcmd |= DMA_GCMD_QIE;
1742	writel(val: iommu->gcmd, addr: iommu->reg + DMAR_GCMD_REG);
1743
1744	/* Make sure hardware complete it */
1745	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1746
1747	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1748	}
1749
1750	/*
1751	* Enable Queued Invalidation interface. This is a must to support
1752	* interrupt-remapping. Also used by DMA-remapping, which replaces
1753	* register based IOTLB invalidation.
1754	*/
1755	int dmar_enable_qi(struct intel_iommu *iommu)
1756	{
1757	struct q_inval *qi;
1758	struct page *desc_page;
1759
1760	if (!ecap_qis(iommu->ecap))
1761	return -ENOENT;
1762
1763	/*
1764	* queued invalidation is already setup and enabled.
1765	*/
1766	if (iommu->qi)
1767	return 0;
1768
1769	iommu->qi = kmalloc(size: sizeof(*qi), GFP_ATOMIC);
1770	if (!iommu->qi)
1771	return -ENOMEM;
1772
1773	qi = iommu->qi;
1774
1775	/*
1776	* Need two pages to accommodate 256 descriptors of 256 bits each
1777	* if the remapping hardware supports scalable mode translation.
1778	*/
1779	desc_page = alloc_pages_node(nid: iommu->node, GFP_ATOMIC | __GFP_ZERO,
1780	order: !!ecap_smts(iommu->ecap));
1781	if (!desc_page) {
1782	kfree(objp: qi);
1783	iommu->qi = NULL;
1784	return -ENOMEM;
1785	}
1786
1787	qi->desc = page_address(desc_page);
1788
1789	qi->desc_status = kcalloc(QI_LENGTH, size: sizeof(int), GFP_ATOMIC);
1790	if (!qi->desc_status) {
1791	free_page((unsigned long) qi->desc);
1792	kfree(objp: qi);
1793	iommu->qi = NULL;
1794	return -ENOMEM;
1795	}
1796
1797	raw_spin_lock_init(&qi->q_lock);
1798
1799	__dmar_enable_qi(iommu);
1800
1801	return 0;
1802	}
1803
1804	/* iommu interrupt handling. Most stuff are MSI-like. */
1805
1806	enum faulttype {
1807	DMA_REMAP,
1808	INTR_REMAP,
1809	UNKNOWN,
1810	};
1811
1812	static const char *dma_remap_fault_reasons[] =
1813	{
1814	"Software",
1815	"Present bit in root entry is clear",
1816	"Present bit in context entry is clear",
1817	"Invalid context entry",
1818	"Access beyond MGAW",
1819	"PTE Write access is not set",
1820	"PTE Read access is not set",
1821	"Next page table ptr is invalid",
1822	"Root table address invalid",
1823	"Context table ptr is invalid",
1824	"non-zero reserved fields in RTP",
1825	"non-zero reserved fields in CTP",
1826	"non-zero reserved fields in PTE",
1827	"PCE for translation request specifies blocking",
1828	};
1829
1830	static const char * const dma_remap_sm_fault_reasons[] = {
1831	"SM: Invalid Root Table Address",
1832	"SM: TTM 0 for request with PASID",
1833	"SM: TTM 0 for page group request",
1834	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x33-0x37 */
1835	"SM: Error attempting to access Root Entry",
1836	"SM: Present bit in Root Entry is clear",
1837	"SM: Non-zero reserved field set in Root Entry",
1838	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x3B-0x3F */
1839	"SM: Error attempting to access Context Entry",
1840	"SM: Present bit in Context Entry is clear",
1841	"SM: Non-zero reserved field set in the Context Entry",
1842	"SM: Invalid Context Entry",
1843	"SM: DTE field in Context Entry is clear",
1844	"SM: PASID Enable field in Context Entry is clear",
1845	"SM: PASID is larger than the max in Context Entry",
1846	"SM: PRE field in Context-Entry is clear",
1847	"SM: RID_PASID field error in Context-Entry",
1848	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x49-0x4F */
1849	"SM: Error attempting to access the PASID Directory Entry",
1850	"SM: Present bit in Directory Entry is clear",
1851	"SM: Non-zero reserved field set in PASID Directory Entry",
1852	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x53-0x57 */
1853	"SM: Error attempting to access PASID Table Entry",
1854	"SM: Present bit in PASID Table Entry is clear",
1855	"SM: Non-zero reserved field set in PASID Table Entry",
1856	"SM: Invalid Scalable-Mode PASID Table Entry",
1857	"SM: ERE field is clear in PASID Table Entry",
1858	"SM: SRE field is clear in PASID Table Entry",
1859	"Unknown", "Unknown",/* 0x5E-0x5F */
1860	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x60-0x67 */
1861	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x68-0x6F */
1862	"SM: Error attempting to access first-level paging entry",
1863	"SM: Present bit in first-level paging entry is clear",
1864	"SM: Non-zero reserved field set in first-level paging entry",
1865	"SM: Error attempting to access FL-PML4 entry",
1866	"SM: First-level entry address beyond MGAW in Nested translation",
1867	"SM: Read permission error in FL-PML4 entry in Nested translation",
1868	"SM: Read permission error in first-level paging entry in Nested translation",
1869	"SM: Write permission error in first-level paging entry in Nested translation",
1870	"SM: Error attempting to access second-level paging entry",
1871	"SM: Read/Write permission error in second-level paging entry",
1872	"SM: Non-zero reserved field set in second-level paging entry",
1873	"SM: Invalid second-level page table pointer",
1874	"SM: A/D bit update needed in second-level entry when set up in no snoop",
1875	"Unknown", "Unknown", "Unknown", /* 0x7D-0x7F */
1876	"SM: Address in first-level translation is not canonical",
1877	"SM: U/S set 0 for first-level translation with user privilege",
1878	"SM: No execute permission for request with PASID and ER=1",
1879	"SM: Address beyond the DMA hardware max",
1880	"SM: Second-level entry address beyond the max",
1881	"SM: No write permission for Write/AtomicOp request",
1882	"SM: No read permission for Read/AtomicOp request",
1883	"SM: Invalid address-interrupt address",
1884	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x88-0x8F */
1885	"SM: A/D bit update needed in first-level entry when set up in no snoop",
1886	};
1887
1888	static const char *irq_remap_fault_reasons[] =
1889	{
1890	"Detected reserved fields in the decoded interrupt-remapped request",
1891	"Interrupt index exceeded the interrupt-remapping table size",
1892	"Present field in the IRTE entry is clear",
1893	"Error accessing interrupt-remapping table pointed by IRTA_REG",
1894	"Detected reserved fields in the IRTE entry",
1895	"Blocked a compatibility format interrupt request",
1896	"Blocked an interrupt request due to source-id verification failure",
1897	};
1898
1899	static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1900	{
1901	if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1902	ARRAY_SIZE(irq_remap_fault_reasons))) {
1903	*fault_type = INTR_REMAP;
1904	return irq_remap_fault_reasons[fault_reason - 0x20];
1905	} else if (fault_reason >= 0x30 && (fault_reason - 0x30 <
1906	ARRAY_SIZE(dma_remap_sm_fault_reasons))) {
1907	*fault_type = DMA_REMAP;
1908	return dma_remap_sm_fault_reasons[fault_reason - 0x30];
1909	} else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1910	*fault_type = DMA_REMAP;
1911	return dma_remap_fault_reasons[fault_reason];
1912	} else {
1913	*fault_type = UNKNOWN;
1914	return "Unknown";
1915	}
1916	}
1917
1918
1919	static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1920	{
1921	if (iommu->irq == irq)
1922	return DMAR_FECTL_REG;
1923	else if (iommu->pr_irq == irq)
1924	return DMAR_PECTL_REG;
1925	else if (iommu->perf_irq == irq)
1926	return DMAR_PERFINTRCTL_REG;
1927	else
1928	BUG();
1929	}
1930
1931	void dmar_msi_unmask(struct irq_data *data)
1932	{
1933	struct intel_iommu *iommu = irq_data_get_irq_handler_data(d: data);
1934	int reg = dmar_msi_reg(iommu, irq: data->irq);
1935	unsigned long flag;
1936
1937	/* unmask it */
1938	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1939	writel(val: 0, addr: iommu->reg + reg);
1940	/* Read a reg to force flush the post write */
1941	readl(addr: iommu->reg + reg);
1942	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1943	}
1944
1945	void dmar_msi_mask(struct irq_data *data)
1946	{
1947	struct intel_iommu *iommu = irq_data_get_irq_handler_data(d: data);
1948	int reg = dmar_msi_reg(iommu, irq: data->irq);
1949	unsigned long flag;
1950
1951	/* mask it */
1952	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1953	writel(DMA_FECTL_IM, addr: iommu->reg + reg);
1954	/* Read a reg to force flush the post write */
1955	readl(addr: iommu->reg + reg);
1956	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1957	}
1958
1959	void dmar_msi_write(int irq, struct msi_msg *msg)
1960	{
1961	struct intel_iommu *iommu = irq_get_handler_data(irq);
1962	int reg = dmar_msi_reg(iommu, irq);
1963	unsigned long flag;
1964
1965	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1966	writel(val: msg->data, addr: iommu->reg + reg + 4);
1967	writel(val: msg->address_lo, addr: iommu->reg + reg + 8);
1968	writel(val: msg->address_hi, addr: iommu->reg + reg + 12);
1969	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1970	}
1971
1972	void dmar_msi_read(int irq, struct msi_msg *msg)
1973	{
1974	struct intel_iommu *iommu = irq_get_handler_data(irq);
1975	int reg = dmar_msi_reg(iommu, irq);
1976	unsigned long flag;
1977
1978	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1979	msg->data = readl(addr: iommu->reg + reg + 4);
1980	msg->address_lo = readl(addr: iommu->reg + reg + 8);
1981	msg->address_hi = readl(addr: iommu->reg + reg + 12);
1982	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1983	}
1984
1985	static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1986	u8 fault_reason, u32 pasid, u16 source_id,
1987	unsigned long long addr)
1988	{
1989	const char *reason;
1990	int fault_type;
1991
1992	reason = dmar_get_fault_reason(fault_reason, fault_type: &fault_type);
1993
1994	if (fault_type == INTR_REMAP) {
1995	pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index 0x%llx [fault reason 0x%02x] %s\n",
1996	source_id >> 8, PCI_SLOT(source_id & 0xFF),
1997	PCI_FUNC(source_id & 0xFF), addr >> 48,
1998	fault_reason, reason);
1999
2000	return 0;
2001	}
2002
2003	if (pasid == IOMMU_PASID_INVALID)
2004	pr_err("[%s NO_PASID] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
2005	type ? "DMA Read" : "DMA Write",
2006	source_id >> 8, PCI_SLOT(source_id & 0xFF),
2007	PCI_FUNC(source_id & 0xFF), addr,
2008	fault_reason, reason);
2009	else
2010	pr_err("[%s PASID 0x%x] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
2011	type ? "DMA Read" : "DMA Write", pasid,
2012	source_id >> 8, PCI_SLOT(source_id & 0xFF),
2013	PCI_FUNC(source_id & 0xFF), addr,
2014	fault_reason, reason);
2015
2016	dmar_fault_dump_ptes(iommu, source_id, addr, pasid);
2017
2018	return 0;
2019	}
2020
2021	#define PRIMARY_FAULT_REG_LEN (16)
2022	irqreturn_t dmar_fault(int irq, void *dev_id)
2023	{
2024	struct intel_iommu *iommu = dev_id;
2025	int reg, fault_index;
2026	u32 fault_status;
2027	unsigned long flag;
2028	static DEFINE_RATELIMIT_STATE(rs,
2029	DEFAULT_RATELIMIT_INTERVAL,
2030	DEFAULT_RATELIMIT_BURST);
2031
2032	raw_spin_lock_irqsave(&iommu->register_lock, flag);
2033	fault_status = readl(addr: iommu->reg + DMAR_FSTS_REG);
2034	if (fault_status && __ratelimit(&rs))
2035	pr_err("DRHD: handling fault status reg %x\n", fault_status);
2036
2037	/* TBD: ignore advanced fault log currently */
2038	if (!(fault_status & DMA_FSTS_PPF))
2039	goto unlock_exit;
2040
2041	fault_index = dma_fsts_fault_record_index(fault_status);
2042	reg = cap_fault_reg_offset(iommu->cap);
2043	while (1) {
2044	/* Disable printing, simply clear the fault when ratelimited */
2045	bool ratelimited = !__ratelimit(&rs);
2046	u8 fault_reason;
2047	u16 source_id;
2048	u64 guest_addr;
2049	u32 pasid;
2050	int type;
2051	u32 data;
2052	bool pasid_present;
2053
2054	/* highest 32 bits */
2055	data = readl(addr: iommu->reg + reg +
2056	fault_index * PRIMARY_FAULT_REG_LEN + 12);
2057	if (!(data & DMA_FRCD_F))
2058	break;
2059
2060	if (!ratelimited) {
2061	fault_reason = dma_frcd_fault_reason(data);
2062	type = dma_frcd_type(data);
2063
2064	pasid = dma_frcd_pasid_value(data);
2065	data = readl(addr: iommu->reg + reg +
2066	fault_index * PRIMARY_FAULT_REG_LEN + 8);
2067	source_id = dma_frcd_source_id(data);
2068
2069	pasid_present = dma_frcd_pasid_present(data);
2070	guest_addr = dmar_readq(iommu->reg + reg +
2071	fault_index * PRIMARY_FAULT_REG_LEN);
2072	guest_addr = dma_frcd_page_addr(guest_addr);
2073	}
2074
2075	/* clear the fault */
2076	writel(DMA_FRCD_F, addr: iommu->reg + reg +
2077	fault_index * PRIMARY_FAULT_REG_LEN + 12);
2078
2079	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2080
2081	if (!ratelimited)
2082	/* Using pasid -1 if pasid is not present */
2083	dmar_fault_do_one(iommu, type, fault_reason,
2084	pasid: pasid_present ? pasid : IOMMU_PASID_INVALID,
2085	source_id, addr: guest_addr);
2086
2087	fault_index++;
2088	if (fault_index >= cap_num_fault_regs(iommu->cap))
2089	fault_index = 0;
2090	raw_spin_lock_irqsave(&iommu->register_lock, flag);
2091	}
2092
2093	writel(DMA_FSTS_PFO | DMA_FSTS_PPF | DMA_FSTS_PRO,
2094	addr: iommu->reg + DMAR_FSTS_REG);
2095
2096	unlock_exit:
2097	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2098	return IRQ_HANDLED;
2099	}
2100
2101	int dmar_set_interrupt(struct intel_iommu *iommu)
2102	{
2103	int irq, ret;
2104
2105	/*
2106	* Check if the fault interrupt is already initialized.
2107	*/
2108	if (iommu->irq)
2109	return 0;
2110
2111	irq = dmar_alloc_hwirq(id: iommu->seq_id, node: iommu->node, arg: iommu);
2112	if (irq > 0) {
2113	iommu->irq = irq;
2114	} else {
2115	pr_err("No free IRQ vectors\n");
2116	return -EINVAL;
2117	}
2118
2119	ret = request_irq(irq, handler: dmar_fault, IRQF_NO_THREAD, name: iommu->name, dev: iommu);
2120	if (ret)
2121	pr_err("Can't request irq\n");
2122	return ret;
2123	}
2124
2125	int __init enable_drhd_fault_handling(void)
2126	{
2127	struct dmar_drhd_unit *drhd;
2128	struct intel_iommu *iommu;
2129
2130	/*
2131	* Enable fault control interrupt.
2132	*/
2133	for_each_iommu(iommu, drhd) {
2134	u32 fault_status;
2135	int ret = dmar_set_interrupt(iommu);
2136
2137	if (ret) {
2138	pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
2139	(unsigned long long)drhd->reg_base_addr, ret);
2140	return -1;
2141	}
2142
2143	/*
2144	* Clear any previous faults.
2145	*/
2146	dmar_fault(irq: iommu->irq, dev_id: iommu);
2147	fault_status = readl(addr: iommu->reg + DMAR_FSTS_REG);
2148	writel(val: fault_status, addr: iommu->reg + DMAR_FSTS_REG);
2149	}
2150
2151	return 0;
2152	}
2153
2154	/*
2155	* Re-enable Queued Invalidation interface.
2156	*/
2157	int dmar_reenable_qi(struct intel_iommu *iommu)
2158	{
2159	if (!ecap_qis(iommu->ecap))
2160	return -ENOENT;
2161
2162	if (!iommu->qi)
2163	return -ENOENT;
2164
2165	/*
2166	* First disable queued invalidation.
2167	*/
2168	dmar_disable_qi(iommu);
2169	/*
2170	* Then enable queued invalidation again. Since there is no pending
2171	* invalidation requests now, it's safe to re-enable queued
2172	* invalidation.
2173	*/
2174	__dmar_enable_qi(iommu);
2175
2176	return 0;
2177	}
2178
2179	/*
2180	* Check interrupt remapping support in DMAR table description.
2181	*/
2182	int __init dmar_ir_support(void)
2183	{
2184	struct acpi_table_dmar *dmar;
2185	dmar = (struct acpi_table_dmar *)dmar_tbl;
2186	if (!dmar)
2187	return 0;
2188	return dmar->flags & 0x1;
2189	}
2190
2191	/* Check whether DMAR units are in use */
2192	static inline bool dmar_in_use(void)
2193	{
2194	return irq_remapping_enabled || intel_iommu_enabled;
2195	}
2196
2197	static int __init dmar_free_unused_resources(void)
2198	{
2199	struct dmar_drhd_unit *dmaru, *dmaru_n;
2200
2201	if (dmar_in_use())
2202	return 0;
2203
2204	if (dmar_dev_scope_status != 1 && !list_empty(head: &dmar_drhd_units))
2205	bus_unregister_notifier(bus: &pci_bus_type, nb: &dmar_pci_bus_nb);
2206
2207	down_write(sem: &dmar_global_lock);
2208	list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
2209	list_del(entry: &dmaru->list);
2210	dmar_free_drhd(dmaru);
2211	}
2212	up_write(sem: &dmar_global_lock);
2213
2214	return 0;
2215	}
2216
2217	late_initcall(dmar_free_unused_resources);
2218
2219	/*
2220	* DMAR Hotplug Support
2221	* For more details, please refer to Intel(R) Virtualization Technology
2222	* for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
2223	* "Remapping Hardware Unit Hot Plug".
2224	*/
2225	static guid_t dmar_hp_guid =
2226	GUID_INIT(0xD8C1A3A6, 0xBE9B, 0x4C9B,
2227	0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF);
2228
2229	/*
2230	* Currently there's only one revision and BIOS will not check the revision id,
2231	* so use 0 for safety.
2232	*/
2233	#define DMAR_DSM_REV_ID 0
2234	#define DMAR_DSM_FUNC_DRHD 1
2235	#define DMAR_DSM_FUNC_ATSR 2
2236	#define DMAR_DSM_FUNC_RHSA 3
2237	#define DMAR_DSM_FUNC_SATC 4
2238
2239	static inline bool dmar_detect_dsm(acpi_handle handle, int func)
2240	{
2241	return acpi_check_dsm(handle, guid: &dmar_hp_guid, DMAR_DSM_REV_ID, funcs: 1 << func);
2242	}
2243
2244	static int dmar_walk_dsm_resource(acpi_handle handle, int func,
2245	dmar_res_handler_t handler, void *arg)
2246	{
2247	int ret = -ENODEV;
2248	union acpi_object *obj;
2249	struct acpi_dmar_header *start;
2250	struct dmar_res_callback callback;
2251	static int res_type[] = {
2252	[DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
2253	[DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
2254	[DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
2255	[DMAR_DSM_FUNC_SATC] = ACPI_DMAR_TYPE_SATC,
2256	};
2257
2258	if (!dmar_detect_dsm(handle, func))
2259	return 0;
2260
2261	obj = acpi_evaluate_dsm_typed(handle, guid: &dmar_hp_guid, DMAR_DSM_REV_ID,
2262	func, NULL, ACPI_TYPE_BUFFER);
2263	if (!obj)
2264	return -ENODEV;
2265
2266	memset(&callback, 0, sizeof(callback));
2267	callback.cb[res_type[func]] = handler;
2268	callback.arg[res_type[func]] = arg;
2269	start = (struct acpi_dmar_header *)obj->buffer.pointer;
2270	ret = dmar_walk_remapping_entries(start, len: obj->buffer.length, cb: &callback);
2271
2272	ACPI_FREE(obj);
2273
2274	return ret;
2275	}
2276
2277	static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
2278	{
2279	int ret;
2280	struct dmar_drhd_unit *dmaru;
2281
2282	dmaru = dmar_find_dmaru(drhd: (struct acpi_dmar_hardware_unit *)header);
2283	if (!dmaru)
2284	return -ENODEV;
2285
2286	ret = dmar_ir_hotplug(dmaru, insert: true);
2287	if (ret == 0)
2288	ret = dmar_iommu_hotplug(dmaru, insert: true);
2289
2290	return ret;
2291	}
2292
2293	static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
2294	{
2295	int i, ret;
2296	struct device *dev;
2297	struct dmar_drhd_unit *dmaru;
2298
2299	dmaru = dmar_find_dmaru(drhd: (struct acpi_dmar_hardware_unit *)header);
2300	if (!dmaru)
2301	return 0;
2302
2303	/*
2304	* All PCI devices managed by this unit should have been destroyed.
2305	*/
2306	if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
2307	for_each_active_dev_scope(dmaru->devices,
2308	dmaru->devices_cnt, i, dev)
2309	return -EBUSY;
2310	}
2311
2312	ret = dmar_ir_hotplug(dmaru, insert: false);
2313	if (ret == 0)
2314	ret = dmar_iommu_hotplug(dmaru, insert: false);
2315
2316	return ret;
2317	}
2318
2319	static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
2320	{
2321	struct dmar_drhd_unit *dmaru;
2322
2323	dmaru = dmar_find_dmaru(drhd: (struct acpi_dmar_hardware_unit *)header);
2324	if (dmaru) {
2325	list_del_rcu(entry: &dmaru->list);
2326	synchronize_rcu();
2327	dmar_free_drhd(dmaru);
2328	}
2329
2330	return 0;
2331	}
2332
2333	static int dmar_hotplug_insert(acpi_handle handle)
2334	{
2335	int ret;
2336	int drhd_count = 0;
2337
2338	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2339	handler: &dmar_validate_one_drhd, arg: (void *)1);
2340	if (ret)
2341	goto out;
2342
2343	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2344	handler: &dmar_parse_one_drhd, arg: (void *)&drhd_count);
2345	if (ret == 0 && drhd_count == 0) {
2346	pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
2347	goto out;
2348	} else if (ret) {
2349	goto release_drhd;
2350	}
2351
2352	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
2353	handler: &dmar_parse_one_rhsa, NULL);
2354	if (ret)
2355	goto release_drhd;
2356
2357	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2358	handler: &dmar_parse_one_atsr, NULL);
2359	if (ret)
2360	goto release_atsr;
2361
2362	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2363	handler: &dmar_hp_add_drhd, NULL);
2364	if (!ret)
2365	return 0;
2366
2367	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2368	handler: &dmar_hp_remove_drhd, NULL);
2369	release_atsr:
2370	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2371	handler: &dmar_release_one_atsr, NULL);
2372	release_drhd:
2373	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2374	handler: &dmar_hp_release_drhd, NULL);
2375	out:
2376	return ret;
2377	}
2378
2379	static int dmar_hotplug_remove(acpi_handle handle)
2380	{
2381	int ret;
2382
2383	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2384	handler: &dmar_check_one_atsr, NULL);
2385	if (ret)
2386	return ret;
2387
2388	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2389	handler: &dmar_hp_remove_drhd, NULL);
2390	if (ret == 0) {
2391	WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2392	&dmar_release_one_atsr, NULL));
2393	WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2394	&dmar_hp_release_drhd, NULL));
2395	} else {
2396	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2397	handler: &dmar_hp_add_drhd, NULL);
2398	}
2399
2400	return ret;
2401	}
2402
2403	static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
2404	void *context, void **retval)
2405	{
2406	acpi_handle *phdl = retval;
2407
2408	if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2409	*phdl = handle;
2410	return AE_CTRL_TERMINATE;
2411	}
2412
2413	return AE_OK;
2414	}
2415
2416	static int dmar_device_hotplug(acpi_handle handle, bool insert)
2417	{
2418	int ret;
2419	acpi_handle tmp = NULL;
2420	acpi_status status;
2421
2422	if (!dmar_in_use())
2423	return 0;
2424
2425	if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2426	tmp = handle;
2427	} else {
2428	status = acpi_walk_namespace(ACPI_TYPE_DEVICE, start_object: handle,
2429	ACPI_UINT32_MAX,
2430	descending_callback: dmar_get_dsm_handle,
2431	NULL, NULL, return_value: &tmp);
2432	if (ACPI_FAILURE(status)) {
2433	pr_warn("Failed to locate _DSM method.\n");
2434	return -ENXIO;
2435	}
2436	}
2437	if (tmp == NULL)
2438	return 0;
2439
2440	down_write(sem: &dmar_global_lock);
2441	if (insert)
2442	ret = dmar_hotplug_insert(handle: tmp);
2443	else
2444	ret = dmar_hotplug_remove(handle: tmp);
2445	up_write(sem: &dmar_global_lock);
2446
2447	return ret;
2448	}
2449
2450	int dmar_device_add(acpi_handle handle)
2451	{
2452	return dmar_device_hotplug(handle, insert: true);
2453	}
2454
2455	int dmar_device_remove(acpi_handle handle)
2456	{
2457	return dmar_device_hotplug(handle, insert: false);
2458	}
2459
2460	/*
2461	* dmar_platform_optin - Is %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in DMAR table
2462	*
2463	* Returns true if the platform has %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in
2464	* the ACPI DMAR table. This means that the platform boot firmware has made
2465	* sure no device can issue DMA outside of RMRR regions.
2466	*/
2467	bool dmar_platform_optin(void)
2468	{
2469	struct acpi_table_dmar *dmar;
2470	acpi_status status;
2471	bool ret;
2472
2473	status = acpi_get_table(ACPI_SIG_DMAR, instance: 0,
2474	out_table: (struct acpi_table_header **)&dmar);
2475	if (ACPI_FAILURE(status))
2476	return false;
2477
2478	ret = !!(dmar->flags & DMAR_PLATFORM_OPT_IN);
2479	acpi_put_table(table: (struct acpi_table_header *)dmar);
2480
2481	return ret;
2482	}
2483	EXPORT_SYMBOL_GPL(dmar_platform_optin);
2484