1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* PCI Bus Services, see include/linux/pci.h for further explanation.
4	*
5	* Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
6	* David Mosberger-Tang
7	*
8	* Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
9	*/
10
11	#include <linux/acpi.h>
12	#include <linux/kernel.h>
13	#include <linux/delay.h>
14	#include <linux/dmi.h>
15	#include <linux/init.h>
16	#include <linux/msi.h>
17	#include <linux/of.h>
18	#include <linux/pci.h>
19	#include <linux/pm.h>
20	#include <linux/slab.h>
21	#include <linux/module.h>
22	#include <linux/spinlock.h>
23	#include <linux/string.h>
24	#include <linux/log2.h>
25	#include <linux/logic_pio.h>
26	#include <linux/pm_wakeup.h>
27	#include <linux/interrupt.h>
28	#include <linux/device.h>
29	#include <linux/pm_runtime.h>
30	#include <linux/pci_hotplug.h>
31	#include <linux/vmalloc.h>
32	#include <asm/dma.h>
33	#include <linux/aer.h>
34	#include <linux/bitfield.h>
35	#include "pci.h"
36
37	DEFINE_MUTEX(pci_slot_mutex);
38
39	const char *pci_power_names[] = {
40	"error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
41	};
42	EXPORT_SYMBOL_GPL(pci_power_names);
43
44	#ifdef CONFIG_X86_32
45	int isa_dma_bridge_buggy;
46	EXPORT_SYMBOL(isa_dma_bridge_buggy);
47	#endif
48
49	int pci_pci_problems;
50	EXPORT_SYMBOL(pci_pci_problems);
51
52	unsigned int pci_pm_d3hot_delay;
53
54	static void pci_pme_list_scan(struct work_struct *work);
55
56	static LIST_HEAD(pci_pme_list);
57	static DEFINE_MUTEX(pci_pme_list_mutex);
58	static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);
59
60	struct pci_pme_device {
61	struct list_head list;
62	struct pci_dev *dev;
63	};
64
65	#define PME_TIMEOUT 1000 /* How long between PME checks */
66
67	/*
68	* Following exit from Conventional Reset, devices must be ready within 1 sec
69	* (PCIe r6.0 sec 6.6.1). A D3cold to D0 transition implies a Conventional
70	* Reset (PCIe r6.0 sec 5.8).
71	*/
72	#define PCI_RESET_WAIT 1000 /* msec */
73
74	/*
75	* Devices may extend the 1 sec period through Request Retry Status
76	* completions (PCIe r6.0 sec 2.3.1). The spec does not provide an upper
77	* limit, but 60 sec ought to be enough for any device to become
78	* responsive.
79	*/
80	#define PCIE_RESET_READY_POLL_MS 60000 /* msec */
81
82	static void pci_dev_d3_sleep(struct pci_dev *dev)
83	{
84	unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay);
85	unsigned int upper;
86
87	if (delay_ms) {
88	/* Use a 20% upper bound, 1ms minimum */
89	upper = max(DIV_ROUND_CLOSEST(delay_ms, 5), 1U);
90	usleep_range(min: delay_ms * USEC_PER_MSEC,
91	max: (delay_ms + upper) * USEC_PER_MSEC);
92	}
93	}
94
95	bool pci_reset_supported(struct pci_dev *dev)
96	{
97	return dev->reset_methods[0] != 0;
98	}
99
100	#ifdef CONFIG_PCI_DOMAINS
101	int pci_domains_supported = 1;
102	#endif
103
104	#define DEFAULT_CARDBUS_IO_SIZE (256)
105	#define DEFAULT_CARDBUS_MEM_SIZE (64*1024*1024)
106	/* pci=cbmemsize=nnM,cbiosize=nn can override this */
107	unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
108	unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
109
110	#define DEFAULT_HOTPLUG_IO_SIZE (256)
111	#define DEFAULT_HOTPLUG_MMIO_SIZE (2*1024*1024)
112	#define DEFAULT_HOTPLUG_MMIO_PREF_SIZE (2*1024*1024)
113	/* hpiosize=nn can override this */
114	unsigned long pci_hotplug_io_size = DEFAULT_HOTPLUG_IO_SIZE;
115	/*
116	* pci=hpmmiosize=nnM overrides non-prefetchable MMIO size,
117	* pci=hpmmioprefsize=nnM overrides prefetchable MMIO size;
118	* pci=hpmemsize=nnM overrides both
119	*/
120	unsigned long pci_hotplug_mmio_size = DEFAULT_HOTPLUG_MMIO_SIZE;
121	unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE;
122
123	#define DEFAULT_HOTPLUG_BUS_SIZE 1
124	unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
125
126
127	/* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */
128	#ifdef CONFIG_PCIE_BUS_TUNE_OFF
129	enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF;
130	#elif defined CONFIG_PCIE_BUS_SAFE
131	enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE;
132	#elif defined CONFIG_PCIE_BUS_PERFORMANCE
133	enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE;
134	#elif defined CONFIG_PCIE_BUS_PEER2PEER
135	enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER;
136	#else
137	enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
138	#endif
139
140	/*
141	* The default CLS is used if arch didn't set CLS explicitly and not
142	* all pci devices agree on the same value. Arch can override either
143	* the dfl or actual value as it sees fit. Don't forget this is
144	* measured in 32-bit words, not bytes.
145	*/
146	u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2;
147	u8 pci_cache_line_size;
148
149	/*
150	* If we set up a device for bus mastering, we need to check the latency
151	* timer as certain BIOSes forget to set it properly.
152	*/
153	unsigned int pcibios_max_latency = 255;
154
155	/* If set, the PCIe ARI capability will not be used. */
156	static bool pcie_ari_disabled;
157
158	/* If set, the PCIe ATS capability will not be used. */
159	static bool pcie_ats_disabled;
160
161	/* If set, the PCI config space of each device is printed during boot. */
162	bool pci_early_dump;
163
164	bool pci_ats_disabled(void)
165	{
166	return pcie_ats_disabled;
167	}
168	EXPORT_SYMBOL_GPL(pci_ats_disabled);
169
170	/* Disable bridge_d3 for all PCIe ports */
171	static bool pci_bridge_d3_disable;
172	/* Force bridge_d3 for all PCIe ports */
173	static bool pci_bridge_d3_force;
174
175	static int __init pcie_port_pm_setup(char *str)
176	{
177	if (!strcmp(str, "off"))
178	pci_bridge_d3_disable = true;
179	else if (!strcmp(str, "force"))
180	pci_bridge_d3_force = true;
181	return 1;
182	}
183	__setup("pcie_port_pm=", pcie_port_pm_setup);
184
185	/**
186	* pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
187	* @bus: pointer to PCI bus structure to search
188	*
189	* Given a PCI bus, returns the highest PCI bus number present in the set
190	* including the given PCI bus and its list of child PCI buses.
191	*/
192	unsigned char pci_bus_max_busnr(struct pci_bus *bus)
193	{
194	struct pci_bus *tmp;
195	unsigned char max, n;
196
197	max = bus->busn_res.end;
198	list_for_each_entry(tmp, &bus->children, node) {
199	n = pci_bus_max_busnr(bus: tmp);
200	if (n > max)
201	max = n;
202	}
203	return max;
204	}
205	EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
206
207	/**
208	* pci_status_get_and_clear_errors - return and clear error bits in PCI_STATUS
209	* @pdev: the PCI device
210	*
211	* Returns error bits set in PCI_STATUS and clears them.
212	*/
213	int pci_status_get_and_clear_errors(struct pci_dev *pdev)
214	{
215	u16 status;
216	int ret;
217
218	ret = pci_read_config_word(dev: pdev, PCI_STATUS, val: &status);
219	if (ret != PCIBIOS_SUCCESSFUL)
220	return -EIO;
221
222	status &= PCI_STATUS_ERROR_BITS;
223	if (status)
224	pci_write_config_word(dev: pdev, PCI_STATUS, val: status);
225
226	return status;
227	}
228	EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors);
229
230	#ifdef CONFIG_HAS_IOMEM
231	static void __iomem *__pci_ioremap_resource(struct pci_dev *pdev, int bar,
232	bool write_combine)
233	{
234	struct resource *res = &pdev->resource[bar];
235	resource_size_t start = res->start;
236	resource_size_t size = resource_size(res);
237
238	/*
239	* Make sure the BAR is actually a memory resource, not an IO resource
240	*/
241	if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) {
242	pci_err(pdev, "can't ioremap BAR %d: %pR\n", bar, res);
243	return NULL;
244	}
245
246	if (write_combine)
247	return ioremap_wc(offset: start, size);
248
249	return ioremap(offset: start, size);
250	}
251
252	void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
253	{
254	return __pci_ioremap_resource(pdev, bar, write_combine: false);
255	}
256	EXPORT_SYMBOL_GPL(pci_ioremap_bar);
257
258	void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar)
259	{
260	return __pci_ioremap_resource(pdev, bar, write_combine: true);
261	}
262	EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar);
263	#endif
264
265	/**
266	* pci_dev_str_match_path - test if a path string matches a device
267	* @dev: the PCI device to test
268	* @path: string to match the device against
269	* @endptr: pointer to the string after the match
270	*
271	* Test if a string (typically from a kernel parameter) formatted as a
272	* path of device/function addresses matches a PCI device. The string must
273	* be of the form:
274	*
275	* [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
276	*
277	* A path for a device can be obtained using 'lspci -t'. Using a path
278	* is more robust against bus renumbering than using only a single bus,
279	* device and function address.
280	*
281	* Returns 1 if the string matches the device, 0 if it does not and
282	* a negative error code if it fails to parse the string.
283	*/
284	static int pci_dev_str_match_path(struct pci_dev *dev, const char *path,
285	const char **endptr)
286	{
287	int ret;
288	unsigned int seg, bus, slot, func;
289	char *wpath, *p;
290	char end;
291
292	*endptr = strchrnul(path, ';');
293
294	wpath = kmemdup_nul(s: path, len: *endptr - path, GFP_ATOMIC);
295	if (!wpath)
296	return -ENOMEM;
297
298	while (1) {
299	p = strrchr(wpath, '/');
300	if (!p)
301	break;
302	ret = sscanf(p, "/%x.%x%c", &slot, &func, &end);
303	if (ret != 2) {
304	ret = -EINVAL;
305	goto free_and_exit;
306	}
307
308	if (dev->devfn != PCI_DEVFN(slot, func)) {
309	ret = 0;
310	goto free_and_exit;
311	}
312
313	/*
314	* Note: we don't need to get a reference to the upstream
315	* bridge because we hold a reference to the top level
316	* device which should hold a reference to the bridge,
317	* and so on.
318	*/
319	dev = pci_upstream_bridge(dev);
320	if (!dev) {
321	ret = 0;
322	goto free_and_exit;
323	}
324
325	*p = 0;
326	}
327
328	ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot,
329	&func, &end);
330	if (ret != 4) {
331	seg = 0;
332	ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end);
333	if (ret != 3) {
334	ret = -EINVAL;
335	goto free_and_exit;
336	}
337	}
338
339	ret = (seg == pci_domain_nr(bus: dev->bus) &&
340	bus == dev->bus->number &&
341	dev->devfn == PCI_DEVFN(slot, func));
342
343	free_and_exit:
344	kfree(objp: wpath);
345	return ret;
346	}
347
348	/**
349	* pci_dev_str_match - test if a string matches a device
350	* @dev: the PCI device to test
351	* @p: string to match the device against
352	* @endptr: pointer to the string after the match
353	*
354	* Test if a string (typically from a kernel parameter) matches a specified
355	* PCI device. The string may be of one of the following formats:
356	*
357	* [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
358	* pci:<vendor>:<device>[:<subvendor>:<subdevice>]
359	*
360	* The first format specifies a PCI bus/device/function address which
361	* may change if new hardware is inserted, if motherboard firmware changes,
362	* or due to changes caused in kernel parameters. If the domain is
363	* left unspecified, it is taken to be 0. In order to be robust against
364	* bus renumbering issues, a path of PCI device/function numbers may be used
365	* to address the specific device. The path for a device can be determined
366	* through the use of 'lspci -t'.
367	*
368	* The second format matches devices using IDs in the configuration
369	* space which may match multiple devices in the system. A value of 0
370	* for any field will match all devices. (Note: this differs from
371	* in-kernel code that uses PCI_ANY_ID which is ~0; this is for
372	* legacy reasons and convenience so users don't have to specify
373	* FFFFFFFFs on the command line.)
374	*
375	* Returns 1 if the string matches the device, 0 if it does not and
376	* a negative error code if the string cannot be parsed.
377	*/
378	static int pci_dev_str_match(struct pci_dev *dev, const char *p,
379	const char **endptr)
380	{
381	int ret;
382	int count;
383	unsigned short vendor, device, subsystem_vendor, subsystem_device;
384
385	if (strncmp(p, "pci:", 4) == 0) {
386	/* PCI vendor/device (subvendor/subdevice) IDs are specified */
387	p += 4;
388	ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device,
389	&subsystem_vendor, &subsystem_device, &count);
390	if (ret != 4) {
391	ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count);
392	if (ret != 2)
393	return -EINVAL;
394
395	subsystem_vendor = 0;
396	subsystem_device = 0;
397	}
398
399	p += count;
400
401	if ((!vendor || vendor == dev->vendor) &&
402	(!device || device == dev->device) &&
403	(!subsystem_vendor ||
404	subsystem_vendor == dev->subsystem_vendor) &&
405	(!subsystem_device ||
406	subsystem_device == dev->subsystem_device))
407	goto found;
408	} else {
409	/*
410	* PCI Bus, Device, Function IDs are specified
411	* (optionally, may include a path of devfns following it)
412	*/
413	ret = pci_dev_str_match_path(dev, path: p, endptr: &p);
414	if (ret < 0)
415	return ret;
416	else if (ret)
417	goto found;
418	}
419
420	*endptr = p;
421	return 0;
422
423	found:
424	*endptr = p;
425	return 1;
426	}
427
428	static u8 __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
429	u8 pos, int cap, int *ttl)
430	{
431	u8 id;
432	u16 ent;
433
434	pci_bus_read_config_byte(bus, devfn, where: pos, val: &pos);
435
436	while ((*ttl)--) {
437	if (pos < 0x40)
438	break;
439	pos &= ~3;
440	pci_bus_read_config_word(bus, devfn, where: pos, val: &ent);
441
442	id = ent & 0xff;
443	if (id == 0xff)
444	break;
445	if (id == cap)
446	return pos;
447	pos = (ent >> 8);
448	}
449	return 0;
450	}
451
452	static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
453	u8 pos, int cap)
454	{
455	int ttl = PCI_FIND_CAP_TTL;
456
457	return __pci_find_next_cap_ttl(bus, devfn, pos, cap, ttl: &ttl);
458	}
459
460	u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
461	{
462	return __pci_find_next_cap(bus: dev->bus, devfn: dev->devfn,
463	pos: pos + PCI_CAP_LIST_NEXT, cap);
464	}
465	EXPORT_SYMBOL_GPL(pci_find_next_capability);
466
467	static u8 __pci_bus_find_cap_start(struct pci_bus *bus,
468	unsigned int devfn, u8 hdr_type)
469	{
470	u16 status;
471
472	pci_bus_read_config_word(bus, devfn, PCI_STATUS, val: &status);
473	if (!(status & PCI_STATUS_CAP_LIST))
474	return 0;
475
476	switch (hdr_type) {
477	case PCI_HEADER_TYPE_NORMAL:
478	case PCI_HEADER_TYPE_BRIDGE:
479	return PCI_CAPABILITY_LIST;
480	case PCI_HEADER_TYPE_CARDBUS:
481	return PCI_CB_CAPABILITY_LIST;
482	}
483
484	return 0;
485	}
486
487	/**
488	* pci_find_capability - query for devices' capabilities
489	* @dev: PCI device to query
490	* @cap: capability code
491	*
492	* Tell if a device supports a given PCI capability.
493	* Returns the address of the requested capability structure within the
494	* device's PCI configuration space or 0 in case the device does not
495	* support it. Possible values for @cap include:
496	*
497	* %PCI_CAP_ID_PM Power Management
498	* %PCI_CAP_ID_AGP Accelerated Graphics Port
499	* %PCI_CAP_ID_VPD Vital Product Data
500	* %PCI_CAP_ID_SLOTID Slot Identification
501	* %PCI_CAP_ID_MSI Message Signalled Interrupts
502	* %PCI_CAP_ID_CHSWP CompactPCI HotSwap
503	* %PCI_CAP_ID_PCIX PCI-X
504	* %PCI_CAP_ID_EXP PCI Express
505	*/
506	u8 pci_find_capability(struct pci_dev *dev, int cap)
507	{
508	u8 pos;
509
510	pos = __pci_bus_find_cap_start(bus: dev->bus, devfn: dev->devfn, hdr_type: dev->hdr_type);
511	if (pos)
512	pos = __pci_find_next_cap(bus: dev->bus, devfn: dev->devfn, pos, cap);
513
514	return pos;
515	}
516	EXPORT_SYMBOL(pci_find_capability);
517
518	/**
519	* pci_bus_find_capability - query for devices' capabilities
520	* @bus: the PCI bus to query
521	* @devfn: PCI device to query
522	* @cap: capability code
523	*
524	* Like pci_find_capability() but works for PCI devices that do not have a
525	* pci_dev structure set up yet.
526	*
527	* Returns the address of the requested capability structure within the
528	* device's PCI configuration space or 0 in case the device does not
529	* support it.
530	*/
531	u8 pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
532	{
533	u8 hdr_type, pos;
534
535	pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, val: &hdr_type);
536
537	pos = __pci_bus_find_cap_start(bus, devfn, hdr_type: hdr_type & PCI_HEADER_TYPE_MASK);
538	if (pos)
539	pos = __pci_find_next_cap(bus, devfn, pos, cap);
540
541	return pos;
542	}
543	EXPORT_SYMBOL(pci_bus_find_capability);
544
545	/**
546	* pci_find_next_ext_capability - Find an extended capability
547	* @dev: PCI device to query
548	* @start: address at which to start looking (0 to start at beginning of list)
549	* @cap: capability code
550	*
551	* Returns the address of the next matching extended capability structure
552	* within the device's PCI configuration space or 0 if the device does
553	* not support it. Some capabilities can occur several times, e.g., the
554	* vendor-specific capability, and this provides a way to find them all.
555	*/
556	u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap)
557	{
558	u32 header;
559	int ttl;
560	u16 pos = PCI_CFG_SPACE_SIZE;
561
562	/* minimum 8 bytes per capability */
563	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
564
565	if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
566	return 0;
567
568	if (start)
569	pos = start;
570
571	if (pci_read_config_dword(dev, where: pos, val: &header) != PCIBIOS_SUCCESSFUL)
572	return 0;
573
574	/*
575	* If we have no capabilities, this is indicated by cap ID,
576	* cap version and next pointer all being 0.
577	*/
578	if (header == 0)
579	return 0;
580
581	while (ttl-- > 0) {
582	if (PCI_EXT_CAP_ID(header) == cap && pos != start)
583	return pos;
584
585	pos = PCI_EXT_CAP_NEXT(header);
586	if (pos < PCI_CFG_SPACE_SIZE)
587	break;
588
589	if (pci_read_config_dword(dev, where: pos, val: &header) != PCIBIOS_SUCCESSFUL)
590	break;
591	}
592
593	return 0;
594	}
595	EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);
596
597	/**
598	* pci_find_ext_capability - Find an extended capability
599	* @dev: PCI device to query
600	* @cap: capability code
601	*
602	* Returns the address of the requested extended capability structure
603	* within the device's PCI configuration space or 0 if the device does
604	* not support it. Possible values for @cap include:
605	*
606	* %PCI_EXT_CAP_ID_ERR Advanced Error Reporting
607	* %PCI_EXT_CAP_ID_VC Virtual Channel
608	* %PCI_EXT_CAP_ID_DSN Device Serial Number
609	* %PCI_EXT_CAP_ID_PWR Power Budgeting
610	*/
611	u16 pci_find_ext_capability(struct pci_dev *dev, int cap)
612	{
613	return pci_find_next_ext_capability(dev, 0, cap);
614	}
615	EXPORT_SYMBOL_GPL(pci_find_ext_capability);
616
617	/**
618	* pci_get_dsn - Read and return the 8-byte Device Serial Number
619	* @dev: PCI device to query
620	*
621	* Looks up the PCI_EXT_CAP_ID_DSN and reads the 8 bytes of the Device Serial
622	* Number.
623	*
624	* Returns the DSN, or zero if the capability does not exist.
625	*/
626	u64 pci_get_dsn(struct pci_dev *dev)
627	{
628	u32 dword;
629	u64 dsn;
630	int pos;
631
632	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN);
633	if (!pos)
634	return 0;
635
636	/*
637	* The Device Serial Number is two dwords offset 4 bytes from the
638	* capability position. The specification says that the first dword is
639	* the lower half, and the second dword is the upper half.
640	*/
641	pos += 4;
642	pci_read_config_dword(dev, where: pos, val: &dword);
643	dsn = (u64)dword;
644	pci_read_config_dword(dev, where: pos + 4, val: &dword);
645	dsn |= ((u64)dword) << 32;
646
647	return dsn;
648	}
649	EXPORT_SYMBOL_GPL(pci_get_dsn);
650
651	static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap)
652	{
653	int rc, ttl = PCI_FIND_CAP_TTL;
654	u8 cap, mask;
655
656	if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
657	mask = HT_3BIT_CAP_MASK;
658	else
659	mask = HT_5BIT_CAP_MASK;
660
661	pos = __pci_find_next_cap_ttl(bus: dev->bus, devfn: dev->devfn, pos,
662	PCI_CAP_ID_HT, ttl: &ttl);
663	while (pos) {
664	rc = pci_read_config_byte(dev, where: pos + 3, val: &cap);
665	if (rc != PCIBIOS_SUCCESSFUL)
666	return 0;
667
668	if ((cap & mask) == ht_cap)
669	return pos;
670
671	pos = __pci_find_next_cap_ttl(bus: dev->bus, devfn: dev->devfn,
672	pos: pos + PCI_CAP_LIST_NEXT,
673	PCI_CAP_ID_HT, ttl: &ttl);
674	}
675
676	return 0;
677	}
678
679	/**
680	* pci_find_next_ht_capability - query a device's HyperTransport capabilities
681	* @dev: PCI device to query
682	* @pos: Position from which to continue searching
683	* @ht_cap: HyperTransport capability code
684	*
685	* To be used in conjunction with pci_find_ht_capability() to search for
686	* all capabilities matching @ht_cap. @pos should always be a value returned
687	* from pci_find_ht_capability().
688	*
689	* NB. To be 100% safe against broken PCI devices, the caller should take
690	* steps to avoid an infinite loop.
691	*/
692	u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap)
693	{
694	return __pci_find_next_ht_cap(dev, pos: pos + PCI_CAP_LIST_NEXT, ht_cap);
695	}
696	EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);
697
698	/**
699	* pci_find_ht_capability - query a device's HyperTransport capabilities
700	* @dev: PCI device to query
701	* @ht_cap: HyperTransport capability code
702	*
703	* Tell if a device supports a given HyperTransport capability.
704	* Returns an address within the device's PCI configuration space
705	* or 0 in case the device does not support the request capability.
706	* The address points to the PCI capability, of type PCI_CAP_ID_HT,
707	* which has a HyperTransport capability matching @ht_cap.
708	*/
709	u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
710	{
711	u8 pos;
712
713	pos = __pci_bus_find_cap_start(bus: dev->bus, devfn: dev->devfn, hdr_type: dev->hdr_type);
714	if (pos)
715	pos = __pci_find_next_ht_cap(dev, pos, ht_cap);
716
717	return pos;
718	}
719	EXPORT_SYMBOL_GPL(pci_find_ht_capability);
720
721	/**
722	* pci_find_vsec_capability - Find a vendor-specific extended capability
723	* @dev: PCI device to query
724	* @vendor: Vendor ID for which capability is defined
725	* @cap: Vendor-specific capability ID
726	*
727	* If @dev has Vendor ID @vendor, search for a VSEC capability with
728	* VSEC ID @cap. If found, return the capability offset in
729	* config space; otherwise return 0.
730	*/
731	u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap)
732	{
733	u16 vsec = 0;
734	u32 header;
735	int ret;
736
737	if (vendor != dev->vendor)
738	return 0;
739
740	while ((vsec = pci_find_next_ext_capability(dev, vsec,
741	PCI_EXT_CAP_ID_VNDR))) {
742	ret = pci_read_config_dword(dev, where: vsec + PCI_VNDR_HEADER, val: &header);
743	if (ret != PCIBIOS_SUCCESSFUL)
744	continue;
745
746	if (PCI_VNDR_HEADER_ID(header) == cap)
747	return vsec;
748	}
749
750	return 0;
751	}
752	EXPORT_SYMBOL_GPL(pci_find_vsec_capability);
753
754	/**
755	* pci_find_dvsec_capability - Find DVSEC for vendor
756	* @dev: PCI device to query
757	* @vendor: Vendor ID to match for the DVSEC
758	* @dvsec: Designated Vendor-specific capability ID
759	*
760	* If DVSEC has Vendor ID @vendor and DVSEC ID @dvsec return the capability
761	* offset in config space; otherwise return 0.
762	*/
763	u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec)
764	{
765	int pos;
766
767	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DVSEC);
768	if (!pos)
769	return 0;
770
771	while (pos) {
772	u16 v, id;
773
774	pci_read_config_word(dev, where: pos + PCI_DVSEC_HEADER1, val: &v);
775	pci_read_config_word(dev, where: pos + PCI_DVSEC_HEADER2, val: &id);
776	if (vendor == v && dvsec == id)
777	return pos;
778
779	pos = pci_find_next_ext_capability(dev, pos, PCI_EXT_CAP_ID_DVSEC);
780	}
781
782	return 0;
783	}
784	EXPORT_SYMBOL_GPL(pci_find_dvsec_capability);
785
786	/**
787	* pci_find_parent_resource - return resource region of parent bus of given
788	* region
789	* @dev: PCI device structure contains resources to be searched
790	* @res: child resource record for which parent is sought
791	*
792	* For given resource region of given device, return the resource region of
793	* parent bus the given region is contained in.
794	*/
795	struct resource *pci_find_parent_resource(const struct pci_dev *dev,
796	struct resource *res)
797	{
798	const struct pci_bus *bus = dev->bus;
799	struct resource *r;
800
801	pci_bus_for_each_resource(bus, r) {
802	if (!r)
803	continue;
804	if (resource_contains(r1: r, r2: res)) {
805
806	/*
807	* If the window is prefetchable but the BAR is
808	* not, the allocator made a mistake.
809	*/
810	if (r->flags & IORESOURCE_PREFETCH &&
811	!(res->flags & IORESOURCE_PREFETCH))
812	return NULL;
813
814	/*
815	* If we're below a transparent bridge, there may
816	* be both a positively-decoded aperture and a
817	* subtractively-decoded region that contain the BAR.
818	* We want the positively-decoded one, so this depends
819	* on pci_bus_for_each_resource() giving us those
820	* first.
821	*/
822	return r;
823	}
824	}
825	return NULL;
826	}
827	EXPORT_SYMBOL(pci_find_parent_resource);
828
829	/**
830	* pci_find_resource - Return matching PCI device resource
831	* @dev: PCI device to query
832	* @res: Resource to look for
833	*
834	* Goes over standard PCI resources (BARs) and checks if the given resource
835	* is partially or fully contained in any of them. In that case the
836	* matching resource is returned, %NULL otherwise.
837	*/
838	struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
839	{
840	int i;
841
842	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
843	struct resource *r = &dev->resource[i];
844
845	if (r->start && resource_contains(r1: r, r2: res))
846	return r;
847	}
848
849	return NULL;
850	}
851	EXPORT_SYMBOL(pci_find_resource);
852
853	/**
854	* pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
855	* @dev: the PCI device to operate on
856	* @pos: config space offset of status word
857	* @mask: mask of bit(s) to care about in status word
858	*
859	* Return 1 when mask bit(s) in status word clear, 0 otherwise.
860	*/
861	int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
862	{
863	int i;
864
865	/* Wait for Transaction Pending bit clean */
866	for (i = 0; i < 4; i++) {
867	u16 status;
868	if (i)
869	msleep(msecs: (1 << (i - 1)) * 100);
870
871	pci_read_config_word(dev, where: pos, val: &status);
872	if (!(status & mask))
873	return 1;
874	}
875
876	return 0;
877	}
878
879	static int pci_acs_enable;
880
881	/**
882	* pci_request_acs - ask for ACS to be enabled if supported
883	*/
884	void pci_request_acs(void)
885	{
886	pci_acs_enable = 1;
887	}
888
889	static const char *disable_acs_redir_param;
890
891	/**
892	* pci_disable_acs_redir - disable ACS redirect capabilities
893	* @dev: the PCI device
894	*
895	* For only devices specified in the disable_acs_redir parameter.
896	*/
897	static void pci_disable_acs_redir(struct pci_dev *dev)
898	{
899	int ret = 0;
900	const char *p;
901	int pos;
902	u16 ctrl;
903
904	if (!disable_acs_redir_param)
905	return;
906
907	p = disable_acs_redir_param;
908	while (*p) {
909	ret = pci_dev_str_match(dev, p, endptr: &p);
910	if (ret < 0) {
911	pr_info_once("PCI: Can't parse disable_acs_redir parameter: %s\n",
912	disable_acs_redir_param);
913
914	break;
915	} else if (ret == 1) {
916	/* Found a match */
917	break;
918	}
919
920	if (*p != ';' && *p != ',') {
921	/* End of param or invalid format */
922	break;
923	}
924	p++;
925	}
926
927	if (ret != 1)
928	return;
929
930	if (!pci_dev_specific_disable_acs_redir(dev))
931	return;
932
933	pos = dev->acs_cap;
934	if (!pos) {
935	pci_warn(dev, "cannot disable ACS redirect for this hardware as it does not have ACS capabilities\n");
936	return;
937	}
938
939	pci_read_config_word(dev, where: pos + PCI_ACS_CTRL, val: &ctrl);
940
941	/* P2P Request & Completion Redirect */
942	ctrl &= ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC);
943
944	pci_write_config_word(dev, where: pos + PCI_ACS_CTRL, val: ctrl);
945
946	pci_info(dev, "disabled ACS redirect\n");
947	}
948
949	/**
950	* pci_std_enable_acs - enable ACS on devices using standard ACS capabilities
951	* @dev: the PCI device
952	*/
953	static void pci_std_enable_acs(struct pci_dev *dev)
954	{
955	int pos;
956	u16 cap;
957	u16 ctrl;
958
959	pos = dev->acs_cap;
960	if (!pos)
961	return;
962
963	pci_read_config_word(dev, where: pos + PCI_ACS_CAP, val: &cap);
964	pci_read_config_word(dev, where: pos + PCI_ACS_CTRL, val: &ctrl);
965
966	/* Source Validation */
967	ctrl |= (cap & PCI_ACS_SV);
968
969	/* P2P Request Redirect */
970	ctrl |= (cap & PCI_ACS_RR);
971
972	/* P2P Completion Redirect */
973	ctrl |= (cap & PCI_ACS_CR);
974
975	/* Upstream Forwarding */
976	ctrl |= (cap & PCI_ACS_UF);
977
978	/* Enable Translation Blocking for external devices and noats */
979	if (pci_ats_disabled() || dev->external_facing || dev->untrusted)
980	ctrl |= (cap & PCI_ACS_TB);
981
982	pci_write_config_word(dev, where: pos + PCI_ACS_CTRL, val: ctrl);
983	}
984
985	/**
986	* pci_enable_acs - enable ACS if hardware support it
987	* @dev: the PCI device
988	*/
989	static void pci_enable_acs(struct pci_dev *dev)
990	{
991	if (!pci_acs_enable)
992	goto disable_acs_redir;
993
994	if (!pci_dev_specific_enable_acs(dev))
995	goto disable_acs_redir;
996
997	pci_std_enable_acs(dev);
998
999	disable_acs_redir:
1000	/*
1001	* Note: pci_disable_acs_redir() must be called even if ACS was not
1002	* enabled by the kernel because it may have been enabled by
1003	* platform firmware. So if we are told to disable it, we should
1004	* always disable it after setting the kernel's default
1005	* preferences.
1006	*/
1007	pci_disable_acs_redir(dev);
1008	}
1009
1010	/**
1011	* pci_restore_bars - restore a device's BAR values (e.g. after wake-up)
1012	* @dev: PCI device to have its BARs restored
1013	*
1014	* Restore the BAR values for a given device, so as to make it
1015	* accessible by its driver.
1016	*/
1017	static void pci_restore_bars(struct pci_dev *dev)
1018	{
1019	int i;
1020
1021	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
1022	pci_update_resource(dev, resno: i);
1023	}
1024
1025	static inline bool platform_pci_power_manageable(struct pci_dev *dev)
1026	{
1027	if (pci_use_mid_pm())
1028	return true;
1029
1030	return acpi_pci_power_manageable(dev);
1031	}
1032
1033	static inline int platform_pci_set_power_state(struct pci_dev *dev,
1034	pci_power_t t)
1035	{
1036	if (pci_use_mid_pm())
1037	return mid_pci_set_power_state(pdev: dev, state: t);
1038
1039	return acpi_pci_set_power_state(dev, state: t);
1040	}
1041
1042	static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
1043	{
1044	if (pci_use_mid_pm())
1045	return mid_pci_get_power_state(pdev: dev);
1046
1047	return acpi_pci_get_power_state(dev);
1048	}
1049
1050	static inline void platform_pci_refresh_power_state(struct pci_dev *dev)
1051	{
1052	if (!pci_use_mid_pm())
1053	acpi_pci_refresh_power_state(dev);
1054	}
1055
1056	static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
1057	{
1058	if (pci_use_mid_pm())
1059	return PCI_POWER_ERROR;
1060
1061	return acpi_pci_choose_state(pdev: dev);
1062	}
1063
1064	static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable)
1065	{
1066	if (pci_use_mid_pm())
1067	return PCI_POWER_ERROR;
1068
1069	return acpi_pci_wakeup(dev, enable);
1070	}
1071
1072	static inline bool platform_pci_need_resume(struct pci_dev *dev)
1073	{
1074	if (pci_use_mid_pm())
1075	return false;
1076
1077	return acpi_pci_need_resume(dev);
1078	}
1079
1080	static inline bool platform_pci_bridge_d3(struct pci_dev *dev)
1081	{
1082	if (pci_use_mid_pm())
1083	return false;
1084
1085	return acpi_pci_bridge_d3(dev);
1086	}
1087
1088	/**
1089	* pci_update_current_state - Read power state of given device and cache it
1090	* @dev: PCI device to handle.
1091	* @state: State to cache in case the device doesn't have the PM capability
1092	*
1093	* The power state is read from the PMCSR register, which however is
1094	* inaccessible in D3cold. The platform firmware is therefore queried first
1095	* to detect accessibility of the register. In case the platform firmware
1096	* reports an incorrect state or the device isn't power manageable by the
1097	* platform at all, we try to detect D3cold by testing accessibility of the
1098	* vendor ID in config space.
1099	*/
1100	void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
1101	{
1102	if (platform_pci_get_power_state(dev) == PCI_D3cold) {
1103	dev->current_state = PCI_D3cold;
1104	} else if (dev->pm_cap) {
1105	u16 pmcsr;
1106
1107	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
1108	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1109	dev->current_state = PCI_D3cold;
1110	return;
1111	}
1112	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1113	} else {
1114	dev->current_state = state;
1115	}
1116	}
1117
1118	/**
1119	* pci_refresh_power_state - Refresh the given device's power state data
1120	* @dev: Target PCI device.
1121	*
1122	* Ask the platform to refresh the devices power state information and invoke
1123	* pci_update_current_state() to update its current PCI power state.
1124	*/
1125	void pci_refresh_power_state(struct pci_dev *dev)
1126	{
1127	platform_pci_refresh_power_state(dev);
1128	pci_update_current_state(dev, state: dev->current_state);
1129	}
1130
1131	/**
1132	* pci_platform_power_transition - Use platform to change device power state
1133	* @dev: PCI device to handle.
1134	* @state: State to put the device into.
1135	*/
1136	int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
1137	{
1138	int error;
1139
1140	error = platform_pci_set_power_state(dev, t: state);
1141	if (!error)
1142	pci_update_current_state(dev, state);
1143	else if (!dev->pm_cap) /* Fall back to PCI_D0 */
1144	dev->current_state = PCI_D0;
1145
1146	return error;
1147	}
1148	EXPORT_SYMBOL_GPL(pci_platform_power_transition);
1149
1150	static int pci_resume_one(struct pci_dev *pci_dev, void *ign)
1151	{
1152	pm_request_resume(dev: &pci_dev->dev);
1153	return 0;
1154	}
1155
1156	/**
1157	* pci_resume_bus - Walk given bus and runtime resume devices on it
1158	* @bus: Top bus of the subtree to walk.
1159	*/
1160	void pci_resume_bus(struct pci_bus *bus)
1161	{
1162	if (bus)
1163	pci_walk_bus(top: bus, cb: pci_resume_one, NULL);
1164	}
1165
1166	static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout)
1167	{
1168	int delay = 1;
1169	bool retrain = false;
1170	struct pci_dev *bridge;
1171
1172	if (pci_is_pcie(dev)) {
1173	bridge = pci_upstream_bridge(dev);
1174	if (bridge)
1175	retrain = true;
1176	}
1177
1178	/*
1179	* After reset, the device should not silently discard config
1180	* requests, but it may still indicate that it needs more time by
1181	* responding to them with CRS completions. The Root Port will
1182	* generally synthesize ~0 (PCI_ERROR_RESPONSE) data to complete
1183	* the read (except when CRS SV is enabled and the read was for the
1184	* Vendor ID; in that case it synthesizes 0x0001 data).
1185	*
1186	* Wait for the device to return a non-CRS completion. Read the
1187	* Command register instead of Vendor ID so we don't have to
1188	* contend with the CRS SV value.
1189	*/
1190	for (;;) {
1191	u32 id;
1192
1193	pci_read_config_dword(dev, PCI_COMMAND, val: &id);
1194	if (!PCI_POSSIBLE_ERROR(id))
1195	break;
1196
1197	if (delay > timeout) {
1198	pci_warn(dev, "not ready %dms after %s; giving up\n",
1199	delay - 1, reset_type);
1200	return -ENOTTY;
1201	}
1202
1203	if (delay > PCI_RESET_WAIT) {
1204	if (retrain) {
1205	retrain = false;
1206	if (pcie_failed_link_retrain(dev: bridge)) {
1207	delay = 1;
1208	continue;
1209	}
1210	}
1211	pci_info(dev, "not ready %dms after %s; waiting\n",
1212	delay - 1, reset_type);
1213	}
1214
1215	msleep(msecs: delay);
1216	delay *= 2;
1217	}
1218
1219	if (delay > PCI_RESET_WAIT)
1220	pci_info(dev, "ready %dms after %s\n", delay - 1,
1221	reset_type);
1222
1223	return 0;
1224	}
1225
1226	/**
1227	* pci_power_up - Put the given device into D0
1228	* @dev: PCI device to power up
1229	*
1230	* On success, return 0 or 1, depending on whether or not it is necessary to
1231	* restore the device's BARs subsequently (1 is returned in that case).
1232	*
1233	* On failure, return a negative error code. Always return failure if @dev
1234	* lacks a Power Management Capability, even if the platform was able to
1235	* put the device in D0 via non-PCI means.
1236	*/
1237	int pci_power_up(struct pci_dev *dev)
1238	{
1239	bool need_restore;
1240	pci_power_t state;
1241	u16 pmcsr;
1242
1243	platform_pci_set_power_state(dev, PCI_D0);
1244
1245	if (!dev->pm_cap) {
1246	state = platform_pci_get_power_state(dev);
1247	if (state == PCI_UNKNOWN)
1248	dev->current_state = PCI_D0;
1249	else
1250	dev->current_state = state;
1251
1252	return -EIO;
1253	}
1254
1255	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
1256	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1257	pci_err(dev, "Unable to change power state from %s to D0, device inaccessible\n",
1258	pci_power_name(dev->current_state));
1259	dev->current_state = PCI_D3cold;
1260	return -EIO;
1261	}
1262
1263	state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1264
1265	need_restore = (state == PCI_D3hot || dev->current_state >= PCI_D3hot) &&
1266	!(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
1267
1268	if (state == PCI_D0)
1269	goto end;
1270
1271	/*
1272	* Force the entire word to 0. This doesn't affect PME_Status, disables
1273	* PME_En, and sets PowerState to 0.
1274	*/
1275	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: 0);
1276
1277	/* Mandatory transition delays; see PCI PM 1.2. */
1278	if (state == PCI_D3hot)
1279	pci_dev_d3_sleep(dev);
1280	else if (state == PCI_D2)
1281	udelay(PCI_PM_D2_DELAY);
1282
1283	end:
1284	dev->current_state = PCI_D0;
1285	if (need_restore)
1286	return 1;
1287
1288	return 0;
1289	}
1290
1291	/**
1292	* pci_set_full_power_state - Put a PCI device into D0 and update its state
1293	* @dev: PCI device to power up
1294	*
1295	* Call pci_power_up() to put @dev into D0, read from its PCI_PM_CTRL register
1296	* to confirm the state change, restore its BARs if they might be lost and
1297	* reconfigure ASPM in accordance with the new power state.
1298	*
1299	* If pci_restore_state() is going to be called right after a power state change
1300	* to D0, it is more efficient to use pci_power_up() directly instead of this
1301	* function.
1302	*/
1303	static int pci_set_full_power_state(struct pci_dev *dev)
1304	{
1305	u16 pmcsr;
1306	int ret;
1307
1308	ret = pci_power_up(dev);
1309	if (ret < 0) {
1310	if (dev->current_state == PCI_D0)
1311	return 0;
1312
1313	return ret;
1314	}
1315
1316	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
1317	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1318	if (dev->current_state != PCI_D0) {
1319	pci_info_ratelimited(dev, "Refused to change power state from %s to D0\n",
1320	pci_power_name(dev->current_state));
1321	} else if (ret > 0) {
1322	/*
1323	* According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
1324	* INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
1325	* from D3hot to D0 _may_ perform an internal reset, thereby
1326	* going to "D0 Uninitialized" rather than "D0 Initialized".
1327	* For example, at least some versions of the 3c905B and the
1328	* 3c556B exhibit this behaviour.
1329	*
1330	* At least some laptop BIOSen (e.g. the Thinkpad T21) leave
1331	* devices in a D3hot state at boot. Consequently, we need to
1332	* restore at least the BARs so that the device will be
1333	* accessible to its driver.
1334	*/
1335	pci_restore_bars(dev);
1336	}
1337
1338	return 0;
1339	}
1340
1341	/**
1342	* __pci_dev_set_current_state - Set current state of a PCI device
1343	* @dev: Device to handle
1344	* @data: pointer to state to be set
1345	*/
1346	static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
1347	{
1348	pci_power_t state = *(pci_power_t *)data;
1349
1350	dev->current_state = state;
1351	return 0;
1352	}
1353
1354	/**
1355	* pci_bus_set_current_state - Walk given bus and set current state of devices
1356	* @bus: Top bus of the subtree to walk.
1357	* @state: state to be set
1358	*/
1359	void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
1360	{
1361	if (bus)
1362	pci_walk_bus(top: bus, cb: __pci_dev_set_current_state, userdata: &state);
1363	}
1364
1365	/**
1366	* pci_set_low_power_state - Put a PCI device into a low-power state.
1367	* @dev: PCI device to handle.
1368	* @state: PCI power state (D1, D2, D3hot) to put the device into.
1369	*
1370	* Use the device's PCI_PM_CTRL register to put it into a low-power state.
1371	*
1372	* RETURN VALUE:
1373	* -EINVAL if the requested state is invalid.
1374	* -EIO if device does not support PCI PM or its PM capabilities register has a
1375	* wrong version, or device doesn't support the requested state.
1376	* 0 if device already is in the requested state.
1377	* 0 if device's power state has been successfully changed.
1378	*/
1379	static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state)
1380	{
1381	u16 pmcsr;
1382
1383	if (!dev->pm_cap)
1384	return -EIO;
1385
1386	/*
1387	* Validate transition: We can enter D0 from any state, but if
1388	* we're already in a low-power state, we can only go deeper. E.g.,
1389	* we can go from D1 to D3, but we can't go directly from D3 to D1;
1390	* we'd have to go from D3 to D0, then to D1.
1391	*/
1392	if (dev->current_state <= PCI_D3cold && dev->current_state > state) {
1393	pci_dbg(dev, "Invalid power transition (from %s to %s)\n",
1394	pci_power_name(dev->current_state),
1395	pci_power_name(state));
1396	return -EINVAL;
1397	}
1398
1399	/* Check if this device supports the desired state */
1400	if ((state == PCI_D1 && !dev->d1_support)
1401	|| (state == PCI_D2 && !dev->d2_support))
1402	return -EIO;
1403
1404	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
1405	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1406	pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n",
1407	pci_power_name(dev->current_state),
1408	pci_power_name(state));
1409	dev->current_state = PCI_D3cold;
1410	return -EIO;
1411	}
1412
1413	pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1414	pmcsr |= state;
1415
1416	/* Enter specified state */
1417	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: pmcsr);
1418
1419	/* Mandatory power management transition delays; see PCI PM 1.2. */
1420	if (state == PCI_D3hot)
1421	pci_dev_d3_sleep(dev);
1422	else if (state == PCI_D2)
1423	udelay(PCI_PM_D2_DELAY);
1424
1425	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
1426	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1427	if (dev->current_state != state)
1428	pci_info_ratelimited(dev, "Refused to change power state from %s to %s\n",
1429	pci_power_name(dev->current_state),
1430	pci_power_name(state));
1431
1432	return 0;
1433	}
1434
1435	/**
1436	* pci_set_power_state - Set the power state of a PCI device
1437	* @dev: PCI device to handle.
1438	* @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
1439	*
1440	* Transition a device to a new power state, using the platform firmware and/or
1441	* the device's PCI PM registers.
1442	*
1443	* RETURN VALUE:
1444	* -EINVAL if the requested state is invalid.
1445	* -EIO if device does not support PCI PM or its PM capabilities register has a
1446	* wrong version, or device doesn't support the requested state.
1447	* 0 if the transition is to D1 or D2 but D1 and D2 are not supported.
1448	* 0 if device already is in the requested state.
1449	* 0 if the transition is to D3 but D3 is not supported.
1450	* 0 if device's power state has been successfully changed.
1451	*/
1452	int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
1453	{
1454	int error;
1455
1456	/* Bound the state we're entering */
1457	if (state > PCI_D3cold)
1458	state = PCI_D3cold;
1459	else if (state < PCI_D0)
1460	state = PCI_D0;
1461	else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
1462
1463	/*
1464	* If the device or the parent bridge do not support PCI
1465	* PM, ignore the request if we're doing anything other
1466	* than putting it into D0 (which would only happen on
1467	* boot).
1468	*/
1469	return 0;
1470
1471	/* Check if we're already there */
1472	if (dev->current_state == state)
1473	return 0;
1474
1475	if (state == PCI_D0)
1476	return pci_set_full_power_state(dev);
1477
1478	/*
1479	* This device is quirked not to be put into D3, so don't put it in
1480	* D3
1481	*/
1482	if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
1483	return 0;
1484
1485	if (state == PCI_D3cold) {
1486	/*
1487	* To put the device in D3cold, put it into D3hot in the native
1488	* way, then put it into D3cold using platform ops.
1489	*/
1490	error = pci_set_low_power_state(dev, PCI_D3hot);
1491
1492	if (pci_platform_power_transition(dev, PCI_D3cold))
1493	return error;
1494
1495	/* Powering off a bridge may power off the whole hierarchy */
1496	if (dev->current_state == PCI_D3cold)
1497	pci_bus_set_current_state(bus: dev->subordinate, PCI_D3cold);
1498	} else {
1499	error = pci_set_low_power_state(dev, state);
1500
1501	if (pci_platform_power_transition(dev, state))
1502	return error;
1503	}
1504
1505	return 0;
1506	}
1507	EXPORT_SYMBOL(pci_set_power_state);
1508
1509	#define PCI_EXP_SAVE_REGS 7
1510
1511	static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
1512	u16 cap, bool extended)
1513	{
1514	struct pci_cap_saved_state *tmp;
1515
1516	hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
1517	if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
1518	return tmp;
1519	}
1520	return NULL;
1521	}
1522
1523	struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
1524	{
1525	return _pci_find_saved_cap(pci_dev: dev, cap, extended: false);
1526	}
1527
1528	struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap)
1529	{
1530	return _pci_find_saved_cap(pci_dev: dev, cap, extended: true);
1531	}
1532
1533	static int pci_save_pcie_state(struct pci_dev *dev)
1534	{
1535	int i = 0;
1536	struct pci_cap_saved_state *save_state;
1537	u16 *cap;
1538
1539	if (!pci_is_pcie(dev))
1540	return 0;
1541
1542	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1543	if (!save_state) {
1544	pci_err(dev, "buffer not found in %s\n", __func__);
1545	return -ENOMEM;
1546	}
1547
1548	cap = (u16 *)&save_state->cap.data[0];
1549	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &cap[i++]);
1550	pcie_capability_read_word(dev, PCI_EXP_LNKCTL, val: &cap[i++]);
1551	pcie_capability_read_word(dev, PCI_EXP_SLTCTL, val: &cap[i++]);
1552	pcie_capability_read_word(dev, PCI_EXP_RTCTL, val: &cap[i++]);
1553	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, val: &cap[i++]);
1554	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, val: &cap[i++]);
1555	pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, val: &cap[i++]);
1556
1557	return 0;
1558	}
1559
1560	void pci_bridge_reconfigure_ltr(struct pci_dev *dev)
1561	{
1562	#ifdef CONFIG_PCIEASPM
1563	struct pci_dev *bridge;
1564	u32 ctl;
1565
1566	bridge = pci_upstream_bridge(dev);
1567	if (bridge && bridge->ltr_path) {
1568	pcie_capability_read_dword(dev: bridge, PCI_EXP_DEVCTL2, val: &ctl);
1569	if (!(ctl & PCI_EXP_DEVCTL2_LTR_EN)) {
1570	pci_dbg(bridge, "re-enabling LTR\n");
1571	pcie_capability_set_word(dev: bridge, PCI_EXP_DEVCTL2,
1572	PCI_EXP_DEVCTL2_LTR_EN);
1573	}
1574	}
1575	#endif
1576	}
1577
1578	static void pci_restore_pcie_state(struct pci_dev *dev)
1579	{
1580	int i = 0;
1581	struct pci_cap_saved_state *save_state;
1582	u16 *cap;
1583
1584	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1585	if (!save_state)
1586	return;
1587
1588	/*
1589	* Downstream ports reset the LTR enable bit when link goes down.
1590	* Check and re-configure the bit here before restoring device.
1591	* PCIe r5.0, sec 7.5.3.16.
1592	*/
1593	pci_bridge_reconfigure_ltr(dev);
1594
1595	cap = (u16 *)&save_state->cap.data[0];
1596	pcie_capability_write_word(dev, PCI_EXP_DEVCTL, val: cap[i++]);
1597	pcie_capability_write_word(dev, PCI_EXP_LNKCTL, val: cap[i++]);
1598	pcie_capability_write_word(dev, PCI_EXP_SLTCTL, val: cap[i++]);
1599	pcie_capability_write_word(dev, PCI_EXP_RTCTL, val: cap[i++]);
1600	pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, val: cap[i++]);
1601	pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, val: cap[i++]);
1602	pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, val: cap[i++]);
1603	}
1604
1605	static int pci_save_pcix_state(struct pci_dev *dev)
1606	{
1607	int pos;
1608	struct pci_cap_saved_state *save_state;
1609
1610	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1611	if (!pos)
1612	return 0;
1613
1614	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1615	if (!save_state) {
1616	pci_err(dev, "buffer not found in %s\n", __func__);
1617	return -ENOMEM;
1618	}
1619
1620	pci_read_config_word(dev, where: pos + PCI_X_CMD,
1621	val: (u16 *)save_state->cap.data);
1622
1623	return 0;
1624	}
1625
1626	static void pci_restore_pcix_state(struct pci_dev *dev)
1627	{
1628	int i = 0, pos;
1629	struct pci_cap_saved_state *save_state;
1630	u16 *cap;
1631
1632	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1633	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1634	if (!save_state || !pos)
1635	return;
1636	cap = (u16 *)&save_state->cap.data[0];
1637
1638	pci_write_config_word(dev, where: pos + PCI_X_CMD, val: cap[i++]);
1639	}
1640
1641	static void pci_save_ltr_state(struct pci_dev *dev)
1642	{
1643	int ltr;
1644	struct pci_cap_saved_state *save_state;
1645	u32 *cap;
1646
1647	if (!pci_is_pcie(dev))
1648	return;
1649
1650	ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);
1651	if (!ltr)
1652	return;
1653
1654	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);
1655	if (!save_state) {
1656	pci_err(dev, "no suspend buffer for LTR; ASPM issues possible after resume\n");
1657	return;
1658	}
1659
1660	/* Some broken devices only support dword access to LTR */
1661	cap = &save_state->cap.data[0];
1662	pci_read_config_dword(dev, where: ltr + PCI_LTR_MAX_SNOOP_LAT, val: cap);
1663	}
1664
1665	static void pci_restore_ltr_state(struct pci_dev *dev)
1666	{
1667	struct pci_cap_saved_state *save_state;
1668	int ltr;
1669	u32 *cap;
1670
1671	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);
1672	ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);
1673	if (!save_state || !ltr)
1674	return;
1675
1676	/* Some broken devices only support dword access to LTR */
1677	cap = &save_state->cap.data[0];
1678	pci_write_config_dword(dev, where: ltr + PCI_LTR_MAX_SNOOP_LAT, val: *cap);
1679	}
1680
1681	/**
1682	* pci_save_state - save the PCI configuration space of a device before
1683	* suspending
1684	* @dev: PCI device that we're dealing with
1685	*/
1686	int pci_save_state(struct pci_dev *dev)
1687	{
1688	int i;
1689	/* XXX: 100% dword access ok here? */
1690	for (i = 0; i < 16; i++) {
1691	pci_read_config_dword(dev, where: i * 4, val: &dev->saved_config_space[i]);
1692	pci_dbg(dev, "save config %#04x: %#010x\n",
1693	i * 4, dev->saved_config_space[i]);
1694	}
1695	dev->state_saved = true;
1696
1697	i = pci_save_pcie_state(dev);
1698	if (i != 0)
1699	return i;
1700
1701	i = pci_save_pcix_state(dev);
1702	if (i != 0)
1703	return i;
1704
1705	pci_save_ltr_state(dev);
1706	pci_save_dpc_state(dev);
1707	pci_save_aer_state(dev);
1708	pci_save_ptm_state(dev);
1709	return pci_save_vc_state(dev);
1710	}
1711	EXPORT_SYMBOL(pci_save_state);
1712
1713	static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
1714	u32 saved_val, int retry, bool force)
1715	{
1716	u32 val;
1717
1718	pci_read_config_dword(dev: pdev, where: offset, val: &val);
1719	if (!force && val == saved_val)
1720	return;
1721
1722	for (;;) {
1723	pci_dbg(pdev, "restore config %#04x: %#010x -> %#010x\n",
1724	offset, val, saved_val);
1725	pci_write_config_dword(dev: pdev, where: offset, val: saved_val);
1726	if (retry-- <= 0)
1727	return;
1728
1729	pci_read_config_dword(dev: pdev, where: offset, val: &val);
1730	if (val == saved_val)
1731	return;
1732
1733	mdelay(1);
1734	}
1735	}
1736
1737	static void pci_restore_config_space_range(struct pci_dev *pdev,
1738	int start, int end, int retry,
1739	bool force)
1740	{
1741	int index;
1742
1743	for (index = end; index >= start; index--)
1744	pci_restore_config_dword(pdev, offset: 4 * index,
1745	saved_val: pdev->saved_config_space[index],
1746	retry, force);
1747	}
1748
1749	static void pci_restore_config_space(struct pci_dev *pdev)
1750	{
1751	if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
1752	pci_restore_config_space_range(pdev, start: 10, end: 15, retry: 0, force: false);
1753	/* Restore BARs before the command register. */
1754	pci_restore_config_space_range(pdev, start: 4, end: 9, retry: 10, force: false);
1755	pci_restore_config_space_range(pdev, start: 0, end: 3, retry: 0, force: false);
1756	} else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
1757	pci_restore_config_space_range(pdev, start: 12, end: 15, retry: 0, force: false);
1758
1759	/*
1760	* Force rewriting of prefetch registers to avoid S3 resume
1761	* issues on Intel PCI bridges that occur when these
1762	* registers are not explicitly written.
1763	*/
1764	pci_restore_config_space_range(pdev, start: 9, end: 11, retry: 0, force: true);
1765	pci_restore_config_space_range(pdev, start: 0, end: 8, retry: 0, force: false);
1766	} else {
1767	pci_restore_config_space_range(pdev, start: 0, end: 15, retry: 0, force: false);
1768	}
1769	}
1770
1771	static void pci_restore_rebar_state(struct pci_dev *pdev)
1772	{
1773	unsigned int pos, nbars, i;
1774	u32 ctrl;
1775
1776	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
1777	if (!pos)
1778	return;
1779
1780	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
1781	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
1782
1783	for (i = 0; i < nbars; i++, pos += 8) {
1784	struct resource *res;
1785	int bar_idx, size;
1786
1787	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
1788	bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
1789	res = pdev->resource + bar_idx;
1790	size = pci_rebar_bytes_to_size(bytes: resource_size(res));
1791	ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
1792	ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
1793	pci_write_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: ctrl);
1794	}
1795	}
1796
1797	/**
1798	* pci_restore_state - Restore the saved state of a PCI device
1799	* @dev: PCI device that we're dealing with
1800	*/
1801	void pci_restore_state(struct pci_dev *dev)
1802	{
1803	if (!dev->state_saved)
1804	return;
1805
1806	/*
1807	* Restore max latencies (in the LTR capability) before enabling
1808	* LTR itself (in the PCIe capability).
1809	*/
1810	pci_restore_ltr_state(dev);
1811
1812	pci_restore_pcie_state(dev);
1813	pci_restore_pasid_state(pdev: dev);
1814	pci_restore_pri_state(pdev: dev);
1815	pci_restore_ats_state(dev);
1816	pci_restore_vc_state(dev);
1817	pci_restore_rebar_state(pdev: dev);
1818	pci_restore_dpc_state(dev);
1819	pci_restore_ptm_state(dev);
1820
1821	pci_aer_clear_status(dev);
1822	pci_restore_aer_state(dev);
1823
1824	pci_restore_config_space(pdev: dev);
1825
1826	pci_restore_pcix_state(dev);
1827	pci_restore_msi_state(dev);
1828
1829	/* Restore ACS and IOV configuration state */
1830	pci_enable_acs(dev);
1831	pci_restore_iov_state(dev);
1832
1833	dev->state_saved = false;
1834	}
1835	EXPORT_SYMBOL(pci_restore_state);
1836
1837	struct pci_saved_state {
1838	u32 config_space[16];
1839	struct pci_cap_saved_data cap[];
1840	};
1841
1842	/**
1843	* pci_store_saved_state - Allocate and return an opaque struct containing
1844	* the device saved state.
1845	* @dev: PCI device that we're dealing with
1846	*
1847	* Return NULL if no state or error.
1848	*/
1849	struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev)
1850	{
1851	struct pci_saved_state *state;
1852	struct pci_cap_saved_state *tmp;
1853	struct pci_cap_saved_data *cap;
1854	size_t size;
1855
1856	if (!dev->state_saved)
1857	return NULL;
1858
1859	size = sizeof(*state) + sizeof(struct pci_cap_saved_data);
1860
1861	hlist_for_each_entry(tmp, &dev->saved_cap_space, next)
1862	size += sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1863
1864	state = kzalloc(size, GFP_KERNEL);
1865	if (!state)
1866	return NULL;
1867
1868	memcpy(state->config_space, dev->saved_config_space,
1869	sizeof(state->config_space));
1870
1871	cap = state->cap;
1872	hlist_for_each_entry(tmp, &dev->saved_cap_space, next) {
1873	size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1874	memcpy(cap, &tmp->cap, len);
1875	cap = (struct pci_cap_saved_data *)((u8 *)cap + len);
1876	}
1877	/* Empty cap_save terminates list */
1878
1879	return state;
1880	}
1881	EXPORT_SYMBOL_GPL(pci_store_saved_state);
1882
1883	/**
1884	* pci_load_saved_state - Reload the provided save state into struct pci_dev.
1885	* @dev: PCI device that we're dealing with
1886	* @state: Saved state returned from pci_store_saved_state()
1887	*/
1888	int pci_load_saved_state(struct pci_dev *dev,
1889	struct pci_saved_state *state)
1890	{
1891	struct pci_cap_saved_data *cap;
1892
1893	dev->state_saved = false;
1894
1895	if (!state)
1896	return 0;
1897
1898	memcpy(dev->saved_config_space, state->config_space,
1899	sizeof(state->config_space));
1900
1901	cap = state->cap;
1902	while (cap->size) {
1903	struct pci_cap_saved_state *tmp;
1904
1905	tmp = _pci_find_saved_cap(pci_dev: dev, cap: cap->cap_nr, extended: cap->cap_extended);
1906	if (!tmp || tmp->cap.size != cap->size)
1907	return -EINVAL;
1908
1909	memcpy(tmp->cap.data, cap->data, tmp->cap.size);
1910	cap = (struct pci_cap_saved_data *)((u8 *)cap +
1911	sizeof(struct pci_cap_saved_data) + cap->size);
1912	}
1913
1914	dev->state_saved = true;
1915	return 0;
1916	}
1917	EXPORT_SYMBOL_GPL(pci_load_saved_state);
1918
1919	/**
1920	* pci_load_and_free_saved_state - Reload the save state pointed to by state,
1921	* and free the memory allocated for it.
1922	* @dev: PCI device that we're dealing with
1923	* @state: Pointer to saved state returned from pci_store_saved_state()
1924	*/
1925	int pci_load_and_free_saved_state(struct pci_dev *dev,
1926	struct pci_saved_state **state)
1927	{
1928	int ret = pci_load_saved_state(dev, *state);
1929	kfree(objp: *state);
1930	*state = NULL;
1931	return ret;
1932	}
1933	EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state);
1934
1935	int __weak pcibios_enable_device(struct pci_dev *dev, int bars)
1936	{
1937	return pci_enable_resources(dev, mask: bars);
1938	}
1939
1940	static int do_pci_enable_device(struct pci_dev *dev, int bars)
1941	{
1942	int err;
1943	struct pci_dev *bridge;
1944	u16 cmd;
1945	u8 pin;
1946
1947	err = pci_set_power_state(dev, PCI_D0);
1948	if (err < 0 && err != -EIO)
1949	return err;
1950
1951	bridge = pci_upstream_bridge(dev);
1952	if (bridge)
1953	pcie_aspm_powersave_config_link(pdev: bridge);
1954
1955	err = pcibios_enable_device(dev, bars);
1956	if (err < 0)
1957	return err;
1958	pci_fixup_device(pass: pci_fixup_enable, dev);
1959
1960	if (dev->msi_enabled || dev->msix_enabled)
1961	return 0;
1962
1963	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, val: &pin);
1964	if (pin) {
1965	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
1966	if (cmd & PCI_COMMAND_INTX_DISABLE)
1967	pci_write_config_word(dev, PCI_COMMAND,
1968	val: cmd & ~PCI_COMMAND_INTX_DISABLE);
1969	}
1970
1971	return 0;
1972	}
1973
1974	/**
1975	* pci_reenable_device - Resume abandoned device
1976	* @dev: PCI device to be resumed
1977	*
1978	* NOTE: This function is a backend of pci_default_resume() and is not supposed
1979	* to be called by normal code, write proper resume handler and use it instead.
1980	*/
1981	int pci_reenable_device(struct pci_dev *dev)
1982	{
1983	if (pci_is_enabled(pdev: dev))
1984	return do_pci_enable_device(dev, bars: (1 << PCI_NUM_RESOURCES) - 1);
1985	return 0;
1986	}
1987	EXPORT_SYMBOL(pci_reenable_device);
1988
1989	static void pci_enable_bridge(struct pci_dev *dev)
1990	{
1991	struct pci_dev *bridge;
1992	int retval;
1993
1994	bridge = pci_upstream_bridge(dev);
1995	if (bridge)
1996	pci_enable_bridge(dev: bridge);
1997
1998	if (pci_is_enabled(pdev: dev)) {
1999	if (!dev->is_busmaster)
2000	pci_set_master(dev);
2001	return;
2002	}
2003
2004	retval = pci_enable_device(dev);
2005	if (retval)
2006	pci_err(dev, "Error enabling bridge (%d), continuing\n",
2007	retval);
2008	pci_set_master(dev);
2009	}
2010
2011	static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
2012	{
2013	struct pci_dev *bridge;
2014	int err;
2015	int i, bars = 0;
2016
2017	/*
2018	* Power state could be unknown at this point, either due to a fresh
2019	* boot or a device removal call. So get the current power state
2020	* so that things like MSI message writing will behave as expected
2021	* (e.g. if the device really is in D0 at enable time).
2022	*/
2023	pci_update_current_state(dev, state: dev->current_state);
2024
2025	if (atomic_inc_return(v: &dev->enable_cnt) > 1)
2026	return 0; /* already enabled */
2027
2028	bridge = pci_upstream_bridge(dev);
2029	if (bridge)
2030	pci_enable_bridge(dev: bridge);
2031
2032	/* only skip sriov related */
2033	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
2034	if (dev->resource[i].flags & flags)
2035	bars |= (1 << i);
2036	for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++)
2037	if (dev->resource[i].flags & flags)
2038	bars |= (1 << i);
2039
2040	err = do_pci_enable_device(dev, bars);
2041	if (err < 0)
2042	atomic_dec(v: &dev->enable_cnt);
2043	return err;
2044	}
2045
2046	/**
2047	* pci_enable_device_io - Initialize a device for use with IO space
2048	* @dev: PCI device to be initialized
2049	*
2050	* Initialize device before it's used by a driver. Ask low-level code
2051	* to enable I/O resources. Wake up the device if it was suspended.
2052	* Beware, this function can fail.
2053	*/
2054	int pci_enable_device_io(struct pci_dev *dev)
2055	{
2056	return pci_enable_device_flags(dev, IORESOURCE_IO);
2057	}
2058	EXPORT_SYMBOL(pci_enable_device_io);
2059
2060	/**
2061	* pci_enable_device_mem - Initialize a device for use with Memory space
2062	* @dev: PCI device to be initialized
2063	*
2064	* Initialize device before it's used by a driver. Ask low-level code
2065	* to enable Memory resources. Wake up the device if it was suspended.
2066	* Beware, this function can fail.
2067	*/
2068	int pci_enable_device_mem(struct pci_dev *dev)
2069	{
2070	return pci_enable_device_flags(dev, IORESOURCE_MEM);
2071	}
2072	EXPORT_SYMBOL(pci_enable_device_mem);
2073
2074	/**
2075	* pci_enable_device - Initialize device before it's used by a driver.
2076	* @dev: PCI device to be initialized
2077	*
2078	* Initialize device before it's used by a driver. Ask low-level code
2079	* to enable I/O and memory. Wake up the device if it was suspended.
2080	* Beware, this function can fail.
2081	*
2082	* Note we don't actually enable the device many times if we call
2083	* this function repeatedly (we just increment the count).
2084	*/
2085	int pci_enable_device(struct pci_dev *dev)
2086	{
2087	return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO);
2088	}
2089	EXPORT_SYMBOL(pci_enable_device);
2090
2091	/*
2092	* Managed PCI resources. This manages device on/off, INTx/MSI/MSI-X
2093	* on/off and BAR regions. pci_dev itself records MSI/MSI-X status, so
2094	* there's no need to track it separately. pci_devres is initialized
2095	* when a device is enabled using managed PCI device enable interface.
2096	*/
2097	struct pci_devres {
2098	unsigned int enabled:1;
2099	unsigned int pinned:1;
2100	unsigned int orig_intx:1;
2101	unsigned int restore_intx:1;
2102	unsigned int mwi:1;
2103	u32 region_mask;
2104	};
2105
2106	static void pcim_release(struct device *gendev, void *res)
2107	{
2108	struct pci_dev *dev = to_pci_dev(gendev);
2109	struct pci_devres *this = res;
2110	int i;
2111
2112	for (i = 0; i < DEVICE_COUNT_RESOURCE; i++)
2113	if (this->region_mask & (1 << i))
2114	pci_release_region(dev, i);
2115
2116	if (this->mwi)
2117	pci_clear_mwi(dev);
2118
2119	if (this->restore_intx)
2120	pci_intx(dev, enable: this->orig_intx);
2121
2122	if (this->enabled && !this->pinned)
2123	pci_disable_device(dev);
2124	}
2125
2126	static struct pci_devres *get_pci_dr(struct pci_dev *pdev)
2127	{
2128	struct pci_devres *dr, *new_dr;
2129
2130	dr = devres_find(dev: &pdev->dev, release: pcim_release, NULL, NULL);
2131	if (dr)
2132	return dr;
2133
2134	new_dr = devres_alloc(pcim_release, sizeof(*new_dr), GFP_KERNEL);
2135	if (!new_dr)
2136	return NULL;
2137	return devres_get(dev: &pdev->dev, new_res: new_dr, NULL, NULL);
2138	}
2139
2140	static struct pci_devres *find_pci_dr(struct pci_dev *pdev)
2141	{
2142	if (pci_is_managed(pdev))
2143	return devres_find(dev: &pdev->dev, release: pcim_release, NULL, NULL);
2144	return NULL;
2145	}
2146
2147	/**
2148	* pcim_enable_device - Managed pci_enable_device()
2149	* @pdev: PCI device to be initialized
2150	*
2151	* Managed pci_enable_device().
2152	*/
2153	int pcim_enable_device(struct pci_dev *pdev)
2154	{
2155	struct pci_devres *dr;
2156	int rc;
2157
2158	dr = get_pci_dr(pdev);
2159	if (unlikely(!dr))
2160	return -ENOMEM;
2161	if (dr->enabled)
2162	return 0;
2163
2164	rc = pci_enable_device(pdev);
2165	if (!rc) {
2166	pdev->is_managed = 1;
2167	dr->enabled = 1;
2168	}
2169	return rc;
2170	}
2171	EXPORT_SYMBOL(pcim_enable_device);
2172
2173	/**
2174	* pcim_pin_device - Pin managed PCI device
2175	* @pdev: PCI device to pin
2176	*
2177	* Pin managed PCI device @pdev. Pinned device won't be disabled on
2178	* driver detach. @pdev must have been enabled with
2179	* pcim_enable_device().
2180	*/
2181	void pcim_pin_device(struct pci_dev *pdev)
2182	{
2183	struct pci_devres *dr;
2184
2185	dr = find_pci_dr(pdev);
2186	WARN_ON(!dr || !dr->enabled);
2187	if (dr)
2188	dr->pinned = 1;
2189	}
2190	EXPORT_SYMBOL(pcim_pin_device);
2191
2192	/*
2193	* pcibios_device_add - provide arch specific hooks when adding device dev
2194	* @dev: the PCI device being added
2195	*
2196	* Permits the platform to provide architecture specific functionality when
2197	* devices are added. This is the default implementation. Architecture
2198	* implementations can override this.
2199	*/
2200	int __weak pcibios_device_add(struct pci_dev *dev)
2201	{
2202	return 0;
2203	}
2204
2205	/**
2206	* pcibios_release_device - provide arch specific hooks when releasing
2207	* device dev
2208	* @dev: the PCI device being released
2209	*
2210	* Permits the platform to provide architecture specific functionality when
2211	* devices are released. This is the default implementation. Architecture
2212	* implementations can override this.
2213	*/
2214	void __weak pcibios_release_device(struct pci_dev *dev) {}
2215
2216	/**
2217	* pcibios_disable_device - disable arch specific PCI resources for device dev
2218	* @dev: the PCI device to disable
2219	*
2220	* Disables architecture specific PCI resources for the device. This
2221	* is the default implementation. Architecture implementations can
2222	* override this.
2223	*/
2224	void __weak pcibios_disable_device(struct pci_dev *dev) {}
2225
2226	/**
2227	* pcibios_penalize_isa_irq - penalize an ISA IRQ
2228	* @irq: ISA IRQ to penalize
2229	* @active: IRQ active or not
2230	*
2231	* Permits the platform to provide architecture-specific functionality when
2232	* penalizing ISA IRQs. This is the default implementation. Architecture
2233	* implementations can override this.
2234	*/
2235	void __weak pcibios_penalize_isa_irq(int irq, int active) {}
2236
2237	static void do_pci_disable_device(struct pci_dev *dev)
2238	{
2239	u16 pci_command;
2240
2241	pci_read_config_word(dev, PCI_COMMAND, val: &pci_command);
2242	if (pci_command & PCI_COMMAND_MASTER) {
2243	pci_command &= ~PCI_COMMAND_MASTER;
2244	pci_write_config_word(dev, PCI_COMMAND, val: pci_command);
2245	}
2246
2247	pcibios_disable_device(dev);
2248	}
2249
2250	/**
2251	* pci_disable_enabled_device - Disable device without updating enable_cnt
2252	* @dev: PCI device to disable
2253	*
2254	* NOTE: This function is a backend of PCI power management routines and is
2255	* not supposed to be called drivers.
2256	*/
2257	void pci_disable_enabled_device(struct pci_dev *dev)
2258	{
2259	if (pci_is_enabled(pdev: dev))
2260	do_pci_disable_device(dev);
2261	}
2262
2263	/**
2264	* pci_disable_device - Disable PCI device after use
2265	* @dev: PCI device to be disabled
2266	*
2267	* Signal to the system that the PCI device is not in use by the system
2268	* anymore. This only involves disabling PCI bus-mastering, if active.
2269	*
2270	* Note we don't actually disable the device until all callers of
2271	* pci_enable_device() have called pci_disable_device().
2272	*/
2273	void pci_disable_device(struct pci_dev *dev)
2274	{
2275	struct pci_devres *dr;
2276
2277	dr = find_pci_dr(pdev: dev);
2278	if (dr)
2279	dr->enabled = 0;
2280
2281	dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0,
2282	"disabling already-disabled device");
2283
2284	if (atomic_dec_return(v: &dev->enable_cnt) != 0)
2285	return;
2286
2287	do_pci_disable_device(dev);
2288
2289	dev->is_busmaster = 0;
2290	}
2291	EXPORT_SYMBOL(pci_disable_device);
2292
2293	/**
2294	* pcibios_set_pcie_reset_state - set reset state for device dev
2295	* @dev: the PCIe device reset
2296	* @state: Reset state to enter into
2297	*
2298	* Set the PCIe reset state for the device. This is the default
2299	* implementation. Architecture implementations can override this.
2300	*/
2301	int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev,
2302	enum pcie_reset_state state)
2303	{
2304	return -EINVAL;
2305	}
2306
2307	/**
2308	* pci_set_pcie_reset_state - set reset state for device dev
2309	* @dev: the PCIe device reset
2310	* @state: Reset state to enter into
2311	*
2312	* Sets the PCI reset state for the device.
2313	*/
2314	int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
2315	{
2316	return pcibios_set_pcie_reset_state(dev, state);
2317	}
2318	EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state);
2319
2320	#ifdef CONFIG_PCIEAER
2321	void pcie_clear_device_status(struct pci_dev *dev)
2322	{
2323	u16 sta;
2324
2325	pcie_capability_read_word(dev, PCI_EXP_DEVSTA, val: &sta);
2326	pcie_capability_write_word(dev, PCI_EXP_DEVSTA, val: sta);
2327	}
2328	#endif
2329
2330	/**
2331	* pcie_clear_root_pme_status - Clear root port PME interrupt status.
2332	* @dev: PCIe root port or event collector.
2333	*/
2334	void pcie_clear_root_pme_status(struct pci_dev *dev)
2335	{
2336	pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME);
2337	}
2338
2339	/**
2340	* pci_check_pme_status - Check if given device has generated PME.
2341	* @dev: Device to check.
2342	*
2343	* Check the PME status of the device and if set, clear it and clear PME enable
2344	* (if set). Return 'true' if PME status and PME enable were both set or
2345	* 'false' otherwise.
2346	*/
2347	bool pci_check_pme_status(struct pci_dev *dev)
2348	{
2349	int pmcsr_pos;
2350	u16 pmcsr;
2351	bool ret = false;
2352
2353	if (!dev->pm_cap)
2354	return false;
2355
2356	pmcsr_pos = dev->pm_cap + PCI_PM_CTRL;
2357	pci_read_config_word(dev, where: pmcsr_pos, val: &pmcsr);
2358	if (!(pmcsr & PCI_PM_CTRL_PME_STATUS))
2359	return false;
2360
2361	/* Clear PME status. */
2362	pmcsr |= PCI_PM_CTRL_PME_STATUS;
2363	if (pmcsr & PCI_PM_CTRL_PME_ENABLE) {
2364	/* Disable PME to avoid interrupt flood. */
2365	pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2366	ret = true;
2367	}
2368
2369	pci_write_config_word(dev, where: pmcsr_pos, val: pmcsr);
2370
2371	return ret;
2372	}
2373
2374	/**
2375	* pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set.
2376	* @dev: Device to handle.
2377	* @pme_poll_reset: Whether or not to reset the device's pme_poll flag.
2378	*
2379	* Check if @dev has generated PME and queue a resume request for it in that
2380	* case.
2381	*/
2382	static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset)
2383	{
2384	if (pme_poll_reset && dev->pme_poll)
2385	dev->pme_poll = false;
2386
2387	if (pci_check_pme_status(dev)) {
2388	pci_wakeup_event(dev);
2389	pm_request_resume(dev: &dev->dev);
2390	}
2391	return 0;
2392	}
2393
2394	/**
2395	* pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary.
2396	* @bus: Top bus of the subtree to walk.
2397	*/
2398	void pci_pme_wakeup_bus(struct pci_bus *bus)
2399	{
2400	if (bus)
2401	pci_walk_bus(top: bus, cb: pci_pme_wakeup, userdata: (void *)true);
2402	}
2403
2404
2405	/**
2406	* pci_pme_capable - check the capability of PCI device to generate PME#
2407	* @dev: PCI device to handle.
2408	* @state: PCI state from which device will issue PME#.
2409	*/
2410	bool pci_pme_capable(struct pci_dev *dev, pci_power_t state)
2411	{
2412	if (!dev->pm_cap)
2413	return false;
2414
2415	return !!(dev->pme_support & (1 << state));
2416	}
2417	EXPORT_SYMBOL(pci_pme_capable);
2418
2419	static void pci_pme_list_scan(struct work_struct *work)
2420	{
2421	struct pci_pme_device *pme_dev, *n;
2422
2423	mutex_lock(&pci_pme_list_mutex);
2424	list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) {
2425	struct pci_dev *pdev = pme_dev->dev;
2426
2427	if (pdev->pme_poll) {
2428	struct pci_dev *bridge = pdev->bus->self;
2429	struct device *dev = &pdev->dev;
2430	int pm_status;
2431
2432	/*
2433	* If bridge is in low power state, the
2434	* configuration space of subordinate devices
2435	* may be not accessible
2436	*/
2437	if (bridge && bridge->current_state != PCI_D0)
2438	continue;
2439
2440	/*
2441	* If the device is in a low power state it
2442	* should not be polled either.
2443	*/
2444	pm_status = pm_runtime_get_if_active(dev, ign_usage_count: true);
2445	if (!pm_status)
2446	continue;
2447
2448	if (pdev->current_state != PCI_D3cold)
2449	pci_pme_wakeup(dev: pdev, NULL);
2450
2451	if (pm_status > 0)
2452	pm_runtime_put(dev);
2453	} else {
2454	list_del(entry: &pme_dev->list);
2455	kfree(objp: pme_dev);
2456	}
2457	}
2458	if (!list_empty(head: &pci_pme_list))
2459	queue_delayed_work(wq: system_freezable_wq, dwork: &pci_pme_work,
2460	delay: msecs_to_jiffies(PME_TIMEOUT));
2461	mutex_unlock(lock: &pci_pme_list_mutex);
2462	}
2463
2464	static void __pci_pme_active(struct pci_dev *dev, bool enable)
2465	{
2466	u16 pmcsr;
2467
2468	if (!dev->pme_support)
2469	return;
2470
2471	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
2472	/* Clear PME_Status by writing 1 to it and enable PME# */
2473	pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE;
2474	if (!enable)
2475	pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2476
2477	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: pmcsr);
2478	}
2479
2480	/**
2481	* pci_pme_restore - Restore PME configuration after config space restore.
2482	* @dev: PCI device to update.
2483	*/
2484	void pci_pme_restore(struct pci_dev *dev)
2485	{
2486	u16 pmcsr;
2487
2488	if (!dev->pme_support)
2489	return;
2490
2491	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
2492	if (dev->wakeup_prepared) {
2493	pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2494	pmcsr &= ~PCI_PM_CTRL_PME_STATUS;
2495	} else {
2496	pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2497	pmcsr |= PCI_PM_CTRL_PME_STATUS;
2498	}
2499	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: pmcsr);
2500	}
2501
2502	/**
2503	* pci_pme_active - enable or disable PCI device's PME# function
2504	* @dev: PCI device to handle.
2505	* @enable: 'true' to enable PME# generation; 'false' to disable it.
2506	*
2507	* The caller must verify that the device is capable of generating PME# before
2508	* calling this function with @enable equal to 'true'.
2509	*/
2510	void pci_pme_active(struct pci_dev *dev, bool enable)
2511	{
2512	__pci_pme_active(dev, enable);
2513
2514	/*
2515	* PCI (as opposed to PCIe) PME requires that the device have
2516	* its PME# line hooked up correctly. Not all hardware vendors
2517	* do this, so the PME never gets delivered and the device
2518	* remains asleep. The easiest way around this is to
2519	* periodically walk the list of suspended devices and check
2520	* whether any have their PME flag set. The assumption is that
2521	* we'll wake up often enough anyway that this won't be a huge
2522	* hit, and the power savings from the devices will still be a
2523	* win.
2524	*
2525	* Although PCIe uses in-band PME message instead of PME# line
2526	* to report PME, PME does not work for some PCIe devices in
2527	* reality. For example, there are devices that set their PME
2528	* status bits, but don't really bother to send a PME message;
2529	* there are PCI Express Root Ports that don't bother to
2530	* trigger interrupts when they receive PME messages from the
2531	* devices below. So PME poll is used for PCIe devices too.
2532	*/
2533
2534	if (dev->pme_poll) {
2535	struct pci_pme_device *pme_dev;
2536	if (enable) {
2537	pme_dev = kmalloc(size: sizeof(struct pci_pme_device),
2538	GFP_KERNEL);
2539	if (!pme_dev) {
2540	pci_warn(dev, "can't enable PME#\n");
2541	return;
2542	}
2543	pme_dev->dev = dev;
2544	mutex_lock(&pci_pme_list_mutex);
2545	list_add(new: &pme_dev->list, head: &pci_pme_list);
2546	if (list_is_singular(head: &pci_pme_list))
2547	queue_delayed_work(wq: system_freezable_wq,
2548	dwork: &pci_pme_work,
2549	delay: msecs_to_jiffies(PME_TIMEOUT));
2550	mutex_unlock(lock: &pci_pme_list_mutex);
2551	} else {
2552	mutex_lock(&pci_pme_list_mutex);
2553	list_for_each_entry(pme_dev, &pci_pme_list, list) {
2554	if (pme_dev->dev == dev) {
2555	list_del(entry: &pme_dev->list);
2556	kfree(objp: pme_dev);
2557	break;
2558	}
2559	}
2560	mutex_unlock(lock: &pci_pme_list_mutex);
2561	}
2562	}
2563
2564	pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled");
2565	}
2566	EXPORT_SYMBOL(pci_pme_active);
2567
2568	/**
2569	* __pci_enable_wake - enable PCI device as wakeup event source
2570	* @dev: PCI device affected
2571	* @state: PCI state from which device will issue wakeup events
2572	* @enable: True to enable event generation; false to disable
2573	*
2574	* This enables the device as a wakeup event source, or disables it.
2575	* When such events involves platform-specific hooks, those hooks are
2576	* called automatically by this routine.
2577	*
2578	* Devices with legacy power management (no standard PCI PM capabilities)
2579	* always require such platform hooks.
2580	*
2581	* RETURN VALUE:
2582	* 0 is returned on success
2583	* -EINVAL is returned if device is not supposed to wake up the system
2584	* Error code depending on the platform is returned if both the platform and
2585	* the native mechanism fail to enable the generation of wake-up events
2586	*/
2587	static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable)
2588	{
2589	int ret = 0;
2590
2591	/*
2592	* Bridges that are not power-manageable directly only signal
2593	* wakeup on behalf of subordinate devices which is set up
2594	* elsewhere, so skip them. However, bridges that are
2595	* power-manageable may signal wakeup for themselves (for example,
2596	* on a hotplug event) and they need to be covered here.
2597	*/
2598	if (!pci_power_manageable(pci_dev: dev))
2599	return 0;
2600
2601	/* Don't do the same thing twice in a row for one device. */
2602	if (!!enable == !!dev->wakeup_prepared)
2603	return 0;
2604
2605	/*
2606	* According to "PCI System Architecture" 4th ed. by Tom Shanley & Don
2607	* Anderson we should be doing PME# wake enable followed by ACPI wake
2608	* enable. To disable wake-up we call the platform first, for symmetry.
2609	*/
2610
2611	if (enable) {
2612	int error;
2613
2614	/*
2615	* Enable PME signaling if the device can signal PME from
2616	* D3cold regardless of whether or not it can signal PME from
2617	* the current target state, because that will allow it to
2618	* signal PME when the hierarchy above it goes into D3cold and
2619	* the device itself ends up in D3cold as a result of that.
2620	*/
2621	if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold))
2622	pci_pme_active(dev, true);
2623	else
2624	ret = 1;
2625	error = platform_pci_set_wakeup(dev, enable: true);
2626	if (ret)
2627	ret = error;
2628	if (!ret)
2629	dev->wakeup_prepared = true;
2630	} else {
2631	platform_pci_set_wakeup(dev, enable: false);
2632	pci_pme_active(dev, false);
2633	dev->wakeup_prepared = false;
2634	}
2635
2636	return ret;
2637	}
2638
2639	/**
2640	* pci_enable_wake - change wakeup settings for a PCI device
2641	* @pci_dev: Target device
2642	* @state: PCI state from which device will issue wakeup events
2643	* @enable: Whether or not to enable event generation
2644	*
2645	* If @enable is set, check device_may_wakeup() for the device before calling
2646	* __pci_enable_wake() for it.
2647	*/
2648	int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable)
2649	{
2650	if (enable && !device_may_wakeup(dev: &pci_dev->dev))
2651	return -EINVAL;
2652
2653	return __pci_enable_wake(dev: pci_dev, state, enable);
2654	}
2655	EXPORT_SYMBOL(pci_enable_wake);
2656
2657	/**
2658	* pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold
2659	* @dev: PCI device to prepare
2660	* @enable: True to enable wake-up event generation; false to disable
2661	*
2662	* Many drivers want the device to wake up the system from D3_hot or D3_cold
2663	* and this function allows them to set that up cleanly - pci_enable_wake()
2664	* should not be called twice in a row to enable wake-up due to PCI PM vs ACPI
2665	* ordering constraints.
2666	*
2667	* This function only returns error code if the device is not allowed to wake
2668	* up the system from sleep or it is not capable of generating PME# from both
2669	* D3_hot and D3_cold and the platform is unable to enable wake-up power for it.
2670	*/
2671	int pci_wake_from_d3(struct pci_dev *dev, bool enable)
2672	{
2673	return pci_pme_capable(dev, PCI_D3cold) ?
2674	pci_enable_wake(dev, PCI_D3cold, enable) :
2675	pci_enable_wake(dev, PCI_D3hot, enable);
2676	}
2677	EXPORT_SYMBOL(pci_wake_from_d3);
2678
2679	/**
2680	* pci_target_state - find an appropriate low power state for a given PCI dev
2681	* @dev: PCI device
2682	* @wakeup: Whether or not wakeup functionality will be enabled for the device.
2683	*
2684	* Use underlying platform code to find a supported low power state for @dev.
2685	* If the platform can't manage @dev, return the deepest state from which it
2686	* can generate wake events, based on any available PME info.
2687	*/
2688	static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup)
2689	{
2690	if (platform_pci_power_manageable(dev)) {
2691	/*
2692	* Call the platform to find the target state for the device.
2693	*/
2694	pci_power_t state = platform_pci_choose_state(dev);
2695
2696	switch (state) {
2697	case PCI_POWER_ERROR:
2698	case PCI_UNKNOWN:
2699	return PCI_D3hot;
2700
2701	case PCI_D1:
2702	case PCI_D2:
2703	if (pci_no_d1d2(dev))
2704	return PCI_D3hot;
2705	}
2706
2707	return state;
2708	}
2709
2710	/*
2711	* If the device is in D3cold even though it's not power-manageable by
2712	* the platform, it may have been powered down by non-standard means.
2713	* Best to let it slumber.
2714	*/
2715	if (dev->current_state == PCI_D3cold)
2716	return PCI_D3cold;
2717	else if (!dev->pm_cap)
2718	return PCI_D0;
2719
2720	if (wakeup && dev->pme_support) {
2721	pci_power_t state = PCI_D3hot;
2722
2723	/*
2724	* Find the deepest state from which the device can generate
2725	* PME#.
2726	*/
2727	while (state && !(dev->pme_support & (1 << state)))
2728	state--;
2729
2730	if (state)
2731	return state;
2732	else if (dev->pme_support & 1)
2733	return PCI_D0;
2734	}
2735
2736	return PCI_D3hot;
2737	}
2738
2739	/**
2740	* pci_prepare_to_sleep - prepare PCI device for system-wide transition
2741	* into a sleep state
2742	* @dev: Device to handle.
2743	*
2744	* Choose the power state appropriate for the device depending on whether
2745	* it can wake up the system and/or is power manageable by the platform
2746	* (PCI_D3hot is the default) and put the device into that state.
2747	*/
2748	int pci_prepare_to_sleep(struct pci_dev *dev)
2749	{
2750	bool wakeup = device_may_wakeup(dev: &dev->dev);
2751	pci_power_t target_state = pci_target_state(dev, wakeup);
2752	int error;
2753
2754	if (target_state == PCI_POWER_ERROR)
2755	return -EIO;
2756
2757	pci_enable_wake(dev, target_state, wakeup);
2758
2759	error = pci_set_power_state(dev, target_state);
2760
2761	if (error)
2762	pci_enable_wake(dev, target_state, false);
2763
2764	return error;
2765	}
2766	EXPORT_SYMBOL(pci_prepare_to_sleep);
2767
2768	/**
2769	* pci_back_from_sleep - turn PCI device on during system-wide transition
2770	* into working state
2771	* @dev: Device to handle.
2772	*
2773	* Disable device's system wake-up capability and put it into D0.
2774	*/
2775	int pci_back_from_sleep(struct pci_dev *dev)
2776	{
2777	int ret = pci_set_power_state(dev, PCI_D0);
2778
2779	if (ret)
2780	return ret;
2781
2782	pci_enable_wake(dev, PCI_D0, false);
2783	return 0;
2784	}
2785	EXPORT_SYMBOL(pci_back_from_sleep);
2786
2787	/**
2788	* pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend.
2789	* @dev: PCI device being suspended.
2790	*
2791	* Prepare @dev to generate wake-up events at run time and put it into a low
2792	* power state.
2793	*/
2794	int pci_finish_runtime_suspend(struct pci_dev *dev)
2795	{
2796	pci_power_t target_state;
2797	int error;
2798
2799	target_state = pci_target_state(dev, wakeup: device_can_wakeup(dev: &dev->dev));
2800	if (target_state == PCI_POWER_ERROR)
2801	return -EIO;
2802
2803	__pci_enable_wake(dev, state: target_state, enable: pci_dev_run_wake(dev));
2804
2805	error = pci_set_power_state(dev, target_state);
2806
2807	if (error)
2808	pci_enable_wake(dev, target_state, false);
2809
2810	return error;
2811	}
2812
2813	/**
2814	* pci_dev_run_wake - Check if device can generate run-time wake-up events.
2815	* @dev: Device to check.
2816	*
2817	* Return true if the device itself is capable of generating wake-up events
2818	* (through the platform or using the native PCIe PME) or if the device supports
2819	* PME and one of its upstream bridges can generate wake-up events.
2820	*/
2821	bool pci_dev_run_wake(struct pci_dev *dev)
2822	{
2823	struct pci_bus *bus = dev->bus;
2824
2825	if (!dev->pme_support)
2826	return false;
2827
2828	/* PME-capable in principle, but not from the target power state */
2829	if (!pci_pme_capable(dev, pci_target_state(dev, wakeup: true)))
2830	return false;
2831
2832	if (device_can_wakeup(dev: &dev->dev))
2833	return true;
2834
2835	while (bus->parent) {
2836	struct pci_dev *bridge = bus->self;
2837
2838	if (device_can_wakeup(dev: &bridge->dev))
2839	return true;
2840
2841	bus = bus->parent;
2842	}
2843
2844	/* We have reached the root bus. */
2845	if (bus->bridge)
2846	return device_can_wakeup(dev: bus->bridge);
2847
2848	return false;
2849	}
2850	EXPORT_SYMBOL_GPL(pci_dev_run_wake);
2851
2852	/**
2853	* pci_dev_need_resume - Check if it is necessary to resume the device.
2854	* @pci_dev: Device to check.
2855	*
2856	* Return 'true' if the device is not runtime-suspended or it has to be
2857	* reconfigured due to wakeup settings difference between system and runtime
2858	* suspend, or the current power state of it is not suitable for the upcoming
2859	* (system-wide) transition.
2860	*/
2861	bool pci_dev_need_resume(struct pci_dev *pci_dev)
2862	{
2863	struct device *dev = &pci_dev->dev;
2864	pci_power_t target_state;
2865
2866	if (!pm_runtime_suspended(dev) || platform_pci_need_resume(dev: pci_dev))
2867	return true;
2868
2869	target_state = pci_target_state(dev: pci_dev, wakeup: device_may_wakeup(dev));
2870
2871	/*
2872	* If the earlier platform check has not triggered, D3cold is just power
2873	* removal on top of D3hot, so no need to resume the device in that
2874	* case.
2875	*/
2876	return target_state != pci_dev->current_state &&
2877	target_state != PCI_D3cold &&
2878	pci_dev->current_state != PCI_D3hot;
2879	}
2880
2881	/**
2882	* pci_dev_adjust_pme - Adjust PME setting for a suspended device.
2883	* @pci_dev: Device to check.
2884	*
2885	* If the device is suspended and it is not configured for system wakeup,
2886	* disable PME for it to prevent it from waking up the system unnecessarily.
2887	*
2888	* Note that if the device's power state is D3cold and the platform check in
2889	* pci_dev_need_resume() has not triggered, the device's configuration need not
2890	* be changed.
2891	*/
2892	void pci_dev_adjust_pme(struct pci_dev *pci_dev)
2893	{
2894	struct device *dev = &pci_dev->dev;
2895
2896	spin_lock_irq(lock: &dev->power.lock);
2897
2898	if (pm_runtime_suspended(dev) && !device_may_wakeup(dev) &&
2899	pci_dev->current_state < PCI_D3cold)
2900	__pci_pme_active(dev: pci_dev, enable: false);
2901
2902	spin_unlock_irq(lock: &dev->power.lock);
2903	}
2904
2905	/**
2906	* pci_dev_complete_resume - Finalize resume from system sleep for a device.
2907	* @pci_dev: Device to handle.
2908	*
2909	* If the device is runtime suspended and wakeup-capable, enable PME for it as
2910	* it might have been disabled during the prepare phase of system suspend if
2911	* the device was not configured for system wakeup.
2912	*/
2913	void pci_dev_complete_resume(struct pci_dev *pci_dev)
2914	{
2915	struct device *dev = &pci_dev->dev;
2916
2917	if (!pci_dev_run_wake(pci_dev))
2918	return;
2919
2920	spin_lock_irq(lock: &dev->power.lock);
2921
2922	if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold)
2923	__pci_pme_active(dev: pci_dev, enable: true);
2924
2925	spin_unlock_irq(lock: &dev->power.lock);
2926	}
2927
2928	/**
2929	* pci_choose_state - Choose the power state of a PCI device.
2930	* @dev: Target PCI device.
2931	* @state: Target state for the whole system.
2932	*
2933	* Returns PCI power state suitable for @dev and @state.
2934	*/
2935	pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
2936	{
2937	if (state.event == PM_EVENT_ON)
2938	return PCI_D0;
2939
2940	return pci_target_state(dev, wakeup: false);
2941	}
2942	EXPORT_SYMBOL(pci_choose_state);
2943
2944	void pci_config_pm_runtime_get(struct pci_dev *pdev)
2945	{
2946	struct device *dev = &pdev->dev;
2947	struct device *parent = dev->parent;
2948
2949	if (parent)
2950	pm_runtime_get_sync(dev: parent);
2951	pm_runtime_get_noresume(dev);
2952	/*
2953	* pdev->current_state is set to PCI_D3cold during suspending,
2954	* so wait until suspending completes
2955	*/
2956	pm_runtime_barrier(dev);
2957	/*
2958	* Only need to resume devices in D3cold, because config
2959	* registers are still accessible for devices suspended but
2960	* not in D3cold.
2961	*/
2962	if (pdev->current_state == PCI_D3cold)
2963	pm_runtime_resume(dev);
2964	}
2965
2966	void pci_config_pm_runtime_put(struct pci_dev *pdev)
2967	{
2968	struct device *dev = &pdev->dev;
2969	struct device *parent = dev->parent;
2970
2971	pm_runtime_put(dev);
2972	if (parent)
2973	pm_runtime_put_sync(dev: parent);
2974	}
2975
2976	static const struct dmi_system_id bridge_d3_blacklist[] = {
2977	#ifdef CONFIG_X86
2978	{
2979	/*
2980	* Gigabyte X299 root port is not marked as hotplug capable
2981	* which allows Linux to power manage it. However, this
2982	* confuses the BIOS SMI handler so don't power manage root
2983	* ports on that system.
2984	*/
2985	.ident = "X299 DESIGNARE EX-CF",
2986	.matches = {
2987	DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."),
2988	DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"),
2989	},
2990	},
2991	{
2992	/*
2993	* Downstream device is not accessible after putting a root port
2994	* into D3cold and back into D0 on Elo Continental Z2 board
2995	*/
2996	.ident = "Elo Continental Z2",
2997	.matches = {
2998	DMI_MATCH(DMI_BOARD_VENDOR, "Elo Touch Solutions"),
2999	DMI_MATCH(DMI_BOARD_NAME, "Geminilake"),
3000	DMI_MATCH(DMI_BOARD_VERSION, "Continental Z2"),
3001	},
3002	},
3003	#endif
3004	{ }
3005	};
3006
3007	/**
3008	* pci_bridge_d3_possible - Is it possible to put the bridge into D3
3009	* @bridge: Bridge to check
3010	*
3011	* This function checks if it is possible to move the bridge to D3.
3012	* Currently we only allow D3 for recent enough PCIe ports and Thunderbolt.
3013	*/
3014	bool pci_bridge_d3_possible(struct pci_dev *bridge)
3015	{
3016	if (!pci_is_pcie(dev: bridge))
3017	return false;
3018
3019	switch (pci_pcie_type(dev: bridge)) {
3020	case PCI_EXP_TYPE_ROOT_PORT:
3021	case PCI_EXP_TYPE_UPSTREAM:
3022	case PCI_EXP_TYPE_DOWNSTREAM:
3023	if (pci_bridge_d3_disable)
3024	return false;
3025
3026	/*
3027	* Hotplug ports handled by firmware in System Management Mode
3028	* may not be put into D3 by the OS (Thunderbolt on non-Macs).
3029	*/
3030	if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge))
3031	return false;
3032
3033	if (pci_bridge_d3_force)
3034	return true;
3035
3036	/* Even the oldest 2010 Thunderbolt controller supports D3. */
3037	if (bridge->is_thunderbolt)
3038	return true;
3039
3040	/* Platform might know better if the bridge supports D3 */
3041	if (platform_pci_bridge_d3(dev: bridge))
3042	return true;
3043
3044	/*
3045	* Hotplug ports handled natively by the OS were not validated
3046	* by vendors for runtime D3 at least until 2018 because there
3047	* was no OS support.
3048	*/
3049	if (bridge->is_hotplug_bridge)
3050	return false;
3051
3052	if (dmi_check_system(list: bridge_d3_blacklist))
3053	return false;
3054
3055	/*
3056	* It should be safe to put PCIe ports from 2015 or newer
3057	* to D3.
3058	*/
3059	if (dmi_get_bios_year() >= 2015)
3060	return true;
3061	break;
3062	}
3063
3064	return false;
3065	}
3066
3067	static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
3068	{
3069	bool *d3cold_ok = data;
3070
3071	if (/* The device needs to be allowed to go D3cold ... */
3072	dev->no_d3cold || !dev->d3cold_allowed ||
3073
3074	/* ... and if it is wakeup capable to do so from D3cold. */
3075	(device_may_wakeup(dev: &dev->dev) &&
3076	!pci_pme_capable(dev, PCI_D3cold)) ||
3077
3078	/* If it is a bridge it must be allowed to go to D3. */
3079	!pci_power_manageable(pci_dev: dev))
3080
3081	*d3cold_ok = false;
3082
3083	return !*d3cold_ok;
3084	}
3085
3086	/*
3087	* pci_bridge_d3_update - Update bridge D3 capabilities
3088	* @dev: PCI device which is changed
3089	*
3090	* Update upstream bridge PM capabilities accordingly depending on if the
3091	* device PM configuration was changed or the device is being removed. The
3092	* change is also propagated upstream.
3093	*/
3094	void pci_bridge_d3_update(struct pci_dev *dev)
3095	{
3096	bool remove = !device_is_registered(dev: &dev->dev);
3097	struct pci_dev *bridge;
3098	bool d3cold_ok = true;
3099
3100	bridge = pci_upstream_bridge(dev);
3101	if (!bridge || !pci_bridge_d3_possible(bridge))
3102	return;
3103
3104	/*
3105	* If D3 is currently allowed for the bridge, removing one of its
3106	* children won't change that.
3107	*/
3108	if (remove && bridge->bridge_d3)
3109	return;
3110
3111	/*
3112	* If D3 is currently allowed for the bridge and a child is added or
3113	* changed, disallowance of D3 can only be caused by that child, so
3114	* we only need to check that single device, not any of its siblings.
3115	*
3116	* If D3 is currently not allowed for the bridge, checking the device
3117	* first may allow us to skip checking its siblings.
3118	*/
3119	if (!remove)
3120	pci_dev_check_d3cold(dev, data: &d3cold_ok);
3121
3122	/*
3123	* If D3 is currently not allowed for the bridge, this may be caused
3124	* either by the device being changed/removed or any of its siblings,
3125	* so we need to go through all children to find out if one of them
3126	* continues to block D3.
3127	*/
3128	if (d3cold_ok && !bridge->bridge_d3)
3129	pci_walk_bus(top: bridge->subordinate, cb: pci_dev_check_d3cold,
3130	userdata: &d3cold_ok);
3131
3132	if (bridge->bridge_d3 != d3cold_ok) {
3133	bridge->bridge_d3 = d3cold_ok;
3134	/* Propagate change to upstream bridges */
3135	pci_bridge_d3_update(dev: bridge);
3136	}
3137	}
3138
3139	/**
3140	* pci_d3cold_enable - Enable D3cold for device
3141	* @dev: PCI device to handle
3142	*
3143	* This function can be used in drivers to enable D3cold from the device
3144	* they handle. It also updates upstream PCI bridge PM capabilities
3145	* accordingly.
3146	*/
3147	void pci_d3cold_enable(struct pci_dev *dev)
3148	{
3149	if (dev->no_d3cold) {
3150	dev->no_d3cold = false;
3151	pci_bridge_d3_update(dev);
3152	}
3153	}
3154	EXPORT_SYMBOL_GPL(pci_d3cold_enable);
3155
3156	/**
3157	* pci_d3cold_disable - Disable D3cold for device
3158	* @dev: PCI device to handle
3159	*
3160	* This function can be used in drivers to disable D3cold from the device
3161	* they handle. It also updates upstream PCI bridge PM capabilities
3162	* accordingly.
3163	*/
3164	void pci_d3cold_disable(struct pci_dev *dev)
3165	{
3166	if (!dev->no_d3cold) {
3167	dev->no_d3cold = true;
3168	pci_bridge_d3_update(dev);
3169	}
3170	}
3171	EXPORT_SYMBOL_GPL(pci_d3cold_disable);
3172
3173	/**
3174	* pci_pm_init - Initialize PM functions of given PCI device
3175	* @dev: PCI device to handle.
3176	*/
3177	void pci_pm_init(struct pci_dev *dev)
3178	{
3179	int pm;
3180	u16 status;
3181	u16 pmc;
3182
3183	pm_runtime_forbid(dev: &dev->dev);
3184	pm_runtime_set_active(dev: &dev->dev);
3185	pm_runtime_enable(dev: &dev->dev);
3186	device_enable_async_suspend(dev: &dev->dev);
3187	dev->wakeup_prepared = false;
3188
3189	dev->pm_cap = 0;
3190	dev->pme_support = 0;
3191
3192	/* find PCI PM capability in list */
3193	pm = pci_find_capability(dev, PCI_CAP_ID_PM);
3194	if (!pm)
3195	return;
3196	/* Check device's ability to generate PME# */
3197	pci_read_config_word(dev, where: pm + PCI_PM_PMC, val: &pmc);
3198
3199	if ((pmc & PCI_PM_CAP_VER_MASK) > 3) {
3200	pci_err(dev, "unsupported PM cap regs version (%u)\n",
3201	pmc & PCI_PM_CAP_VER_MASK);
3202	return;
3203	}
3204
3205	dev->pm_cap = pm;
3206	dev->d3hot_delay = PCI_PM_D3HOT_WAIT;
3207	dev->d3cold_delay = PCI_PM_D3COLD_WAIT;
3208	dev->bridge_d3 = pci_bridge_d3_possible(bridge: dev);
3209	dev->d3cold_allowed = true;
3210
3211	dev->d1_support = false;
3212	dev->d2_support = false;
3213	if (!pci_no_d1d2(dev)) {
3214	if (pmc & PCI_PM_CAP_D1)
3215	dev->d1_support = true;
3216	if (pmc & PCI_PM_CAP_D2)
3217	dev->d2_support = true;
3218
3219	if (dev->d1_support || dev->d2_support)
3220	pci_info(dev, "supports%s%s\n",
3221	dev->d1_support ? " D1" : "",
3222	dev->d2_support ? " D2" : "");
3223	}
3224
3225	pmc &= PCI_PM_CAP_PME_MASK;
3226	if (pmc) {
3227	pci_info(dev, "PME# supported from%s%s%s%s%s\n",
3228	(pmc & PCI_PM_CAP_PME_D0) ? " D0" : "",
3229	(pmc & PCI_PM_CAP_PME_D1) ? " D1" : "",
3230	(pmc & PCI_PM_CAP_PME_D2) ? " D2" : "",
3231	(pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "",
3232	(pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : "");
3233	dev->pme_support = FIELD_GET(PCI_PM_CAP_PME_MASK, pmc);
3234	dev->pme_poll = true;
3235	/*
3236	* Make device's PM flags reflect the wake-up capability, but
3237	* let the user space enable it to wake up the system as needed.
3238	*/
3239	device_set_wakeup_capable(dev: &dev->dev, capable: true);
3240	/* Disable the PME# generation functionality */
3241	pci_pme_active(dev, false);
3242	}
3243
3244	pci_read_config_word(dev, PCI_STATUS, val: &status);
3245	if (status & PCI_STATUS_IMM_READY)
3246	dev->imm_ready = 1;
3247	}
3248
3249	static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop)
3250	{
3251	unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI;
3252
3253	switch (prop) {
3254	case PCI_EA_P_MEM:
3255	case PCI_EA_P_VF_MEM:
3256	flags |= IORESOURCE_MEM;
3257	break;
3258	case PCI_EA_P_MEM_PREFETCH:
3259	case PCI_EA_P_VF_MEM_PREFETCH:
3260	flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
3261	break;
3262	case PCI_EA_P_IO:
3263	flags |= IORESOURCE_IO;
3264	break;
3265	default:
3266	return 0;
3267	}
3268
3269	return flags;
3270	}
3271
3272	static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei,
3273	u8 prop)
3274	{
3275	if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO)
3276	return &dev->resource[bei];
3277	#ifdef CONFIG_PCI_IOV
3278	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 &&
3279	(prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH))
3280	return &dev->resource[PCI_IOV_RESOURCES +
3281	bei - PCI_EA_BEI_VF_BAR0];
3282	#endif
3283	else if (bei == PCI_EA_BEI_ROM)
3284	return &dev->resource[PCI_ROM_RESOURCE];
3285	else
3286	return NULL;
3287	}
3288
3289	/* Read an Enhanced Allocation (EA) entry */
3290	static int pci_ea_read(struct pci_dev *dev, int offset)
3291	{
3292	struct resource *res;
3293	int ent_size, ent_offset = offset;
3294	resource_size_t start, end;
3295	unsigned long flags;
3296	u32 dw0, bei, base, max_offset;
3297	u8 prop;
3298	bool support_64 = (sizeof(resource_size_t) >= 8);
3299
3300	pci_read_config_dword(dev, where: ent_offset, val: &dw0);
3301	ent_offset += 4;
3302
3303	/* Entry size field indicates DWORDs after 1st */
3304	ent_size = (FIELD_GET(PCI_EA_ES, dw0) + 1) << 2;
3305
3306	if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */
3307	goto out;
3308
3309	bei = FIELD_GET(PCI_EA_BEI, dw0);
3310	prop = FIELD_GET(PCI_EA_PP, dw0);
3311
3312	/*
3313	* If the Property is in the reserved range, try the Secondary
3314	* Property instead.
3315	*/
3316	if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED)
3317	prop = FIELD_GET(PCI_EA_SP, dw0);
3318	if (prop > PCI_EA_P_BRIDGE_IO)
3319	goto out;
3320
3321	res = pci_ea_get_resource(dev, bei, prop);
3322	if (!res) {
3323	pci_err(dev, "Unsupported EA entry BEI: %u\n", bei);
3324	goto out;
3325	}
3326
3327	flags = pci_ea_flags(dev, prop);
3328	if (!flags) {
3329	pci_err(dev, "Unsupported EA properties: %#x\n", prop);
3330	goto out;
3331	}
3332
3333	/* Read Base */
3334	pci_read_config_dword(dev, where: ent_offset, val: &base);
3335	start = (base & PCI_EA_FIELD_MASK);
3336	ent_offset += 4;
3337
3338	/* Read MaxOffset */
3339	pci_read_config_dword(dev, where: ent_offset, val: &max_offset);
3340	ent_offset += 4;
3341
3342	/* Read Base MSBs (if 64-bit entry) */
3343	if (base & PCI_EA_IS_64) {
3344	u32 base_upper;
3345
3346	pci_read_config_dword(dev, where: ent_offset, val: &base_upper);
3347	ent_offset += 4;
3348
3349	flags |= IORESOURCE_MEM_64;
3350
3351	/* entry starts above 32-bit boundary, can't use */
3352	if (!support_64 && base_upper)
3353	goto out;
3354
3355	if (support_64)
3356	start |= ((u64)base_upper << 32);
3357	}
3358
3359	end = start + (max_offset | 0x03);
3360
3361	/* Read MaxOffset MSBs (if 64-bit entry) */
3362	if (max_offset & PCI_EA_IS_64) {
3363	u32 max_offset_upper;
3364
3365	pci_read_config_dword(dev, where: ent_offset, val: &max_offset_upper);
3366	ent_offset += 4;
3367
3368	flags |= IORESOURCE_MEM_64;
3369
3370	/* entry too big, can't use */
3371	if (!support_64 && max_offset_upper)
3372	goto out;
3373
3374	if (support_64)
3375	end += ((u64)max_offset_upper << 32);
3376	}
3377
3378	if (end < start) {
3379	pci_err(dev, "EA Entry crosses address boundary\n");
3380	goto out;
3381	}
3382
3383	if (ent_size != ent_offset - offset) {
3384	pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n",
3385	ent_size, ent_offset - offset);
3386	goto out;
3387	}
3388
3389	res->name = pci_name(pdev: dev);
3390	res->start = start;
3391	res->end = end;
3392	res->flags = flags;
3393
3394	if (bei <= PCI_EA_BEI_BAR5)
3395	pci_info(dev, "BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n",
3396	bei, res, prop);
3397	else if (bei == PCI_EA_BEI_ROM)
3398	pci_info(dev, "ROM: %pR (from Enhanced Allocation, properties %#02x)\n",
3399	res, prop);
3400	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5)
3401	pci_info(dev, "VF BAR %d: %pR (from Enhanced Allocation, properties %#02x)\n",
3402	bei - PCI_EA_BEI_VF_BAR0, res, prop);
3403	else
3404	pci_info(dev, "BEI %d res: %pR (from Enhanced Allocation, properties %#02x)\n",
3405	bei, res, prop);
3406
3407	out:
3408	return offset + ent_size;
3409	}
3410
3411	/* Enhanced Allocation Initialization */
3412	void pci_ea_init(struct pci_dev *dev)
3413	{
3414	int ea;
3415	u8 num_ent;
3416	int offset;
3417	int i;
3418
3419	/* find PCI EA capability in list */
3420	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
3421	if (!ea)
3422	return;
3423
3424	/* determine the number of entries */
3425	pci_bus_read_config_byte(bus: dev->bus, devfn: dev->devfn, where: ea + PCI_EA_NUM_ENT,
3426	val: &num_ent);
3427	num_ent &= PCI_EA_NUM_ENT_MASK;
3428
3429	offset = ea + PCI_EA_FIRST_ENT;
3430
3431	/* Skip DWORD 2 for type 1 functions */
3432	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
3433	offset += 4;
3434
3435	/* parse each EA entry */
3436	for (i = 0; i < num_ent; ++i)
3437	offset = pci_ea_read(dev, offset);
3438	}
3439
3440	static void pci_add_saved_cap(struct pci_dev *pci_dev,
3441	struct pci_cap_saved_state *new_cap)
3442	{
3443	hlist_add_head(n: &new_cap->next, h: &pci_dev->saved_cap_space);
3444	}
3445
3446	/**
3447	* _pci_add_cap_save_buffer - allocate buffer for saving given
3448	* capability registers
3449	* @dev: the PCI device
3450	* @cap: the capability to allocate the buffer for
3451	* @extended: Standard or Extended capability ID
3452	* @size: requested size of the buffer
3453	*/
3454	static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap,
3455	bool extended, unsigned int size)
3456	{
3457	int pos;
3458	struct pci_cap_saved_state *save_state;
3459
3460	if (extended)
3461	pos = pci_find_ext_capability(dev, cap);
3462	else
3463	pos = pci_find_capability(dev, cap);
3464
3465	if (!pos)
3466	return 0;
3467
3468	save_state = kzalloc(size: sizeof(*save_state) + size, GFP_KERNEL);
3469	if (!save_state)
3470	return -ENOMEM;
3471
3472	save_state->cap.cap_nr = cap;
3473	save_state->cap.cap_extended = extended;
3474	save_state->cap.size = size;
3475	pci_add_saved_cap(pci_dev: dev, new_cap: save_state);
3476
3477	return 0;
3478	}
3479
3480	int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size)
3481	{
3482	return _pci_add_cap_save_buffer(dev, cap, extended: false, size);
3483	}
3484
3485	int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size)
3486	{
3487	return _pci_add_cap_save_buffer(dev, cap, extended: true, size);
3488	}
3489
3490	/**
3491	* pci_allocate_cap_save_buffers - allocate buffers for saving capabilities
3492	* @dev: the PCI device
3493	*/
3494	void pci_allocate_cap_save_buffers(struct pci_dev *dev)
3495	{
3496	int error;
3497
3498	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP,
3499	PCI_EXP_SAVE_REGS * sizeof(u16));
3500	if (error)
3501	pci_err(dev, "unable to preallocate PCI Express save buffer\n");
3502
3503	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, size: sizeof(u16));
3504	if (error)
3505	pci_err(dev, "unable to preallocate PCI-X save buffer\n");
3506
3507	error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_LTR,
3508	size: 2 * sizeof(u16));
3509	if (error)
3510	pci_err(dev, "unable to allocate suspend buffer for LTR\n");
3511
3512	pci_allocate_vc_save_buffers(dev);
3513	}
3514
3515	void pci_free_cap_save_buffers(struct pci_dev *dev)
3516	{
3517	struct pci_cap_saved_state *tmp;
3518	struct hlist_node *n;
3519
3520	hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next)
3521	kfree(objp: tmp);
3522	}
3523
3524	/**
3525	* pci_configure_ari - enable or disable ARI forwarding
3526	* @dev: the PCI device
3527	*
3528	* If @dev and its upstream bridge both support ARI, enable ARI in the
3529	* bridge. Otherwise, disable ARI in the bridge.
3530	*/
3531	void pci_configure_ari(struct pci_dev *dev)
3532	{
3533	u32 cap;
3534	struct pci_dev *bridge;
3535
3536	if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn)
3537	return;
3538
3539	bridge = dev->bus->self;
3540	if (!bridge)
3541	return;
3542
3543	pcie_capability_read_dword(dev: bridge, PCI_EXP_DEVCAP2, val: &cap);
3544	if (!(cap & PCI_EXP_DEVCAP2_ARI))
3545	return;
3546
3547	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) {
3548	pcie_capability_set_word(dev: bridge, PCI_EXP_DEVCTL2,
3549	PCI_EXP_DEVCTL2_ARI);
3550	bridge->ari_enabled = 1;
3551	} else {
3552	pcie_capability_clear_word(dev: bridge, PCI_EXP_DEVCTL2,
3553	PCI_EXP_DEVCTL2_ARI);
3554	bridge->ari_enabled = 0;
3555	}
3556	}
3557
3558	static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags)
3559	{
3560	int pos;
3561	u16 cap, ctrl;
3562
3563	pos = pdev->acs_cap;
3564	if (!pos)
3565	return false;
3566
3567	/*
3568	* Except for egress control, capabilities are either required
3569	* or only required if controllable. Features missing from the
3570	* capability field can therefore be assumed as hard-wired enabled.
3571	*/
3572	pci_read_config_word(dev: pdev, where: pos + PCI_ACS_CAP, val: &cap);
3573	acs_flags &= (cap | PCI_ACS_EC);
3574
3575	pci_read_config_word(dev: pdev, where: pos + PCI_ACS_CTRL, val: &ctrl);
3576	return (ctrl & acs_flags) == acs_flags;
3577	}
3578
3579	/**
3580	* pci_acs_enabled - test ACS against required flags for a given device
3581	* @pdev: device to test
3582	* @acs_flags: required PCI ACS flags
3583	*
3584	* Return true if the device supports the provided flags. Automatically
3585	* filters out flags that are not implemented on multifunction devices.
3586	*
3587	* Note that this interface checks the effective ACS capabilities of the
3588	* device rather than the actual capabilities. For instance, most single
3589	* function endpoints are not required to support ACS because they have no
3590	* opportunity for peer-to-peer access. We therefore return 'true'
3591	* regardless of whether the device exposes an ACS capability. This makes
3592	* it much easier for callers of this function to ignore the actual type
3593	* or topology of the device when testing ACS support.
3594	*/
3595	bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags)
3596	{
3597	int ret;
3598
3599	ret = pci_dev_specific_acs_enabled(dev: pdev, acs_flags);
3600	if (ret >= 0)
3601	return ret > 0;
3602
3603	/*
3604	* Conventional PCI and PCI-X devices never support ACS, either
3605	* effectively or actually. The shared bus topology implies that
3606	* any device on the bus can receive or snoop DMA.
3607	*/
3608	if (!pci_is_pcie(dev: pdev))
3609	return false;
3610
3611	switch (pci_pcie_type(dev: pdev)) {
3612	/*
3613	* PCI/X-to-PCIe bridges are not specifically mentioned by the spec,
3614	* but since their primary interface is PCI/X, we conservatively
3615	* handle them as we would a non-PCIe device.
3616	*/
3617	case PCI_EXP_TYPE_PCIE_BRIDGE:
3618	/*
3619	* PCIe 3.0, 6.12.1 excludes ACS on these devices. "ACS is never
3620	* applicable... must never implement an ACS Extended Capability...".
3621	* This seems arbitrary, but we take a conservative interpretation
3622	* of this statement.
3623	*/
3624	case PCI_EXP_TYPE_PCI_BRIDGE:
3625	case PCI_EXP_TYPE_RC_EC:
3626	return false;
3627	/*
3628	* PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should
3629	* implement ACS in order to indicate their peer-to-peer capabilities,
3630	* regardless of whether they are single- or multi-function devices.
3631	*/
3632	case PCI_EXP_TYPE_DOWNSTREAM:
3633	case PCI_EXP_TYPE_ROOT_PORT:
3634	return pci_acs_flags_enabled(pdev, acs_flags);
3635	/*
3636	* PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be
3637	* implemented by the remaining PCIe types to indicate peer-to-peer
3638	* capabilities, but only when they are part of a multifunction
3639	* device. The footnote for section 6.12 indicates the specific
3640	* PCIe types included here.
3641	*/
3642	case PCI_EXP_TYPE_ENDPOINT:
3643	case PCI_EXP_TYPE_UPSTREAM:
3644	case PCI_EXP_TYPE_LEG_END:
3645	case PCI_EXP_TYPE_RC_END:
3646	if (!pdev->multifunction)
3647	break;
3648
3649	return pci_acs_flags_enabled(pdev, acs_flags);
3650	}
3651
3652	/*
3653	* PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable
3654	* to single function devices with the exception of downstream ports.
3655	*/
3656	return true;
3657	}
3658
3659	/**
3660	* pci_acs_path_enabled - test ACS flags from start to end in a hierarchy
3661	* @start: starting downstream device
3662	* @end: ending upstream device or NULL to search to the root bus
3663	* @acs_flags: required flags
3664	*
3665	* Walk up a device tree from start to end testing PCI ACS support. If
3666	* any step along the way does not support the required flags, return false.
3667	*/
3668	bool pci_acs_path_enabled(struct pci_dev *start,
3669	struct pci_dev *end, u16 acs_flags)
3670	{
3671	struct pci_dev *pdev, *parent = start;
3672
3673	do {
3674	pdev = parent;
3675
3676	if (!pci_acs_enabled(pdev, acs_flags))
3677	return false;
3678
3679	if (pci_is_root_bus(pbus: pdev->bus))
3680	return (end == NULL);
3681
3682	parent = pdev->bus->self;
3683	} while (pdev != end);
3684
3685	return true;
3686	}
3687
3688	/**
3689	* pci_acs_init - Initialize ACS if hardware supports it
3690	* @dev: the PCI device
3691	*/
3692	void pci_acs_init(struct pci_dev *dev)
3693	{
3694	dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
3695
3696	/*
3697	* Attempt to enable ACS regardless of capability because some Root
3698	* Ports (e.g. those quirked with *_intel_pch_acs_*) do not have
3699	* the standard ACS capability but still support ACS via those
3700	* quirks.
3701	*/
3702	pci_enable_acs(dev);
3703	}
3704
3705	/**
3706	* pci_rebar_find_pos - find position of resize ctrl reg for BAR
3707	* @pdev: PCI device
3708	* @bar: BAR to find
3709	*
3710	* Helper to find the position of the ctrl register for a BAR.
3711	* Returns -ENOTSUPP if resizable BARs are not supported at all.
3712	* Returns -ENOENT if no ctrl register for the BAR could be found.
3713	*/
3714	static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
3715	{
3716	unsigned int pos, nbars, i;
3717	u32 ctrl;
3718
3719	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
3720	if (!pos)
3721	return -ENOTSUPP;
3722
3723	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
3724	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
3725
3726	for (i = 0; i < nbars; i++, pos += 8) {
3727	int bar_idx;
3728
3729	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
3730	bar_idx = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, ctrl);
3731	if (bar_idx == bar)
3732	return pos;
3733	}
3734
3735	return -ENOENT;
3736	}
3737
3738	/**
3739	* pci_rebar_get_possible_sizes - get possible sizes for BAR
3740	* @pdev: PCI device
3741	* @bar: BAR to query
3742	*
3743	* Get the possible sizes of a resizable BAR as bitmask defined in the spec
3744	* (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable.
3745	*/
3746	u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar)
3747	{
3748	int pos;
3749	u32 cap;
3750
3751	pos = pci_rebar_find_pos(pdev, bar);
3752	if (pos < 0)
3753	return 0;
3754
3755	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CAP, val: &cap);
3756	cap = FIELD_GET(PCI_REBAR_CAP_SIZES, cap);
3757
3758	/* Sapphire RX 5600 XT Pulse has an invalid cap dword for BAR 0 */
3759	if (pdev->vendor == PCI_VENDOR_ID_ATI && pdev->device == 0x731f &&
3760	bar == 0 && cap == 0x700)
3761	return 0x3f00;
3762
3763	return cap;
3764	}
3765	EXPORT_SYMBOL(pci_rebar_get_possible_sizes);
3766
3767	/**
3768	* pci_rebar_get_current_size - get the current size of a BAR
3769	* @pdev: PCI device
3770	* @bar: BAR to set size to
3771	*
3772	* Read the size of a BAR from the resizable BAR config.
3773	* Returns size if found or negative error code.
3774	*/
3775	int pci_rebar_get_current_size(struct pci_dev *pdev, int bar)
3776	{
3777	int pos;
3778	u32 ctrl;
3779
3780	pos = pci_rebar_find_pos(pdev, bar);
3781	if (pos < 0)
3782	return pos;
3783
3784	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
3785	return FIELD_GET(PCI_REBAR_CTRL_BAR_SIZE, ctrl);
3786	}
3787
3788	/**
3789	* pci_rebar_set_size - set a new size for a BAR
3790	* @pdev: PCI device
3791	* @bar: BAR to set size to
3792	* @size: new size as defined in the spec (0=1MB, 19=512GB)
3793	*
3794	* Set the new size of a BAR as defined in the spec.
3795	* Returns zero if resizing was successful, error code otherwise.
3796	*/
3797	int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size)
3798	{
3799	int pos;
3800	u32 ctrl;
3801
3802	pos = pci_rebar_find_pos(pdev, bar);
3803	if (pos < 0)
3804	return pos;
3805
3806	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
3807	ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
3808	ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
3809	pci_write_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: ctrl);
3810	return 0;
3811	}
3812
3813	/**
3814	* pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port
3815	* @dev: the PCI device
3816	* @cap_mask: mask of desired AtomicOp sizes, including one or more of:
3817	* PCI_EXP_DEVCAP2_ATOMIC_COMP32
3818	* PCI_EXP_DEVCAP2_ATOMIC_COMP64
3819	* PCI_EXP_DEVCAP2_ATOMIC_COMP128
3820	*
3821	* Return 0 if all upstream bridges support AtomicOp routing, egress
3822	* blocking is disabled on all upstream ports, and the root port supports
3823	* the requested completion capabilities (32-bit, 64-bit and/or 128-bit
3824	* AtomicOp completion), or negative otherwise.
3825	*/
3826	int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
3827	{
3828	struct pci_bus *bus = dev->bus;
3829	struct pci_dev *bridge;
3830	u32 cap, ctl2;
3831
3832	/*
3833	* Per PCIe r5.0, sec 9.3.5.10, the AtomicOp Requester Enable bit
3834	* in Device Control 2 is reserved in VFs and the PF value applies
3835	* to all associated VFs.
3836	*/
3837	if (dev->is_virtfn)
3838	return -EINVAL;
3839
3840	if (!pci_is_pcie(dev))
3841	return -EINVAL;
3842
3843	/*
3844	* Per PCIe r4.0, sec 6.15, endpoints and root ports may be
3845	* AtomicOp requesters. For now, we only support endpoints as
3846	* requesters and root ports as completers. No endpoints as
3847	* completers, and no peer-to-peer.
3848	*/
3849
3850	switch (pci_pcie_type(dev)) {
3851	case PCI_EXP_TYPE_ENDPOINT:
3852	case PCI_EXP_TYPE_LEG_END:
3853	case PCI_EXP_TYPE_RC_END:
3854	break;
3855	default:
3856	return -EINVAL;
3857	}
3858
3859	while (bus->parent) {
3860	bridge = bus->self;
3861
3862	pcie_capability_read_dword(dev: bridge, PCI_EXP_DEVCAP2, val: &cap);
3863
3864	switch (pci_pcie_type(dev: bridge)) {
3865	/* Ensure switch ports support AtomicOp routing */
3866	case PCI_EXP_TYPE_UPSTREAM:
3867	case PCI_EXP_TYPE_DOWNSTREAM:
3868	if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
3869	return -EINVAL;
3870	break;
3871
3872	/* Ensure root port supports all the sizes we care about */
3873	case PCI_EXP_TYPE_ROOT_PORT:
3874	if ((cap & cap_mask) != cap_mask)
3875	return -EINVAL;
3876	break;
3877	}
3878
3879	/* Ensure upstream ports don't block AtomicOps on egress */
3880	if (pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_UPSTREAM) {
3881	pcie_capability_read_dword(dev: bridge, PCI_EXP_DEVCTL2,
3882	val: &ctl2);
3883	if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK)
3884	return -EINVAL;
3885	}
3886
3887	bus = bus->parent;
3888	}
3889
3890	pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
3891	PCI_EXP_DEVCTL2_ATOMIC_REQ);
3892	return 0;
3893	}
3894	EXPORT_SYMBOL(pci_enable_atomic_ops_to_root);
3895
3896	/**
3897	* pci_swizzle_interrupt_pin - swizzle INTx for device behind bridge
3898	* @dev: the PCI device
3899	* @pin: the INTx pin (1=INTA, 2=INTB, 3=INTC, 4=INTD)
3900	*
3901	* Perform INTx swizzling for a device behind one level of bridge. This is
3902	* required by section 9.1 of the PCI-to-PCI bridge specification for devices
3903	* behind bridges on add-in cards. For devices with ARI enabled, the slot
3904	* number is always 0 (see the Implementation Note in section 2.2.8.1 of
3905	* the PCI Express Base Specification, Revision 2.1)
3906	*/
3907	u8 pci_swizzle_interrupt_pin(const struct pci_dev *dev, u8 pin)
3908	{
3909	int slot;
3910
3911	if (pci_ari_enabled(bus: dev->bus))
3912	slot = 0;
3913	else
3914	slot = PCI_SLOT(dev->devfn);
3915
3916	return (((pin - 1) + slot) % 4) + 1;
3917	}
3918
3919	int pci_get_interrupt_pin(struct pci_dev *dev, struct pci_dev **bridge)
3920	{
3921	u8 pin;
3922
3923	pin = dev->pin;
3924	if (!pin)
3925	return -1;
3926
3927	while (!pci_is_root_bus(pbus: dev->bus)) {
3928	pin = pci_swizzle_interrupt_pin(dev, pin);
3929	dev = dev->bus->self;
3930	}
3931	*bridge = dev;
3932	return pin;
3933	}
3934
3935	/**
3936	* pci_common_swizzle - swizzle INTx all the way to root bridge
3937	* @dev: the PCI device
3938	* @pinp: pointer to the INTx pin value (1=INTA, 2=INTB, 3=INTD, 4=INTD)
3939	*
3940	* Perform INTx swizzling for a device. This traverses through all PCI-to-PCI
3941	* bridges all the way up to a PCI root bus.
3942	*/
3943	u8 pci_common_swizzle(struct pci_dev *dev, u8 *pinp)
3944	{
3945	u8 pin = *pinp;
3946
3947	while (!pci_is_root_bus(pbus: dev->bus)) {
3948	pin = pci_swizzle_interrupt_pin(dev, pin);
3949	dev = dev->bus->self;
3950	}
3951	*pinp = pin;
3952	return PCI_SLOT(dev->devfn);
3953	}
3954	EXPORT_SYMBOL_GPL(pci_common_swizzle);
3955
3956	/**
3957	* pci_release_region - Release a PCI bar
3958	* @pdev: PCI device whose resources were previously reserved by
3959	* pci_request_region()
3960	* @bar: BAR to release
3961	*
3962	* Releases the PCI I/O and memory resources previously reserved by a
3963	* successful call to pci_request_region(). Call this function only
3964	* after all use of the PCI regions has ceased.
3965	*/
3966	void pci_release_region(struct pci_dev *pdev, int bar)
3967	{
3968	struct pci_devres *dr;
3969
3970	if (pci_resource_len(pdev, bar) == 0)
3971	return;
3972	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO)
3973	release_region(pci_resource_start(pdev, bar),
3974	pci_resource_len(pdev, bar));
3975	else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM)
3976	release_mem_region(pci_resource_start(pdev, bar),
3977	pci_resource_len(pdev, bar));
3978
3979	dr = find_pci_dr(pdev);
3980	if (dr)
3981	dr->region_mask &= ~(1 << bar);
3982	}
3983	EXPORT_SYMBOL(pci_release_region);
3984
3985	/**
3986	* __pci_request_region - Reserved PCI I/O and memory resource
3987	* @pdev: PCI device whose resources are to be reserved
3988	* @bar: BAR to be reserved
3989	* @res_name: Name to be associated with resource.
3990	* @exclusive: whether the region access is exclusive or not
3991	*
3992	* Mark the PCI region associated with PCI device @pdev BAR @bar as
3993	* being reserved by owner @res_name. Do not access any
3994	* address inside the PCI regions unless this call returns
3995	* successfully.
3996	*
3997	* If @exclusive is set, then the region is marked so that userspace
3998	* is explicitly not allowed to map the resource via /dev/mem or
3999	* sysfs MMIO access.
4000	*
4001	* Returns 0 on success, or %EBUSY on error. A warning
4002	* message is also printed on failure.
4003	*/
4004	static int __pci_request_region(struct pci_dev *pdev, int bar,
4005	const char *res_name, int exclusive)
4006	{
4007	struct pci_devres *dr;
4008
4009	if (pci_resource_len(pdev, bar) == 0)
4010	return 0;
4011
4012	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) {
4013	if (!request_region(pci_resource_start(pdev, bar),
4014	pci_resource_len(pdev, bar), res_name))
4015	goto err_out;
4016	} else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
4017	if (!__request_mem_region(pci_resource_start(pdev, bar),
4018	pci_resource_len(pdev, bar), res_name,
4019	exclusive))
4020	goto err_out;
4021	}
4022
4023	dr = find_pci_dr(pdev);
4024	if (dr)
4025	dr->region_mask |= 1 << bar;
4026
4027	return 0;
4028
4029	err_out:
4030	pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar,
4031	&pdev->resource[bar]);
4032	return -EBUSY;
4033	}
4034
4035	/**
4036	* pci_request_region - Reserve PCI I/O and memory resource
4037	* @pdev: PCI device whose resources are to be reserved
4038	* @bar: BAR to be reserved
4039	* @res_name: Name to be associated with resource
4040	*
4041	* Mark the PCI region associated with PCI device @pdev BAR @bar as
4042	* being reserved by owner @res_name. Do not access any
4043	* address inside the PCI regions unless this call returns
4044	* successfully.
4045	*
4046	* Returns 0 on success, or %EBUSY on error. A warning
4047	* message is also printed on failure.
4048	*/
4049	int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name)
4050	{
4051	return __pci_request_region(pdev, bar, res_name, exclusive: 0);
4052	}
4053	EXPORT_SYMBOL(pci_request_region);
4054
4055	/**
4056	* pci_release_selected_regions - Release selected PCI I/O and memory resources
4057	* @pdev: PCI device whose resources were previously reserved
4058	* @bars: Bitmask of BARs to be released
4059	*
4060	* Release selected PCI I/O and memory resources previously reserved.
4061	* Call this function only after all use of the PCI regions has ceased.
4062	*/
4063	void pci_release_selected_regions(struct pci_dev *pdev, int bars)
4064	{
4065	int i;
4066
4067	for (i = 0; i < PCI_STD_NUM_BARS; i++)
4068	if (bars & (1 << i))
4069	pci_release_region(pdev, i);
4070	}
4071	EXPORT_SYMBOL(pci_release_selected_regions);
4072
4073	static int __pci_request_selected_regions(struct pci_dev *pdev, int bars,
4074	const char *res_name, int excl)
4075	{
4076	int i;
4077
4078	for (i = 0; i < PCI_STD_NUM_BARS; i++)
4079	if (bars & (1 << i))
4080	if (__pci_request_region(pdev, bar: i, res_name, exclusive: excl))
4081	goto err_out;
4082	return 0;
4083
4084	err_out:
4085	while (--i >= 0)
4086	if (bars & (1 << i))
4087	pci_release_region(pdev, i);
4088
4089	return -EBUSY;
4090	}
4091
4092
4093	/**
4094	* pci_request_selected_regions - Reserve selected PCI I/O and memory resources
4095	* @pdev: PCI device whose resources are to be reserved
4096	* @bars: Bitmask of BARs to be requested
4097	* @res_name: Name to be associated with resource
4098	*/
4099	int pci_request_selected_regions(struct pci_dev *pdev, int bars,
4100	const char *res_name)
4101	{
4102	return __pci_request_selected_regions(pdev, bars, res_name, excl: 0);
4103	}
4104	EXPORT_SYMBOL(pci_request_selected_regions);
4105
4106	int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars,
4107	const char *res_name)
4108	{
4109	return __pci_request_selected_regions(pdev, bars, res_name,
4110	IORESOURCE_EXCLUSIVE);
4111	}
4112	EXPORT_SYMBOL(pci_request_selected_regions_exclusive);
4113
4114	/**
4115	* pci_release_regions - Release reserved PCI I/O and memory resources
4116	* @pdev: PCI device whose resources were previously reserved by
4117	* pci_request_regions()
4118	*
4119	* Releases all PCI I/O and memory resources previously reserved by a
4120	* successful call to pci_request_regions(). Call this function only
4121	* after all use of the PCI regions has ceased.
4122	*/
4123
4124	void pci_release_regions(struct pci_dev *pdev)
4125	{
4126	pci_release_selected_regions(pdev, (1 << PCI_STD_NUM_BARS) - 1);
4127	}
4128	EXPORT_SYMBOL(pci_release_regions);
4129
4130	/**
4131	* pci_request_regions - Reserve PCI I/O and memory resources
4132	* @pdev: PCI device whose resources are to be reserved
4133	* @res_name: Name to be associated with resource.
4134	*
4135	* Mark all PCI regions associated with PCI device @pdev as
4136	* being reserved by owner @res_name. Do not access any
4137	* address inside the PCI regions unless this call returns
4138	* successfully.
4139	*
4140	* Returns 0 on success, or %EBUSY on error. A warning
4141	* message is also printed on failure.
4142	*/
4143	int pci_request_regions(struct pci_dev *pdev, const char *res_name)
4144	{
4145	return pci_request_selected_regions(pdev,
4146	((1 << PCI_STD_NUM_BARS) - 1), res_name);
4147	}
4148	EXPORT_SYMBOL(pci_request_regions);
4149
4150	/**
4151	* pci_request_regions_exclusive - Reserve PCI I/O and memory resources
4152	* @pdev: PCI device whose resources are to be reserved
4153	* @res_name: Name to be associated with resource.
4154	*
4155	* Mark all PCI regions associated with PCI device @pdev as being reserved
4156	* by owner @res_name. Do not access any address inside the PCI regions
4157	* unless this call returns successfully.
4158	*
4159	* pci_request_regions_exclusive() will mark the region so that /dev/mem
4160	* and the sysfs MMIO access will not be allowed.
4161	*
4162	* Returns 0 on success, or %EBUSY on error. A warning message is also
4163	* printed on failure.
4164	*/
4165	int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name)
4166	{
4167	return pci_request_selected_regions_exclusive(pdev,
4168	((1 << PCI_STD_NUM_BARS) - 1), res_name);
4169	}
4170	EXPORT_SYMBOL(pci_request_regions_exclusive);
4171
4172	/*
4173	* Record the PCI IO range (expressed as CPU physical address + size).
4174	* Return a negative value if an error has occurred, zero otherwise
4175	*/
4176	int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr,
4177	resource_size_t size)
4178	{
4179	int ret = 0;
4180	#ifdef PCI_IOBASE
4181	struct logic_pio_hwaddr *range;
4182
4183	if (!size || addr + size < addr)
4184	return -EINVAL;
4185
4186	range = kzalloc(sizeof(*range), GFP_ATOMIC);
4187	if (!range)
4188	return -ENOMEM;
4189
4190	range->fwnode = fwnode;
4191	range->size = size;
4192	range->hw_start = addr;
4193	range->flags = LOGIC_PIO_CPU_MMIO;
4194
4195	ret = logic_pio_register_range(range);
4196	if (ret)
4197	kfree(range);
4198
4199	/* Ignore duplicates due to deferred probing */
4200	if (ret == -EEXIST)
4201	ret = 0;
4202	#endif
4203
4204	return ret;
4205	}
4206
4207	phys_addr_t pci_pio_to_address(unsigned long pio)
4208	{
4209	#ifdef PCI_IOBASE
4210	if (pio < MMIO_UPPER_LIMIT)
4211	return logic_pio_to_hwaddr(pio);
4212	#endif
4213
4214	return (phys_addr_t) OF_BAD_ADDR;
4215	}
4216	EXPORT_SYMBOL_GPL(pci_pio_to_address);
4217
4218	unsigned long __weak pci_address_to_pio(phys_addr_t address)
4219	{
4220	#ifdef PCI_IOBASE
4221	return logic_pio_trans_cpuaddr(address);
4222	#else
4223	if (address > IO_SPACE_LIMIT)
4224	return (unsigned long)-1;
4225
4226	return (unsigned long) address;
4227	#endif
4228	}
4229
4230	/**
4231	* pci_remap_iospace - Remap the memory mapped I/O space
4232	* @res: Resource describing the I/O space
4233	* @phys_addr: physical address of range to be mapped
4234	*
4235	* Remap the memory mapped I/O space described by the @res and the CPU
4236	* physical address @phys_addr into virtual address space. Only
4237	* architectures that have memory mapped IO functions defined (and the
4238	* PCI_IOBASE value defined) should call this function.
4239	*/
4240	#ifndef pci_remap_iospace
4241	int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr)
4242	{
4243	#if defined(PCI_IOBASE) && defined(CONFIG_MMU)
4244	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
4245
4246	if (!(res->flags & IORESOURCE_IO))
4247	return -EINVAL;
4248
4249	if (res->end > IO_SPACE_LIMIT)
4250	return -EINVAL;
4251
4252	return ioremap_page_range(vaddr, vaddr + resource_size(res), phys_addr,
4253	pgprot_device(PAGE_KERNEL));
4254	#else
4255	/*
4256	* This architecture does not have memory mapped I/O space,
4257	* so this function should never be called
4258	*/
4259	WARN_ONCE(1, "This architecture does not support memory mapped I/O\n");
4260	return -ENODEV;
4261	#endif
4262	}
4263	EXPORT_SYMBOL(pci_remap_iospace);
4264	#endif
4265
4266	/**
4267	* pci_unmap_iospace - Unmap the memory mapped I/O space
4268	* @res: resource to be unmapped
4269	*
4270	* Unmap the CPU virtual address @res from virtual address space. Only
4271	* architectures that have memory mapped IO functions defined (and the
4272	* PCI_IOBASE value defined) should call this function.
4273	*/
4274	void pci_unmap_iospace(struct resource *res)
4275	{
4276	#if defined(PCI_IOBASE) && defined(CONFIG_MMU)
4277	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
4278
4279	vunmap_range(vaddr, vaddr + resource_size(res));
4280	#endif
4281	}
4282	EXPORT_SYMBOL(pci_unmap_iospace);
4283
4284	static void devm_pci_unmap_iospace(struct device *dev, void *ptr)
4285	{
4286	struct resource **res = ptr;
4287
4288	pci_unmap_iospace(*res);
4289	}
4290
4291	/**
4292	* devm_pci_remap_iospace - Managed pci_remap_iospace()
4293	* @dev: Generic device to remap IO address for
4294	* @res: Resource describing the I/O space
4295	* @phys_addr: physical address of range to be mapped
4296	*
4297	* Managed pci_remap_iospace(). Map is automatically unmapped on driver
4298	* detach.
4299	*/
4300	int devm_pci_remap_iospace(struct device *dev, const struct resource *res,
4301	phys_addr_t phys_addr)
4302	{
4303	const struct resource **ptr;
4304	int error;
4305
4306	ptr = devres_alloc(devm_pci_unmap_iospace, sizeof(*ptr), GFP_KERNEL);
4307	if (!ptr)
4308	return -ENOMEM;
4309
4310	error = pci_remap_iospace(res, phys_addr);
4311	if (error) {
4312	devres_free(res: ptr);
4313	} else {
4314	*ptr = res;
4315	devres_add(dev, res: ptr);
4316	}
4317
4318	return error;
4319	}
4320	EXPORT_SYMBOL(devm_pci_remap_iospace);
4321
4322	/**
4323	* devm_pci_remap_cfgspace - Managed pci_remap_cfgspace()
4324	* @dev: Generic device to remap IO address for
4325	* @offset: Resource address to map
4326	* @size: Size of map
4327	*
4328	* Managed pci_remap_cfgspace(). Map is automatically unmapped on driver
4329	* detach.
4330	*/
4331	void __iomem *devm_pci_remap_cfgspace(struct device *dev,
4332	resource_size_t offset,
4333	resource_size_t size)
4334	{
4335	void __iomem **ptr, *addr;
4336
4337	ptr = devres_alloc(devm_ioremap_release, sizeof(*ptr), GFP_KERNEL);
4338	if (!ptr)
4339	return NULL;
4340
4341	addr = pci_remap_cfgspace(offset, size);
4342	if (addr) {
4343	*ptr = addr;
4344	devres_add(dev, res: ptr);
4345	} else
4346	devres_free(res: ptr);
4347
4348	return addr;
4349	}
4350	EXPORT_SYMBOL(devm_pci_remap_cfgspace);
4351
4352	/**
4353	* devm_pci_remap_cfg_resource - check, request region and ioremap cfg resource
4354	* @dev: generic device to handle the resource for
4355	* @res: configuration space resource to be handled
4356	*
4357	* Checks that a resource is a valid memory region, requests the memory
4358	* region and ioremaps with pci_remap_cfgspace() API that ensures the
4359	* proper PCI configuration space memory attributes are guaranteed.
4360	*
4361	* All operations are managed and will be undone on driver detach.
4362	*
4363	* Returns a pointer to the remapped memory or an ERR_PTR() encoded error code
4364	* on failure. Usage example::
4365	*
4366	* res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4367	* base = devm_pci_remap_cfg_resource(&pdev->dev, res);
4368	* if (IS_ERR(base))
4369	* return PTR_ERR(base);
4370	*/
4371	void __iomem *devm_pci_remap_cfg_resource(struct device *dev,
4372	struct resource *res)
4373	{
4374	resource_size_t size;
4375	const char *name;
4376	void __iomem *dest_ptr;
4377
4378	BUG_ON(!dev);
4379
4380	if (!res || resource_type(res) != IORESOURCE_MEM) {
4381	dev_err(dev, "invalid resource\n");
4382	return IOMEM_ERR_PTR(-EINVAL);
4383	}
4384
4385	size = resource_size(res);
4386
4387	if (res->name)
4388	name = devm_kasprintf(dev, GFP_KERNEL, fmt: "%s %s", dev_name(dev),
4389	res->name);
4390	else
4391	name = devm_kstrdup(dev, s: dev_name(dev), GFP_KERNEL);
4392	if (!name)
4393	return IOMEM_ERR_PTR(-ENOMEM);
4394
4395	if (!devm_request_mem_region(dev, res->start, size, name)) {
4396	dev_err(dev, "can't request region for resource %pR\n", res);
4397	return IOMEM_ERR_PTR(-EBUSY);
4398	}
4399
4400	dest_ptr = devm_pci_remap_cfgspace(dev, res->start, size);
4401	if (!dest_ptr) {
4402	dev_err(dev, "ioremap failed for resource %pR\n", res);
4403	devm_release_mem_region(dev, res->start, size);
4404	dest_ptr = IOMEM_ERR_PTR(-ENOMEM);
4405	}
4406
4407	return dest_ptr;
4408	}
4409	EXPORT_SYMBOL(devm_pci_remap_cfg_resource);
4410
4411	static void __pci_set_master(struct pci_dev *dev, bool enable)
4412	{
4413	u16 old_cmd, cmd;
4414
4415	pci_read_config_word(dev, PCI_COMMAND, val: &old_cmd);
4416	if (enable)
4417	cmd = old_cmd | PCI_COMMAND_MASTER;
4418	else
4419	cmd = old_cmd & ~PCI_COMMAND_MASTER;
4420	if (cmd != old_cmd) {
4421	pci_dbg(dev, "%s bus mastering\n",
4422	enable ? "enabling" : "disabling");
4423	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
4424	}
4425	dev->is_busmaster = enable;
4426	}
4427
4428	/**
4429	* pcibios_setup - process "pci=" kernel boot arguments
4430	* @str: string used to pass in "pci=" kernel boot arguments
4431	*
4432	* Process kernel boot arguments. This is the default implementation.
4433	* Architecture specific implementations can override this as necessary.
4434	*/
4435	char * __weak __init pcibios_setup(char *str)
4436	{
4437	return str;
4438	}
4439
4440	/**
4441	* pcibios_set_master - enable PCI bus-mastering for device dev
4442	* @dev: the PCI device to enable
4443	*
4444	* Enables PCI bus-mastering for the device. This is the default
4445	* implementation. Architecture specific implementations can override
4446	* this if necessary.
4447	*/
4448	void __weak pcibios_set_master(struct pci_dev *dev)
4449	{
4450	u8 lat;
4451
4452	/* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */
4453	if (pci_is_pcie(dev))
4454	return;
4455
4456	pci_read_config_byte(dev, PCI_LATENCY_TIMER, val: &lat);
4457	if (lat < 16)
4458	lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency;
4459	else if (lat > pcibios_max_latency)
4460	lat = pcibios_max_latency;
4461	else
4462	return;
4463
4464	pci_write_config_byte(dev, PCI_LATENCY_TIMER, val: lat);
4465	}
4466
4467	/**
4468	* pci_set_master - enables bus-mastering for device dev
4469	* @dev: the PCI device to enable
4470	*
4471	* Enables bus-mastering on the device and calls pcibios_set_master()
4472	* to do the needed arch specific settings.
4473	*/
4474	void pci_set_master(struct pci_dev *dev)
4475	{
4476	__pci_set_master(dev, enable: true);
4477	pcibios_set_master(dev);
4478	}
4479	EXPORT_SYMBOL(pci_set_master);
4480
4481	/**
4482	* pci_clear_master - disables bus-mastering for device dev
4483	* @dev: the PCI device to disable
4484	*/
4485	void pci_clear_master(struct pci_dev *dev)
4486	{
4487	__pci_set_master(dev, enable: false);
4488	}
4489	EXPORT_SYMBOL(pci_clear_master);
4490
4491	/**
4492	* pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed
4493	* @dev: the PCI device for which MWI is to be enabled
4494	*
4495	* Helper function for pci_set_mwi.
4496	* Originally copied from drivers/net/acenic.c.
4497	* Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>.
4498	*
4499	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4500	*/
4501	int pci_set_cacheline_size(struct pci_dev *dev)
4502	{
4503	u8 cacheline_size;
4504
4505	if (!pci_cache_line_size)
4506	return -EINVAL;
4507
4508	/* Validate current setting: the PCI_CACHE_LINE_SIZE must be
4509	equal to or multiple of the right value. */
4510	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, val: &cacheline_size);
4511	if (cacheline_size >= pci_cache_line_size &&
4512	(cacheline_size % pci_cache_line_size) == 0)
4513	return 0;
4514
4515	/* Write the correct value. */
4516	pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, val: pci_cache_line_size);
4517	/* Read it back. */
4518	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, val: &cacheline_size);
4519	if (cacheline_size == pci_cache_line_size)
4520	return 0;
4521
4522	pci_dbg(dev, "cache line size of %d is not supported\n",
4523	pci_cache_line_size << 2);
4524
4525	return -EINVAL;
4526	}
4527	EXPORT_SYMBOL_GPL(pci_set_cacheline_size);
4528
4529	/**
4530	* pci_set_mwi - enables memory-write-invalidate PCI transaction
4531	* @dev: the PCI device for which MWI is enabled
4532	*
4533	* Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4534	*
4535	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4536	*/
4537	int pci_set_mwi(struct pci_dev *dev)
4538	{
4539	#ifdef PCI_DISABLE_MWI
4540	return 0;
4541	#else
4542	int rc;
4543	u16 cmd;
4544
4545	rc = pci_set_cacheline_size(dev);
4546	if (rc)
4547	return rc;
4548
4549	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
4550	if (!(cmd & PCI_COMMAND_INVALIDATE)) {
4551	pci_dbg(dev, "enabling Mem-Wr-Inval\n");
4552	cmd |= PCI_COMMAND_INVALIDATE;
4553	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
4554	}
4555	return 0;
4556	#endif
4557	}
4558	EXPORT_SYMBOL(pci_set_mwi);
4559
4560	/**
4561	* pcim_set_mwi - a device-managed pci_set_mwi()
4562	* @dev: the PCI device for which MWI is enabled
4563	*
4564	* Managed pci_set_mwi().
4565	*
4566	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4567	*/
4568	int pcim_set_mwi(struct pci_dev *dev)
4569	{
4570	struct pci_devres *dr;
4571
4572	dr = find_pci_dr(pdev: dev);
4573	if (!dr)
4574	return -ENOMEM;
4575
4576	dr->mwi = 1;
4577	return pci_set_mwi(dev);
4578	}
4579	EXPORT_SYMBOL(pcim_set_mwi);
4580
4581	/**
4582	* pci_try_set_mwi - enables memory-write-invalidate PCI transaction
4583	* @dev: the PCI device for which MWI is enabled
4584	*
4585	* Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4586	* Callers are not required to check the return value.
4587	*
4588	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4589	*/
4590	int pci_try_set_mwi(struct pci_dev *dev)
4591	{
4592	#ifdef PCI_DISABLE_MWI
4593	return 0;
4594	#else
4595	return pci_set_mwi(dev);
4596	#endif
4597	}
4598	EXPORT_SYMBOL(pci_try_set_mwi);
4599
4600	/**
4601	* pci_clear_mwi - disables Memory-Write-Invalidate for device dev
4602	* @dev: the PCI device to disable
4603	*
4604	* Disables PCI Memory-Write-Invalidate transaction on the device
4605	*/
4606	void pci_clear_mwi(struct pci_dev *dev)
4607	{
4608	#ifndef PCI_DISABLE_MWI
4609	u16 cmd;
4610
4611	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
4612	if (cmd & PCI_COMMAND_INVALIDATE) {
4613	cmd &= ~PCI_COMMAND_INVALIDATE;
4614	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
4615	}
4616	#endif
4617	}
4618	EXPORT_SYMBOL(pci_clear_mwi);
4619
4620	/**
4621	* pci_disable_parity - disable parity checking for device
4622	* @dev: the PCI device to operate on
4623	*
4624	* Disable parity checking for device @dev
4625	*/
4626	void pci_disable_parity(struct pci_dev *dev)
4627	{
4628	u16 cmd;
4629
4630	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
4631	if (cmd & PCI_COMMAND_PARITY) {
4632	cmd &= ~PCI_COMMAND_PARITY;
4633	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
4634	}
4635	}
4636
4637	/**
4638	* pci_intx - enables/disables PCI INTx for device dev
4639	* @pdev: the PCI device to operate on
4640	* @enable: boolean: whether to enable or disable PCI INTx
4641	*
4642	* Enables/disables PCI INTx for device @pdev
4643	*/
4644	void pci_intx(struct pci_dev *pdev, int enable)
4645	{
4646	u16 pci_command, new;
4647
4648	pci_read_config_word(dev: pdev, PCI_COMMAND, val: &pci_command);
4649
4650	if (enable)
4651	new = pci_command & ~PCI_COMMAND_INTX_DISABLE;
4652	else
4653	new = pci_command | PCI_COMMAND_INTX_DISABLE;
4654
4655	if (new != pci_command) {
4656	struct pci_devres *dr;
4657
4658	pci_write_config_word(dev: pdev, PCI_COMMAND, val: new);
4659
4660	dr = find_pci_dr(pdev);
4661	if (dr && !dr->restore_intx) {
4662	dr->restore_intx = 1;
4663	dr->orig_intx = !enable;
4664	}
4665	}
4666	}
4667	EXPORT_SYMBOL_GPL(pci_intx);
4668
4669	static bool pci_check_and_set_intx_mask(struct pci_dev *dev, bool mask)
4670	{
4671	struct pci_bus *bus = dev->bus;
4672	bool mask_updated = true;
4673	u32 cmd_status_dword;
4674	u16 origcmd, newcmd;
4675	unsigned long flags;
4676	bool irq_pending;
4677
4678	/*
4679	* We do a single dword read to retrieve both command and status.
4680	* Document assumptions that make this possible.
4681	*/
4682	BUILD_BUG_ON(PCI_COMMAND % 4);
4683	BUILD_BUG_ON(PCI_COMMAND + 2 != PCI_STATUS);
4684
4685	raw_spin_lock_irqsave(&pci_lock, flags);
4686
4687	bus->ops->read(bus, dev->devfn, PCI_COMMAND, 4, &cmd_status_dword);
4688
4689	irq_pending = (cmd_status_dword >> 16) & PCI_STATUS_INTERRUPT;
4690
4691	/*
4692	* Check interrupt status register to see whether our device
4693	* triggered the interrupt (when masking) or the next IRQ is
4694	* already pending (when unmasking).
4695	*/
4696	if (mask != irq_pending) {
4697	mask_updated = false;
4698	goto done;
4699	}
4700
4701	origcmd = cmd_status_dword;
4702	newcmd = origcmd & ~PCI_COMMAND_INTX_DISABLE;
4703	if (mask)
4704	newcmd |= PCI_COMMAND_INTX_DISABLE;
4705	if (newcmd != origcmd)
4706	bus->ops->write(bus, dev->devfn, PCI_COMMAND, 2, newcmd);
4707
4708	done:
4709	raw_spin_unlock_irqrestore(&pci_lock, flags);
4710
4711	return mask_updated;
4712	}
4713
4714	/**
4715	* pci_check_and_mask_intx - mask INTx on pending interrupt
4716	* @dev: the PCI device to operate on
4717	*
4718	* Check if the device dev has its INTx line asserted, mask it and return
4719	* true in that case. False is returned if no interrupt was pending.
4720	*/
4721	bool pci_check_and_mask_intx(struct pci_dev *dev)
4722	{
4723	return pci_check_and_set_intx_mask(dev, mask: true);
4724	}
4725	EXPORT_SYMBOL_GPL(pci_check_and_mask_intx);
4726
4727	/**
4728	* pci_check_and_unmask_intx - unmask INTx if no interrupt is pending
4729	* @dev: the PCI device to operate on
4730	*
4731	* Check if the device dev has its INTx line asserted, unmask it if not and
4732	* return true. False is returned and the mask remains active if there was
4733	* still an interrupt pending.
4734	*/
4735	bool pci_check_and_unmask_intx(struct pci_dev *dev)
4736	{
4737	return pci_check_and_set_intx_mask(dev, mask: false);
4738	}
4739	EXPORT_SYMBOL_GPL(pci_check_and_unmask_intx);
4740
4741	/**
4742	* pci_wait_for_pending_transaction - wait for pending transaction
4743	* @dev: the PCI device to operate on
4744	*
4745	* Return 0 if transaction is pending 1 otherwise.
4746	*/
4747	int pci_wait_for_pending_transaction(struct pci_dev *dev)
4748	{
4749	if (!pci_is_pcie(dev))
4750	return 1;
4751
4752	return pci_wait_for_pending(dev, pos: pci_pcie_cap(dev) + PCI_EXP_DEVSTA,
4753	PCI_EXP_DEVSTA_TRPND);
4754	}
4755	EXPORT_SYMBOL(pci_wait_for_pending_transaction);
4756
4757	/**
4758	* pcie_flr - initiate a PCIe function level reset
4759	* @dev: device to reset
4760	*
4761	* Initiate a function level reset unconditionally on @dev without
4762	* checking any flags and DEVCAP
4763	*/
4764	int pcie_flr(struct pci_dev *dev)
4765	{
4766	if (!pci_wait_for_pending_transaction(dev))
4767	pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
4768
4769	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
4770
4771	if (dev->imm_ready)
4772	return 0;
4773
4774	/*
4775	* Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within
4776	* 100ms, but may silently discard requests while the FLR is in
4777	* progress. Wait 100ms before trying to access the device.
4778	*/
4779	msleep(msecs: 100);
4780
4781	return pci_dev_wait(dev, reset_type: "FLR", PCIE_RESET_READY_POLL_MS);
4782	}
4783	EXPORT_SYMBOL_GPL(pcie_flr);
4784
4785	/**
4786	* pcie_reset_flr - initiate a PCIe function level reset
4787	* @dev: device to reset
4788	* @probe: if true, return 0 if device can be reset this way
4789	*
4790	* Initiate a function level reset on @dev.
4791	*/
4792	int pcie_reset_flr(struct pci_dev *dev, bool probe)
4793	{
4794	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4795	return -ENOTTY;
4796
4797	if (!(dev->devcap & PCI_EXP_DEVCAP_FLR))
4798	return -ENOTTY;
4799
4800	if (probe)
4801	return 0;
4802
4803	return pcie_flr(dev);
4804	}
4805	EXPORT_SYMBOL_GPL(pcie_reset_flr);
4806
4807	static int pci_af_flr(struct pci_dev *dev, bool probe)
4808	{
4809	int pos;
4810	u8 cap;
4811
4812	pos = pci_find_capability(dev, PCI_CAP_ID_AF);
4813	if (!pos)
4814	return -ENOTTY;
4815
4816	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4817	return -ENOTTY;
4818
4819	pci_read_config_byte(dev, where: pos + PCI_AF_CAP, val: &cap);
4820	if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
4821	return -ENOTTY;
4822
4823	if (probe)
4824	return 0;
4825
4826	/*
4827	* Wait for Transaction Pending bit to clear. A word-aligned test
4828	* is used, so we use the control offset rather than status and shift
4829	* the test bit to match.
4830	*/
4831	if (!pci_wait_for_pending(dev, pos: pos + PCI_AF_CTRL,
4832	PCI_AF_STATUS_TP << 8))
4833	pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");
4834
4835	pci_write_config_byte(dev, where: pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
4836
4837	if (dev->imm_ready)
4838	return 0;
4839
4840	/*
4841	* Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006,
4842	* updated 27 July 2006; a device must complete an FLR within
4843	* 100ms, but may silently discard requests while the FLR is in
4844	* progress. Wait 100ms before trying to access the device.
4845	*/
4846	msleep(msecs: 100);
4847
4848	return pci_dev_wait(dev, reset_type: "AF_FLR", PCIE_RESET_READY_POLL_MS);
4849	}
4850
4851	/**
4852	* pci_pm_reset - Put device into PCI_D3 and back into PCI_D0.
4853	* @dev: Device to reset.
4854	* @probe: if true, return 0 if the device can be reset this way.
4855	*
4856	* If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is
4857	* unset, it will be reinitialized internally when going from PCI_D3hot to
4858	* PCI_D0. If that's the case and the device is not in a low-power state
4859	* already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset.
4860	*
4861	* NOTE: This causes the caller to sleep for twice the device power transition
4862	* cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms
4863	* by default (i.e. unless the @dev's d3hot_delay field has a different value).
4864	* Moreover, only devices in D0 can be reset by this function.
4865	*/
4866	static int pci_pm_reset(struct pci_dev *dev, bool probe)
4867	{
4868	u16 csr;
4869
4870	if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)
4871	return -ENOTTY;
4872
4873	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &csr);
4874	if (csr & PCI_PM_CTRL_NO_SOFT_RESET)
4875	return -ENOTTY;
4876
4877	if (probe)
4878	return 0;
4879
4880	if (dev->current_state != PCI_D0)
4881	return -EINVAL;
4882
4883	csr &= ~PCI_PM_CTRL_STATE_MASK;
4884	csr |= PCI_D3hot;
4885	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: csr);
4886	pci_dev_d3_sleep(dev);
4887
4888	csr &= ~PCI_PM_CTRL_STATE_MASK;
4889	csr |= PCI_D0;
4890	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: csr);
4891	pci_dev_d3_sleep(dev);
4892
4893	return pci_dev_wait(dev, reset_type: "PM D3hot->D0", PCIE_RESET_READY_POLL_MS);
4894	}
4895
4896	/**
4897	* pcie_wait_for_link_status - Wait for link status change
4898	* @pdev: Device whose link to wait for.
4899	* @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE.
4900	* @active: Waiting for active or inactive?
4901	*
4902	* Return 0 if successful, or -ETIMEDOUT if status has not changed within
4903	* PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds.
4904	*/
4905	static int pcie_wait_for_link_status(struct pci_dev *pdev,
4906	bool use_lt, bool active)
4907	{
4908	u16 lnksta_mask, lnksta_match;
4909	unsigned long end_jiffies;
4910	u16 lnksta;
4911
4912	lnksta_mask = use_lt ? PCI_EXP_LNKSTA_LT : PCI_EXP_LNKSTA_DLLLA;
4913	lnksta_match = active ? lnksta_mask : 0;
4914
4915	end_jiffies = jiffies + msecs_to_jiffies(PCIE_LINK_RETRAIN_TIMEOUT_MS);
4916	do {
4917	pcie_capability_read_word(dev: pdev, PCI_EXP_LNKSTA, val: &lnksta);
4918	if ((lnksta & lnksta_mask) == lnksta_match)
4919	return 0;
4920	msleep(msecs: 1);
4921	} while (time_before(jiffies, end_jiffies));
4922
4923	return -ETIMEDOUT;
4924	}
4925
4926	/**
4927	* pcie_retrain_link - Request a link retrain and wait for it to complete
4928	* @pdev: Device whose link to retrain.
4929	* @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE, for status.
4930	*
4931	* Retrain completion status is retrieved from the Link Status Register
4932	* according to @use_lt. It is not verified whether the use of the DLLLA
4933	* bit is valid.
4934	*
4935	* Return 0 if successful, or -ETIMEDOUT if training has not completed
4936	* within PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds.
4937	*/
4938	int pcie_retrain_link(struct pci_dev *pdev, bool use_lt)
4939	{
4940	int rc;
4941
4942	/*
4943	* Ensure the updated LNKCTL parameters are used during link
4944	* training by checking that there is no ongoing link training to
4945	* avoid LTSSM race as recommended in Implementation Note at the
4946	* end of PCIe r6.0.1 sec 7.5.3.7.
4947	*/
4948	rc = pcie_wait_for_link_status(pdev, use_lt, active: !use_lt);
4949	if (rc)
4950	return rc;
4951
4952	pcie_capability_set_word(dev: pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL);
4953	if (pdev->clear_retrain_link) {
4954	/*
4955	* Due to an erratum in some devices the Retrain Link bit
4956	* needs to be cleared again manually to allow the link
4957	* training to succeed.
4958	*/
4959	pcie_capability_clear_word(dev: pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL);
4960	}
4961
4962	return pcie_wait_for_link_status(pdev, use_lt, active: !use_lt);
4963	}
4964
4965	/**
4966	* pcie_wait_for_link_delay - Wait until link is active or inactive
4967	* @pdev: Bridge device
4968	* @active: waiting for active or inactive?
4969	* @delay: Delay to wait after link has become active (in ms)
4970	*
4971	* Use this to wait till link becomes active or inactive.
4972	*/
4973	static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active,
4974	int delay)
4975	{
4976	int rc;
4977
4978	/*
4979	* Some controllers might not implement link active reporting. In this
4980	* case, we wait for 1000 ms + any delay requested by the caller.
4981	*/
4982	if (!pdev->link_active_reporting) {
4983	msleep(PCIE_LINK_RETRAIN_TIMEOUT_MS + delay);
4984	return true;
4985	}
4986
4987	/*
4988	* PCIe r4.0 sec 6.6.1, a component must enter LTSSM Detect within 20ms,
4989	* after which we should expect an link active if the reset was
4990	* successful. If so, software must wait a minimum 100ms before sending
4991	* configuration requests to devices downstream this port.
4992	*
4993	* If the link fails to activate, either the device was physically
4994	* removed or the link is permanently failed.
4995	*/
4996	if (active)
4997	msleep(msecs: 20);
4998	rc = pcie_wait_for_link_status(pdev, use_lt: false, active);
4999	if (active) {
5000	if (rc)
5001	rc = pcie_failed_link_retrain(dev: pdev);
5002	if (rc)
5003	return false;
5004
5005	msleep(msecs: delay);
5006	return true;
5007	}
5008
5009	if (rc)
5010	return false;
5011
5012	return true;
5013	}
5014
5015	/**
5016	* pcie_wait_for_link - Wait until link is active or inactive
5017	* @pdev: Bridge device
5018	* @active: waiting for active or inactive?
5019	*
5020	* Use this to wait till link becomes active or inactive.
5021	*/
5022	bool pcie_wait_for_link(struct pci_dev *pdev, bool active)
5023	{
5024	return pcie_wait_for_link_delay(pdev, active, delay: 100);
5025	}
5026
5027	/*
5028	* Find maximum D3cold delay required by all the devices on the bus. The
5029	* spec says 100 ms, but firmware can lower it and we allow drivers to
5030	* increase it as well.
5031	*
5032	* Called with @pci_bus_sem locked for reading.
5033	*/
5034	static int pci_bus_max_d3cold_delay(const struct pci_bus *bus)
5035	{
5036	const struct pci_dev *pdev;
5037	int min_delay = 100;
5038	int max_delay = 0;
5039
5040	list_for_each_entry(pdev, &bus->devices, bus_list) {
5041	if (pdev->d3cold_delay < min_delay)
5042	min_delay = pdev->d3cold_delay;
5043	if (pdev->d3cold_delay > max_delay)
5044	max_delay = pdev->d3cold_delay;
5045	}
5046
5047	return max(min_delay, max_delay);
5048	}
5049
5050	/**
5051	* pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible
5052	* @dev: PCI bridge
5053	* @reset_type: reset type in human-readable form
5054	*
5055	* Handle necessary delays before access to the devices on the secondary
5056	* side of the bridge are permitted after D3cold to D0 transition
5057	* or Conventional Reset.
5058	*
5059	* For PCIe this means the delays in PCIe 5.0 section 6.6.1. For
5060	* conventional PCI it means Tpvrh + Trhfa specified in PCI 3.0 section
5061	* 4.3.2.
5062	*
5063	* Return 0 on success or -ENOTTY if the first device on the secondary bus
5064	* failed to become accessible.
5065	*/
5066	int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type)
5067	{
5068	struct pci_dev *child;
5069	int delay;
5070
5071	if (pci_dev_is_disconnected(dev))
5072	return 0;
5073
5074	if (!pci_is_bridge(dev))
5075	return 0;
5076
5077	down_read(sem: &pci_bus_sem);
5078
5079	/*
5080	* We only deal with devices that are present currently on the bus.
5081	* For any hot-added devices the access delay is handled in pciehp
5082	* board_added(). In case of ACPI hotplug the firmware is expected
5083	* to configure the devices before OS is notified.
5084	*/
5085	if (!dev->subordinate || list_empty(head: &dev->subordinate->devices)) {
5086	up_read(sem: &pci_bus_sem);
5087	return 0;
5088	}
5089
5090	/* Take d3cold_delay requirements into account */
5091	delay = pci_bus_max_d3cold_delay(bus: dev->subordinate);
5092	if (!delay) {
5093	up_read(sem: &pci_bus_sem);
5094	return 0;
5095	}
5096
5097	child = list_first_entry(&dev->subordinate->devices, struct pci_dev,
5098	bus_list);
5099	up_read(sem: &pci_bus_sem);
5100
5101	/*
5102	* Conventional PCI and PCI-X we need to wait Tpvrh + Trhfa before
5103	* accessing the device after reset (that is 1000 ms + 100 ms).
5104	*/
5105	if (!pci_is_pcie(dev)) {
5106	pci_dbg(dev, "waiting %d ms for secondary bus\n", 1000 + delay);
5107	msleep(msecs: 1000 + delay);
5108	return 0;
5109	}
5110
5111	/*
5112	* For PCIe downstream and root ports that do not support speeds
5113	* greater than 5 GT/s need to wait minimum 100 ms. For higher
5114	* speeds (gen3) we need to wait first for the data link layer to
5115	* become active.
5116	*
5117	* However, 100 ms is the minimum and the PCIe spec says the
5118	* software must allow at least 1s before it can determine that the
5119	* device that did not respond is a broken device. Also device can
5120	* take longer than that to respond if it indicates so through Request
5121	* Retry Status completions.
5122	*
5123	* Therefore we wait for 100 ms and check for the device presence
5124	* until the timeout expires.
5125	*/
5126	if (!pcie_downstream_port(dev))
5127	return 0;
5128
5129	if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) {
5130	u16 status;
5131
5132	pci_dbg(dev, "waiting %d ms for downstream link\n", delay);
5133	msleep(msecs: delay);
5134
5135	if (!pci_dev_wait(dev: child, reset_type, PCI_RESET_WAIT - delay))
5136	return 0;
5137
5138	/*
5139	* If the port supports active link reporting we now check
5140	* whether the link is active and if not bail out early with
5141	* the assumption that the device is not present anymore.
5142	*/
5143	if (!dev->link_active_reporting)
5144	return -ENOTTY;
5145
5146	pcie_capability_read_word(dev, PCI_EXP_LNKSTA, val: &status);
5147	if (!(status & PCI_EXP_LNKSTA_DLLLA))
5148	return -ENOTTY;
5149
5150	return pci_dev_wait(dev: child, reset_type,
5151	PCIE_RESET_READY_POLL_MS - PCI_RESET_WAIT);
5152	}
5153
5154	pci_dbg(dev, "waiting %d ms for downstream link, after activation\n",
5155	delay);
5156	if (!pcie_wait_for_link_delay(pdev: dev, active: true, delay)) {
5157	/* Did not train, no need to wait any further */
5158	pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n");
5159	return -ENOTTY;
5160	}
5161
5162	return pci_dev_wait(dev: child, reset_type,
5163	PCIE_RESET_READY_POLL_MS - delay);
5164	}
5165
5166	void pci_reset_secondary_bus(struct pci_dev *dev)
5167	{
5168	u16 ctrl;
5169
5170	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, val: &ctrl);
5171	ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
5172	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, val: ctrl);
5173
5174	/*
5175	* PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms. Double
5176	* this to 2ms to ensure that we meet the minimum requirement.
5177	*/
5178	msleep(msecs: 2);
5179
5180	ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
5181	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, val: ctrl);
5182	}
5183
5184	void __weak pcibios_reset_secondary_bus(struct pci_dev *dev)
5185	{
5186	pci_reset_secondary_bus(dev);
5187	}
5188
5189	/**
5190	* pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge.
5191	* @dev: Bridge device
5192	*
5193	* Use the bridge control register to assert reset on the secondary bus.
5194	* Devices on the secondary bus are left in power-on state.
5195	*/
5196	int pci_bridge_secondary_bus_reset(struct pci_dev *dev)
5197	{
5198	pcibios_reset_secondary_bus(dev);
5199
5200	return pci_bridge_wait_for_secondary_bus(dev, reset_type: "bus reset");
5201	}
5202	EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset);
5203
5204	static int pci_parent_bus_reset(struct pci_dev *dev, bool probe)
5205	{
5206	struct pci_dev *pdev;
5207
5208	if (pci_is_root_bus(pbus: dev->bus) || dev->subordinate ||
5209	!dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
5210	return -ENOTTY;
5211
5212	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
5213	if (pdev != dev)
5214	return -ENOTTY;
5215
5216	if (probe)
5217	return 0;
5218
5219	return pci_bridge_secondary_bus_reset(dev->bus->self);
5220	}
5221
5222	static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe)
5223	{
5224	int rc = -ENOTTY;
5225
5226	if (!hotplug || !try_module_get(module: hotplug->owner))
5227	return rc;
5228
5229	if (hotplug->ops->reset_slot)
5230	rc = hotplug->ops->reset_slot(hotplug, probe);
5231
5232	module_put(module: hotplug->owner);
5233
5234	return rc;
5235	}
5236
5237	static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe)
5238	{
5239	if (dev->multifunction || dev->subordinate || !dev->slot ||
5240	dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
5241	return -ENOTTY;
5242
5243	return pci_reset_hotplug_slot(hotplug: dev->slot->hotplug, probe);
5244	}
5245
5246	static int pci_reset_bus_function(struct pci_dev *dev, bool probe)
5247	{
5248	int rc;
5249
5250	rc = pci_dev_reset_slot_function(dev, probe);
5251	if (rc != -ENOTTY)
5252	return rc;
5253	return pci_parent_bus_reset(dev, probe);
5254	}
5255
5256	void pci_dev_lock(struct pci_dev *dev)
5257	{
5258	/* block PM suspend, driver probe, etc. */
5259	device_lock(dev: &dev->dev);
5260	pci_cfg_access_lock(dev);
5261	}
5262	EXPORT_SYMBOL_GPL(pci_dev_lock);
5263
5264	/* Return 1 on successful lock, 0 on contention */
5265	int pci_dev_trylock(struct pci_dev *dev)
5266	{
5267	if (device_trylock(dev: &dev->dev)) {
5268	if (pci_cfg_access_trylock(dev))
5269	return 1;
5270	device_unlock(dev: &dev->dev);
5271	}
5272
5273	return 0;
5274	}
5275	EXPORT_SYMBOL_GPL(pci_dev_trylock);
5276
5277	void pci_dev_unlock(struct pci_dev *dev)
5278	{
5279	pci_cfg_access_unlock(dev);
5280	device_unlock(dev: &dev->dev);
5281	}
5282	EXPORT_SYMBOL_GPL(pci_dev_unlock);
5283
5284	static void pci_dev_save_and_disable(struct pci_dev *dev)
5285	{
5286	const struct pci_error_handlers *err_handler =
5287	dev->driver ? dev->driver->err_handler : NULL;
5288
5289	/*
5290	* dev->driver->err_handler->reset_prepare() is protected against
5291	* races with ->remove() by the device lock, which must be held by
5292	* the caller.
5293	*/
5294	if (err_handler && err_handler->reset_prepare)
5295	err_handler->reset_prepare(dev);
5296
5297	/*
5298	* Wake-up device prior to save. PM registers default to D0 after
5299	* reset and a simple register restore doesn't reliably return
5300	* to a non-D0 state anyway.
5301	*/
5302	pci_set_power_state(dev, PCI_D0);
5303
5304	pci_save_state(dev);
5305	/*
5306	* Disable the device by clearing the Command register, except for
5307	* INTx-disable which is set. This not only disables MMIO and I/O port
5308	* BARs, but also prevents the device from being Bus Master, preventing
5309	* DMA from the device including MSI/MSI-X interrupts. For PCI 2.3
5310	* compliant devices, INTx-disable prevents legacy interrupts.
5311	*/
5312	pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
5313	}
5314
5315	static void pci_dev_restore(struct pci_dev *dev)
5316	{
5317	const struct pci_error_handlers *err_handler =
5318	dev->driver ? dev->driver->err_handler : NULL;
5319
5320	pci_restore_state(dev);
5321
5322	/*
5323	* dev->driver->err_handler->reset_done() is protected against
5324	* races with ->remove() by the device lock, which must be held by
5325	* the caller.
5326	*/
5327	if (err_handler && err_handler->reset_done)
5328	err_handler->reset_done(dev);
5329	}
5330
5331	/* dev->reset_methods[] is a 0-terminated list of indices into this array */
5332	static const struct pci_reset_fn_method pci_reset_fn_methods[] = {
5333	{ },
5334	{ pci_dev_specific_reset, .name = "device_specific" },
5335	{ pci_dev_acpi_reset, .name = "acpi" },
5336	{ pcie_reset_flr, .name = "flr" },
5337	{ pci_af_flr, .name = "af_flr" },
5338	{ pci_pm_reset, .name = "pm" },
5339	{ pci_reset_bus_function, .name = "bus" },
5340	};
5341
5342	static ssize_t reset_method_show(struct device *dev,
5343	struct device_attribute *attr, char *buf)
5344	{
5345	struct pci_dev *pdev = to_pci_dev(dev);
5346	ssize_t len = 0;
5347	int i, m;
5348
5349	for (i = 0; i < PCI_NUM_RESET_METHODS; i++) {
5350	m = pdev->reset_methods[i];
5351	if (!m)
5352	break;
5353
5354	len += sysfs_emit_at(buf, at: len, fmt: "%s%s", len ? " " : "",
5355	pci_reset_fn_methods[m].name);
5356	}
5357
5358	if (len)
5359	len += sysfs_emit_at(buf, at: len, fmt: "\n");
5360
5361	return len;
5362	}
5363
5364	static int reset_method_lookup(const char *name)
5365	{
5366	int m;
5367
5368	for (m = 1; m < PCI_NUM_RESET_METHODS; m++) {
5369	if (sysfs_streq(s1: name, s2: pci_reset_fn_methods[m].name))
5370	return m;
5371	}
5372
5373	return 0; /* not found */
5374	}
5375
5376	static ssize_t reset_method_store(struct device *dev,
5377	struct device_attribute *attr,
5378	const char *buf, size_t count)
5379	{
5380	struct pci_dev *pdev = to_pci_dev(dev);
5381	char *options, *name;
5382	int m, n;
5383	u8 reset_methods[PCI_NUM_RESET_METHODS] = { 0 };
5384
5385	if (sysfs_streq(s1: buf, s2: "")) {
5386	pdev->reset_methods[0] = 0;
5387	pci_warn(pdev, "All device reset methods disabled by user");
5388	return count;
5389	}
5390
5391	if (sysfs_streq(s1: buf, s2: "default")) {
5392	pci_init_reset_methods(dev: pdev);
5393	return count;
5394	}
5395
5396	options = kstrndup(s: buf, len: count, GFP_KERNEL);
5397	if (!options)
5398	return -ENOMEM;
5399
5400	n = 0;
5401	while ((name = strsep(&options, " ")) != NULL) {
5402	if (sysfs_streq(s1: name, s2: ""))
5403	continue;
5404
5405	name = strim(name);
5406
5407	m = reset_method_lookup(name);
5408	if (!m) {
5409	pci_err(pdev, "Invalid reset method '%s'", name);
5410	goto error;
5411	}
5412
5413	if (pci_reset_fn_methods[m].reset_fn(pdev, PCI_RESET_PROBE)) {
5414	pci_err(pdev, "Unsupported reset method '%s'", name);
5415	goto error;
5416	}
5417
5418	if (n == PCI_NUM_RESET_METHODS - 1) {
5419	pci_err(pdev, "Too many reset methods\n");
5420	goto error;
5421	}
5422
5423	reset_methods[n++] = m;
5424	}
5425
5426	reset_methods[n] = 0;
5427
5428	/* Warn if dev-specific supported but not highest priority */
5429	if (pci_reset_fn_methods[1].reset_fn(pdev, PCI_RESET_PROBE) == 0 &&
5430	reset_methods[0] != 1)
5431	pci_warn(pdev, "Device-specific reset disabled/de-prioritized by user");
5432	memcpy(pdev->reset_methods, reset_methods, sizeof(pdev->reset_methods));
5433	kfree(objp: options);
5434	return count;
5435
5436	error:
5437	/* Leave previous methods unchanged */
5438	kfree(objp: options);
5439	return -EINVAL;
5440	}
5441	static DEVICE_ATTR_RW(reset_method);
5442
5443	static struct attribute *pci_dev_reset_method_attrs[] = {
5444	&dev_attr_reset_method.attr,
5445	NULL,
5446	};
5447
5448	static umode_t pci_dev_reset_method_attr_is_visible(struct kobject *kobj,
5449	struct attribute *a, int n)
5450	{
5451	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
5452
5453	if (!pci_reset_supported(dev: pdev))
5454	return 0;
5455
5456	return a->mode;
5457	}
5458
5459	const struct attribute_group pci_dev_reset_method_attr_group = {
5460	.attrs = pci_dev_reset_method_attrs,
5461	.is_visible = pci_dev_reset_method_attr_is_visible,
5462	};
5463
5464	/**
5465	* __pci_reset_function_locked - reset a PCI device function while holding
5466	* the @dev mutex lock.
5467	* @dev: PCI device to reset
5468	*
5469	* Some devices allow an individual function to be reset without affecting
5470	* other functions in the same device. The PCI device must be responsive
5471	* to PCI config space in order to use this function.
5472	*
5473	* The device function is presumed to be unused and the caller is holding
5474	* the device mutex lock when this function is called.
5475	*
5476	* Resetting the device will make the contents of PCI configuration space
5477	* random, so any caller of this must be prepared to reinitialise the
5478	* device including MSI, bus mastering, BARs, decoding IO and memory spaces,
5479	* etc.
5480	*
5481	* Returns 0 if the device function was successfully reset or negative if the
5482	* device doesn't support resetting a single function.
5483	*/
5484	int __pci_reset_function_locked(struct pci_dev *dev)
5485	{
5486	int i, m, rc;
5487
5488	might_sleep();
5489
5490	/*
5491	* A reset method returns -ENOTTY if it doesn't support this device and
5492	* we should try the next method.
5493	*
5494	* If it returns 0 (success), we're finished. If it returns any other
5495	* error, we're also finished: this indicates that further reset
5496	* mechanisms might be broken on the device.
5497	*/
5498	for (i = 0; i < PCI_NUM_RESET_METHODS; i++) {
5499	m = dev->reset_methods[i];
5500	if (!m)
5501	return -ENOTTY;
5502
5503	rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_DO_RESET);
5504	if (!rc)
5505	return 0;
5506	if (rc != -ENOTTY)
5507	return rc;
5508	}
5509
5510	return -ENOTTY;
5511	}
5512	EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
5513
5514	/**
5515	* pci_init_reset_methods - check whether device can be safely reset
5516	* and store supported reset mechanisms.
5517	* @dev: PCI device to check for reset mechanisms
5518	*
5519	* Some devices allow an individual function to be reset without affecting
5520	* other functions in the same device. The PCI device must be in D0-D3hot
5521	* state.
5522	*
5523	* Stores reset mechanisms supported by device in reset_methods byte array
5524	* which is a member of struct pci_dev.
5525	*/
5526	void pci_init_reset_methods(struct pci_dev *dev)
5527	{
5528	int m, i, rc;
5529
5530	BUILD_BUG_ON(ARRAY_SIZE(pci_reset_fn_methods) != PCI_NUM_RESET_METHODS);
5531
5532	might_sleep();
5533
5534	i = 0;
5535	for (m = 1; m < PCI_NUM_RESET_METHODS; m++) {
5536	rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_PROBE);
5537	if (!rc)
5538	dev->reset_methods[i++] = m;
5539	else if (rc != -ENOTTY)
5540	break;
5541	}
5542
5543	dev->reset_methods[i] = 0;
5544	}
5545
5546	/**
5547	* pci_reset_function - quiesce and reset a PCI device function
5548	* @dev: PCI device to reset
5549	*
5550	* Some devices allow an individual function to be reset without affecting
5551	* other functions in the same device. The PCI device must be responsive
5552	* to PCI config space in order to use this function.
5553	*
5554	* This function does not just reset the PCI portion of a device, but
5555	* clears all the state associated with the device. This function differs
5556	* from __pci_reset_function_locked() in that it saves and restores device state
5557	* over the reset and takes the PCI device lock.
5558	*
5559	* Returns 0 if the device function was successfully reset or negative if the
5560	* device doesn't support resetting a single function.
5561	*/
5562	int pci_reset_function(struct pci_dev *dev)
5563	{
5564	int rc;
5565
5566	if (!pci_reset_supported(dev))
5567	return -ENOTTY;
5568
5569	pci_dev_lock(dev);
5570	pci_dev_save_and_disable(dev);
5571
5572	rc = __pci_reset_function_locked(dev);
5573
5574	pci_dev_restore(dev);
5575	pci_dev_unlock(dev);
5576
5577	return rc;
5578	}
5579	EXPORT_SYMBOL_GPL(pci_reset_function);
5580
5581	/**
5582	* pci_reset_function_locked - quiesce and reset a PCI device function
5583	* @dev: PCI device to reset
5584	*
5585	* Some devices allow an individual function to be reset without affecting
5586	* other functions in the same device. The PCI device must be responsive
5587	* to PCI config space in order to use this function.
5588	*
5589	* This function does not just reset the PCI portion of a device, but
5590	* clears all the state associated with the device. This function differs
5591	* from __pci_reset_function_locked() in that it saves and restores device state
5592	* over the reset. It also differs from pci_reset_function() in that it
5593	* requires the PCI device lock to be held.
5594	*
5595	* Returns 0 if the device function was successfully reset or negative if the
5596	* device doesn't support resetting a single function.
5597	*/
5598	int pci_reset_function_locked(struct pci_dev *dev)
5599	{
5600	int rc;
5601
5602	if (!pci_reset_supported(dev))
5603	return -ENOTTY;
5604
5605	pci_dev_save_and_disable(dev);
5606
5607	rc = __pci_reset_function_locked(dev);
5608
5609	pci_dev_restore(dev);
5610
5611	return rc;
5612	}
5613	EXPORT_SYMBOL_GPL(pci_reset_function_locked);
5614
5615	/**
5616	* pci_try_reset_function - quiesce and reset a PCI device function
5617	* @dev: PCI device to reset
5618	*
5619	* Same as above, except return -EAGAIN if unable to lock device.
5620	*/
5621	int pci_try_reset_function(struct pci_dev *dev)
5622	{
5623	int rc;
5624
5625	if (!pci_reset_supported(dev))
5626	return -ENOTTY;
5627
5628	if (!pci_dev_trylock(dev))
5629	return -EAGAIN;
5630
5631	pci_dev_save_and_disable(dev);
5632	rc = __pci_reset_function_locked(dev);
5633	pci_dev_restore(dev);
5634	pci_dev_unlock(dev);
5635
5636	return rc;
5637	}
5638	EXPORT_SYMBOL_GPL(pci_try_reset_function);
5639
5640	/* Do any devices on or below this bus prevent a bus reset? */
5641	static bool pci_bus_resettable(struct pci_bus *bus)
5642	{
5643	struct pci_dev *dev;
5644
5645
5646	if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5647	return false;
5648
5649	list_for_each_entry(dev, &bus->devices, bus_list) {
5650	if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5651	(dev->subordinate && !pci_bus_resettable(bus: dev->subordinate)))
5652	return false;
5653	}
5654
5655	return true;
5656	}
5657
5658	/* Lock devices from the top of the tree down */
5659	static void pci_bus_lock(struct pci_bus *bus)
5660	{
5661	struct pci_dev *dev;
5662
5663	list_for_each_entry(dev, &bus->devices, bus_list) {
5664	pci_dev_lock(dev);
5665	if (dev->subordinate)
5666	pci_bus_lock(bus: dev->subordinate);
5667	}
5668	}
5669
5670	/* Unlock devices from the bottom of the tree up */
5671	static void pci_bus_unlock(struct pci_bus *bus)
5672	{
5673	struct pci_dev *dev;
5674
5675	list_for_each_entry(dev, &bus->devices, bus_list) {
5676	if (dev->subordinate)
5677	pci_bus_unlock(bus: dev->subordinate);
5678	pci_dev_unlock(dev);
5679	}
5680	}
5681
5682	/* Return 1 on successful lock, 0 on contention */
5683	static int pci_bus_trylock(struct pci_bus *bus)
5684	{
5685	struct pci_dev *dev;
5686
5687	list_for_each_entry(dev, &bus->devices, bus_list) {
5688	if (!pci_dev_trylock(dev))
5689	goto unlock;
5690	if (dev->subordinate) {
5691	if (!pci_bus_trylock(bus: dev->subordinate)) {
5692	pci_dev_unlock(dev);
5693	goto unlock;
5694	}
5695	}
5696	}
5697	return 1;
5698
5699	unlock:
5700	list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) {
5701	if (dev->subordinate)
5702	pci_bus_unlock(bus: dev->subordinate);
5703	pci_dev_unlock(dev);
5704	}
5705	return 0;
5706	}
5707
5708	/* Do any devices on or below this slot prevent a bus reset? */
5709	static bool pci_slot_resettable(struct pci_slot *slot)
5710	{
5711	struct pci_dev *dev;
5712
5713	if (slot->bus->self &&
5714	(slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5715	return false;
5716
5717	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5718	if (!dev->slot || dev->slot != slot)
5719	continue;
5720	if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5721	(dev->subordinate && !pci_bus_resettable(bus: dev->subordinate)))
5722	return false;
5723	}
5724
5725	return true;
5726	}
5727
5728	/* Lock devices from the top of the tree down */
5729	static void pci_slot_lock(struct pci_slot *slot)
5730	{
5731	struct pci_dev *dev;
5732
5733	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5734	if (!dev->slot || dev->slot != slot)
5735	continue;
5736	pci_dev_lock(dev);
5737	if (dev->subordinate)
5738	pci_bus_lock(bus: dev->subordinate);
5739	}
5740	}
5741
5742	/* Unlock devices from the bottom of the tree up */
5743	static void pci_slot_unlock(struct pci_slot *slot)
5744	{
5745	struct pci_dev *dev;
5746
5747	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5748	if (!dev->slot || dev->slot != slot)
5749	continue;
5750	if (dev->subordinate)
5751	pci_bus_unlock(bus: dev->subordinate);
5752	pci_dev_unlock(dev);
5753	}
5754	}
5755
5756	/* Return 1 on successful lock, 0 on contention */
5757	static int pci_slot_trylock(struct pci_slot *slot)
5758	{
5759	struct pci_dev *dev;
5760
5761	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5762	if (!dev->slot || dev->slot != slot)
5763	continue;
5764	if (!pci_dev_trylock(dev))
5765	goto unlock;
5766	if (dev->subordinate) {
5767	if (!pci_bus_trylock(bus: dev->subordinate)) {
5768	pci_dev_unlock(dev);
5769	goto unlock;
5770	}
5771	}
5772	}
5773	return 1;
5774
5775	unlock:
5776	list_for_each_entry_continue_reverse(dev,
5777	&slot->bus->devices, bus_list) {
5778	if (!dev->slot || dev->slot != slot)
5779	continue;
5780	if (dev->subordinate)
5781	pci_bus_unlock(bus: dev->subordinate);
5782	pci_dev_unlock(dev);
5783	}
5784	return 0;
5785	}
5786
5787	/*
5788	* Save and disable devices from the top of the tree down while holding
5789	* the @dev mutex lock for the entire tree.
5790	*/
5791	static void pci_bus_save_and_disable_locked(struct pci_bus *bus)
5792	{
5793	struct pci_dev *dev;
5794
5795	list_for_each_entry(dev, &bus->devices, bus_list) {
5796	pci_dev_save_and_disable(dev);
5797	if (dev->subordinate)
5798	pci_bus_save_and_disable_locked(bus: dev->subordinate);
5799	}
5800	}
5801
5802	/*
5803	* Restore devices from top of the tree down while holding @dev mutex lock
5804	* for the entire tree. Parent bridges need to be restored before we can
5805	* get to subordinate devices.
5806	*/
5807	static void pci_bus_restore_locked(struct pci_bus *bus)
5808	{
5809	struct pci_dev *dev;
5810
5811	list_for_each_entry(dev, &bus->devices, bus_list) {
5812	pci_dev_restore(dev);
5813	if (dev->subordinate)
5814	pci_bus_restore_locked(bus: dev->subordinate);
5815	}
5816	}
5817
5818	/*
5819	* Save and disable devices from the top of the tree down while holding
5820	* the @dev mutex lock for the entire tree.
5821	*/
5822	static void pci_slot_save_and_disable_locked(struct pci_slot *slot)
5823	{
5824	struct pci_dev *dev;
5825
5826	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5827	if (!dev->slot || dev->slot != slot)
5828	continue;
5829	pci_dev_save_and_disable(dev);
5830	if (dev->subordinate)
5831	pci_bus_save_and_disable_locked(bus: dev->subordinate);
5832	}
5833	}
5834
5835	/*
5836	* Restore devices from top of the tree down while holding @dev mutex lock
5837	* for the entire tree. Parent bridges need to be restored before we can
5838	* get to subordinate devices.
5839	*/
5840	static void pci_slot_restore_locked(struct pci_slot *slot)
5841	{
5842	struct pci_dev *dev;
5843
5844	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5845	if (!dev->slot || dev->slot != slot)
5846	continue;
5847	pci_dev_restore(dev);
5848	if (dev->subordinate)
5849	pci_bus_restore_locked(bus: dev->subordinate);
5850	}
5851	}
5852
5853	static int pci_slot_reset(struct pci_slot *slot, bool probe)
5854	{
5855	int rc;
5856
5857	if (!slot || !pci_slot_resettable(slot))
5858	return -ENOTTY;
5859
5860	if (!probe)
5861	pci_slot_lock(slot);
5862
5863	might_sleep();
5864
5865	rc = pci_reset_hotplug_slot(hotplug: slot->hotplug, probe);
5866
5867	if (!probe)
5868	pci_slot_unlock(slot);
5869
5870	return rc;
5871	}
5872
5873	/**
5874	* pci_probe_reset_slot - probe whether a PCI slot can be reset
5875	* @slot: PCI slot to probe
5876	*
5877	* Return 0 if slot can be reset, negative if a slot reset is not supported.
5878	*/
5879	int pci_probe_reset_slot(struct pci_slot *slot)
5880	{
5881	return pci_slot_reset(slot, PCI_RESET_PROBE);
5882	}
5883	EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
5884
5885	/**
5886	* __pci_reset_slot - Try to reset a PCI slot
5887	* @slot: PCI slot to reset
5888	*
5889	* A PCI bus may host multiple slots, each slot may support a reset mechanism
5890	* independent of other slots. For instance, some slots may support slot power
5891	* control. In the case of a 1:1 bus to slot architecture, this function may
5892	* wrap the bus reset to avoid spurious slot related events such as hotplug.
5893	* Generally a slot reset should be attempted before a bus reset. All of the
5894	* function of the slot and any subordinate buses behind the slot are reset
5895	* through this function. PCI config space of all devices in the slot and
5896	* behind the slot is saved before and restored after reset.
5897	*
5898	* Same as above except return -EAGAIN if the slot cannot be locked
5899	*/
5900	static int __pci_reset_slot(struct pci_slot *slot)
5901	{
5902	int rc;
5903
5904	rc = pci_slot_reset(slot, PCI_RESET_PROBE);
5905	if (rc)
5906	return rc;
5907
5908	if (pci_slot_trylock(slot)) {
5909	pci_slot_save_and_disable_locked(slot);
5910	might_sleep();
5911	rc = pci_reset_hotplug_slot(hotplug: slot->hotplug, PCI_RESET_DO_RESET);
5912	pci_slot_restore_locked(slot);
5913	pci_slot_unlock(slot);
5914	} else
5915	rc = -EAGAIN;
5916
5917	return rc;
5918	}
5919
5920	static int pci_bus_reset(struct pci_bus *bus, bool probe)
5921	{
5922	int ret;
5923
5924	if (!bus->self || !pci_bus_resettable(bus))
5925	return -ENOTTY;
5926
5927	if (probe)
5928	return 0;
5929
5930	pci_bus_lock(bus);
5931
5932	might_sleep();
5933
5934	ret = pci_bridge_secondary_bus_reset(bus->self);
5935
5936	pci_bus_unlock(bus);
5937
5938	return ret;
5939	}
5940
5941	/**
5942	* pci_bus_error_reset - reset the bridge's subordinate bus
5943	* @bridge: The parent device that connects to the bus to reset
5944	*
5945	* This function will first try to reset the slots on this bus if the method is
5946	* available. If slot reset fails or is not available, this will fall back to a
5947	* secondary bus reset.
5948	*/
5949	int pci_bus_error_reset(struct pci_dev *bridge)
5950	{
5951	struct pci_bus *bus = bridge->subordinate;
5952	struct pci_slot *slot;
5953
5954	if (!bus)
5955	return -ENOTTY;
5956
5957	mutex_lock(&pci_slot_mutex);
5958	if (list_empty(head: &bus->slots))
5959	goto bus_reset;
5960
5961	list_for_each_entry(slot, &bus->slots, list)
5962	if (pci_probe_reset_slot(slot))
5963	goto bus_reset;
5964
5965	list_for_each_entry(slot, &bus->slots, list)
5966	if (pci_slot_reset(slot, PCI_RESET_DO_RESET))
5967	goto bus_reset;
5968
5969	mutex_unlock(lock: &pci_slot_mutex);
5970	return 0;
5971	bus_reset:
5972	mutex_unlock(lock: &pci_slot_mutex);
5973	return pci_bus_reset(bus: bridge->subordinate, PCI_RESET_DO_RESET);
5974	}
5975
5976	/**
5977	* pci_probe_reset_bus - probe whether a PCI bus can be reset
5978	* @bus: PCI bus to probe
5979	*
5980	* Return 0 if bus can be reset, negative if a bus reset is not supported.
5981	*/
5982	int pci_probe_reset_bus(struct pci_bus *bus)
5983	{
5984	return pci_bus_reset(bus, PCI_RESET_PROBE);
5985	}
5986	EXPORT_SYMBOL_GPL(pci_probe_reset_bus);
5987
5988	/**
5989	* __pci_reset_bus - Try to reset a PCI bus
5990	* @bus: top level PCI bus to reset
5991	*
5992	* Same as above except return -EAGAIN if the bus cannot be locked
5993	*/
5994	static int __pci_reset_bus(struct pci_bus *bus)
5995	{
5996	int rc;
5997
5998	rc = pci_bus_reset(bus, PCI_RESET_PROBE);
5999	if (rc)
6000	return rc;
6001
6002	if (pci_bus_trylock(bus)) {
6003	pci_bus_save_and_disable_locked(bus);
6004	might_sleep();
6005	rc = pci_bridge_secondary_bus_reset(bus->self);
6006	pci_bus_restore_locked(bus);
6007	pci_bus_unlock(bus);
6008	} else
6009	rc = -EAGAIN;
6010
6011	return rc;
6012	}
6013
6014	/**
6015	* pci_reset_bus - Try to reset a PCI bus
6016	* @pdev: top level PCI device to reset via slot/bus
6017	*
6018	* Same as above except return -EAGAIN if the bus cannot be locked
6019	*/
6020	int pci_reset_bus(struct pci_dev *pdev)
6021	{
6022	return (!pci_probe_reset_slot(pdev->slot)) ?
6023	__pci_reset_slot(slot: pdev->slot) : __pci_reset_bus(bus: pdev->bus);
6024	}
6025	EXPORT_SYMBOL_GPL(pci_reset_bus);
6026
6027	/**
6028	* pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count
6029	* @dev: PCI device to query
6030	*
6031	* Returns mmrbc: maximum designed memory read count in bytes or
6032	* appropriate error value.
6033	*/
6034	int pcix_get_max_mmrbc(struct pci_dev *dev)
6035	{
6036	int cap;
6037	u32 stat;
6038
6039	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
6040	if (!cap)
6041	return -EINVAL;
6042
6043	if (pci_read_config_dword(dev, where: cap + PCI_X_STATUS, val: &stat))
6044	return -EINVAL;
6045
6046	return 512 << FIELD_GET(PCI_X_STATUS_MAX_READ, stat);
6047	}
6048	EXPORT_SYMBOL(pcix_get_max_mmrbc);
6049
6050	/**
6051	* pcix_get_mmrbc - get PCI-X maximum memory read byte count
6052	* @dev: PCI device to query
6053	*
6054	* Returns mmrbc: maximum memory read count in bytes or appropriate error
6055	* value.
6056	*/
6057	int pcix_get_mmrbc(struct pci_dev *dev)
6058	{
6059	int cap;
6060	u16 cmd;
6061
6062	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
6063	if (!cap)
6064	return -EINVAL;
6065
6066	if (pci_read_config_word(dev, where: cap + PCI_X_CMD, val: &cmd))
6067	return -EINVAL;
6068
6069	return 512 << FIELD_GET(PCI_X_CMD_MAX_READ, cmd);
6070	}
6071	EXPORT_SYMBOL(pcix_get_mmrbc);
6072
6073	/**
6074	* pcix_set_mmrbc - set PCI-X maximum memory read byte count
6075	* @dev: PCI device to query
6076	* @mmrbc: maximum memory read count in bytes
6077	* valid values are 512, 1024, 2048, 4096
6078	*
6079	* If possible sets maximum memory read byte count, some bridges have errata
6080	* that prevent this.
6081	*/
6082	int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc)
6083	{
6084	int cap;
6085	u32 stat, v, o;
6086	u16 cmd;
6087
6088	if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(n: mmrbc))
6089	return -EINVAL;
6090
6091	v = ffs(mmrbc) - 10;
6092
6093	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
6094	if (!cap)
6095	return -EINVAL;
6096
6097	if (pci_read_config_dword(dev, where: cap + PCI_X_STATUS, val: &stat))
6098	return -EINVAL;
6099
6100	if (v > FIELD_GET(PCI_X_STATUS_MAX_READ, stat))
6101	return -E2BIG;
6102
6103	if (pci_read_config_word(dev, where: cap + PCI_X_CMD, val: &cmd))
6104	return -EINVAL;
6105
6106	o = FIELD_GET(PCI_X_CMD_MAX_READ, cmd);
6107	if (o != v) {
6108	if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC))
6109	return -EIO;
6110
6111	cmd &= ~PCI_X_CMD_MAX_READ;
6112	cmd |= FIELD_PREP(PCI_X_CMD_MAX_READ, v);
6113	if (pci_write_config_word(dev, where: cap + PCI_X_CMD, val: cmd))
6114	return -EIO;
6115	}
6116	return 0;
6117	}
6118	EXPORT_SYMBOL(pcix_set_mmrbc);
6119
6120	/**
6121	* pcie_get_readrq - get PCI Express read request size
6122	* @dev: PCI device to query
6123	*
6124	* Returns maximum memory read request in bytes or appropriate error value.
6125	*/
6126	int pcie_get_readrq(struct pci_dev *dev)
6127	{
6128	u16 ctl;
6129
6130	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &ctl);
6131
6132	return 128 << FIELD_GET(PCI_EXP_DEVCTL_READRQ, ctl);
6133	}
6134	EXPORT_SYMBOL(pcie_get_readrq);
6135
6136	/**
6137	* pcie_set_readrq - set PCI Express maximum memory read request
6138	* @dev: PCI device to query
6139	* @rq: maximum memory read count in bytes
6140	* valid values are 128, 256, 512, 1024, 2048, 4096
6141	*
6142	* If possible sets maximum memory read request in bytes
6143	*/
6144	int pcie_set_readrq(struct pci_dev *dev, int rq)
6145	{
6146	u16 v;
6147	int ret;
6148	struct pci_host_bridge *bridge = pci_find_host_bridge(bus: dev->bus);
6149
6150	if (rq < 128 || rq > 4096 || !is_power_of_2(n: rq))
6151	return -EINVAL;
6152
6153	/*
6154	* If using the "performance" PCIe config, we clamp the read rq
6155	* size to the max packet size to keep the host bridge from
6156	* generating requests larger than we can cope with.
6157	*/
6158	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
6159	int mps = pcie_get_mps(dev);
6160
6161	if (mps < rq)
6162	rq = mps;
6163	}
6164
6165	v = FIELD_PREP(PCI_EXP_DEVCTL_READRQ, ffs(rq) - 8);
6166
6167	if (bridge->no_inc_mrrs) {
6168	int max_mrrs = pcie_get_readrq(dev);
6169
6170	if (rq > max_mrrs) {
6171	pci_info(dev, "can't set Max_Read_Request_Size to %d; max is %d\n", rq, max_mrrs);
6172	return -EINVAL;
6173	}
6174	}
6175
6176	ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
6177	PCI_EXP_DEVCTL_READRQ, set: v);
6178
6179	return pcibios_err_to_errno(err: ret);
6180	}
6181	EXPORT_SYMBOL(pcie_set_readrq);
6182
6183	/**
6184	* pcie_get_mps - get PCI Express maximum payload size
6185	* @dev: PCI device to query
6186	*
6187	* Returns maximum payload size in bytes
6188	*/
6189	int pcie_get_mps(struct pci_dev *dev)
6190	{
6191	u16 ctl;
6192
6193	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &ctl);
6194
6195	return 128 << FIELD_GET(PCI_EXP_DEVCTL_PAYLOAD, ctl);
6196	}
6197	EXPORT_SYMBOL(pcie_get_mps);
6198
6199	/**
6200	* pcie_set_mps - set PCI Express maximum payload size
6201	* @dev: PCI device to query
6202	* @mps: maximum payload size in bytes
6203	* valid values are 128, 256, 512, 1024, 2048, 4096
6204	*
6205	* If possible sets maximum payload size
6206	*/
6207	int pcie_set_mps(struct pci_dev *dev, int mps)
6208	{
6209	u16 v;
6210	int ret;
6211
6212	if (mps < 128 || mps > 4096 || !is_power_of_2(n: mps))
6213	return -EINVAL;
6214
6215	v = ffs(mps) - 8;
6216	if (v > dev->pcie_mpss)
6217	return -EINVAL;
6218	v = FIELD_PREP(PCI_EXP_DEVCTL_PAYLOAD, v);
6219
6220	ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
6221	PCI_EXP_DEVCTL_PAYLOAD, set: v);
6222
6223	return pcibios_err_to_errno(err: ret);
6224	}
6225	EXPORT_SYMBOL(pcie_set_mps);
6226
6227	/**
6228	* pcie_bandwidth_available - determine minimum link settings of a PCIe
6229	* device and its bandwidth limitation
6230	* @dev: PCI device to query
6231	* @limiting_dev: storage for device causing the bandwidth limitation
6232	* @speed: storage for speed of limiting device
6233	* @width: storage for width of limiting device
6234	*
6235	* Walk up the PCI device chain and find the point where the minimum
6236	* bandwidth is available. Return the bandwidth available there and (if
6237	* limiting_dev, speed, and width pointers are supplied) information about
6238	* that point. The bandwidth returned is in Mb/s, i.e., megabits/second of
6239	* raw bandwidth.
6240	*/
6241	u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev,
6242	enum pci_bus_speed *speed,
6243	enum pcie_link_width *width)
6244	{
6245	u16 lnksta;
6246	enum pci_bus_speed next_speed;
6247	enum pcie_link_width next_width;
6248	u32 bw, next_bw;
6249
6250	if (speed)
6251	*speed = PCI_SPEED_UNKNOWN;
6252	if (width)
6253	*width = PCIE_LNK_WIDTH_UNKNOWN;
6254
6255	bw = 0;
6256
6257	while (dev) {
6258	pcie_capability_read_word(dev, PCI_EXP_LNKSTA, val: &lnksta);
6259
6260	next_speed = pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS,
6261	lnksta)];
6262	next_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
6263
6264	next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed);
6265
6266	/* Check if current device limits the total bandwidth */
6267	if (!bw || next_bw <= bw) {
6268	bw = next_bw;
6269
6270	if (limiting_dev)
6271	*limiting_dev = dev;
6272	if (speed)
6273	*speed = next_speed;
6274	if (width)
6275	*width = next_width;
6276	}
6277
6278	dev = pci_upstream_bridge(dev);
6279	}
6280
6281	return bw;
6282	}
6283	EXPORT_SYMBOL(pcie_bandwidth_available);
6284
6285	/**
6286	* pcie_get_speed_cap - query for the PCI device's link speed capability
6287	* @dev: PCI device to query
6288	*
6289	* Query the PCI device speed capability. Return the maximum link speed
6290	* supported by the device.
6291	*/
6292	enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev)
6293	{
6294	u32 lnkcap2, lnkcap;
6295
6296	/*
6297	* Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18. The
6298	* implementation note there recommends using the Supported Link
6299	* Speeds Vector in Link Capabilities 2 when supported.
6300	*
6301	* Without Link Capabilities 2, i.e., prior to PCIe r3.0, software
6302	* should use the Supported Link Speeds field in Link Capabilities,
6303	* where only 2.5 GT/s and 5.0 GT/s speeds were defined.
6304	*/
6305	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, val: &lnkcap2);
6306
6307	/* PCIe r3.0-compliant */
6308	if (lnkcap2)
6309	return PCIE_LNKCAP2_SLS2SPEED(lnkcap2);
6310
6311	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, val: &lnkcap);
6312	if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB)
6313	return PCIE_SPEED_5_0GT;
6314	else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB)
6315	return PCIE_SPEED_2_5GT;
6316
6317	return PCI_SPEED_UNKNOWN;
6318	}
6319	EXPORT_SYMBOL(pcie_get_speed_cap);
6320
6321	/**
6322	* pcie_get_width_cap - query for the PCI device's link width capability
6323	* @dev: PCI device to query
6324	*
6325	* Query the PCI device width capability. Return the maximum link width
6326	* supported by the device.
6327	*/
6328	enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev)
6329	{
6330	u32 lnkcap;
6331
6332	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, val: &lnkcap);
6333	if (lnkcap)
6334	return FIELD_GET(PCI_EXP_LNKCAP_MLW, lnkcap);
6335
6336	return PCIE_LNK_WIDTH_UNKNOWN;
6337	}
6338	EXPORT_SYMBOL(pcie_get_width_cap);
6339
6340	/**
6341	* pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability
6342	* @dev: PCI device
6343	* @speed: storage for link speed
6344	* @width: storage for link width
6345	*
6346	* Calculate a PCI device's link bandwidth by querying for its link speed
6347	* and width, multiplying them, and applying encoding overhead. The result
6348	* is in Mb/s, i.e., megabits/second of raw bandwidth.
6349	*/
6350	u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed,
6351	enum pcie_link_width *width)
6352	{
6353	*speed = pcie_get_speed_cap(dev);
6354	*width = pcie_get_width_cap(dev);
6355
6356	if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN)
6357	return 0;
6358
6359	return *width * PCIE_SPEED2MBS_ENC(*speed);
6360	}
6361
6362	/**
6363	* __pcie_print_link_status - Report the PCI device's link speed and width
6364	* @dev: PCI device to query
6365	* @verbose: Print info even when enough bandwidth is available
6366	*
6367	* If the available bandwidth at the device is less than the device is
6368	* capable of, report the device's maximum possible bandwidth and the
6369	* upstream link that limits its performance. If @verbose, always print
6370	* the available bandwidth, even if the device isn't constrained.
6371	*/
6372	void __pcie_print_link_status(struct pci_dev *dev, bool verbose)
6373	{
6374	enum pcie_link_width width, width_cap;
6375	enum pci_bus_speed speed, speed_cap;
6376	struct pci_dev *limiting_dev = NULL;
6377	u32 bw_avail, bw_cap;
6378
6379	bw_cap = pcie_bandwidth_capable(dev, speed: &speed_cap, width: &width_cap);
6380	bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width);
6381
6382	if (bw_avail >= bw_cap && verbose)
6383	pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n",
6384	bw_cap / 1000, bw_cap % 1000,
6385	pci_speed_string(speed_cap), width_cap);
6386	else if (bw_avail < bw_cap)
6387	pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n",
6388	bw_avail / 1000, bw_avail % 1000,
6389	pci_speed_string(speed), width,
6390	limiting_dev ? pci_name(limiting_dev) : "<unknown>",
6391	bw_cap / 1000, bw_cap % 1000,
6392	pci_speed_string(speed_cap), width_cap);
6393	}
6394
6395	/**
6396	* pcie_print_link_status - Report the PCI device's link speed and width
6397	* @dev: PCI device to query
6398	*
6399	* Report the available bandwidth at the device.
6400	*/
6401	void pcie_print_link_status(struct pci_dev *dev)
6402	{
6403	__pcie_print_link_status(dev, verbose: true);
6404	}
6405	EXPORT_SYMBOL(pcie_print_link_status);
6406
6407	/**
6408	* pci_select_bars - Make BAR mask from the type of resource
6409	* @dev: the PCI device for which BAR mask is made
6410	* @flags: resource type mask to be selected
6411	*
6412	* This helper routine makes bar mask from the type of resource.
6413	*/
6414	int pci_select_bars(struct pci_dev *dev, unsigned long flags)
6415	{
6416	int i, bars = 0;
6417	for (i = 0; i < PCI_NUM_RESOURCES; i++)
6418	if (pci_resource_flags(dev, i) & flags)
6419	bars |= (1 << i);
6420	return bars;
6421	}
6422	EXPORT_SYMBOL(pci_select_bars);
6423
6424	/* Some architectures require additional programming to enable VGA */
6425	static arch_set_vga_state_t arch_set_vga_state;
6426
6427	void __init pci_register_set_vga_state(arch_set_vga_state_t func)
6428	{
6429	arch_set_vga_state = func; /* NULL disables */
6430	}
6431
6432	static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode,
6433	unsigned int command_bits, u32 flags)
6434	{
6435	if (arch_set_vga_state)
6436	return arch_set_vga_state(dev, decode, command_bits,
6437	flags);
6438	return 0;
6439	}
6440
6441	/**
6442	* pci_set_vga_state - set VGA decode state on device and parents if requested
6443	* @dev: the PCI device
6444	* @decode: true = enable decoding, false = disable decoding
6445	* @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY
6446	* @flags: traverse ancestors and change bridges
6447	* CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE
6448	*/
6449	int pci_set_vga_state(struct pci_dev *dev, bool decode,
6450	unsigned int command_bits, u32 flags)
6451	{
6452	struct pci_bus *bus;
6453	struct pci_dev *bridge;
6454	u16 cmd;
6455	int rc;
6456
6457	WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY)));
6458
6459	/* ARCH specific VGA enables */
6460	rc = pci_set_vga_state_arch(dev, decode, command_bits, flags);
6461	if (rc)
6462	return rc;
6463
6464	if (flags & PCI_VGA_STATE_CHANGE_DECODES) {
6465	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
6466	if (decode)
6467	cmd |= command_bits;
6468	else
6469	cmd &= ~command_bits;
6470	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
6471	}
6472
6473	if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE))
6474	return 0;
6475
6476	bus = dev->bus;
6477	while (bus) {
6478	bridge = bus->self;
6479	if (bridge) {
6480	pci_read_config_word(dev: bridge, PCI_BRIDGE_CONTROL,
6481	val: &cmd);
6482	if (decode)
6483	cmd |= PCI_BRIDGE_CTL_VGA;
6484	else
6485	cmd &= ~PCI_BRIDGE_CTL_VGA;
6486	pci_write_config_word(dev: bridge, PCI_BRIDGE_CONTROL,
6487	val: cmd);
6488	}
6489	bus = bus->parent;
6490	}
6491	return 0;
6492	}
6493
6494	#ifdef CONFIG_ACPI
6495	bool pci_pr3_present(struct pci_dev *pdev)
6496	{
6497	struct acpi_device *adev;
6498
6499	if (acpi_disabled)
6500	return false;
6501
6502	adev = ACPI_COMPANION(&pdev->dev);
6503	if (!adev)
6504	return false;
6505
6506	return adev->power.flags.power_resources &&
6507	acpi_has_method(handle: adev->handle, name: "_PR3");
6508	}
6509	EXPORT_SYMBOL_GPL(pci_pr3_present);
6510	#endif
6511
6512	/**
6513	* pci_add_dma_alias - Add a DMA devfn alias for a device
6514	* @dev: the PCI device for which alias is added
6515	* @devfn_from: alias slot and function
6516	* @nr_devfns: number of subsequent devfns to alias
6517	*
6518	* This helper encodes an 8-bit devfn as a bit number in dma_alias_mask
6519	* which is used to program permissible bus-devfn source addresses for DMA
6520	* requests in an IOMMU. These aliases factor into IOMMU group creation
6521	* and are useful for devices generating DMA requests beyond or different
6522	* from their logical bus-devfn. Examples include device quirks where the
6523	* device simply uses the wrong devfn, as well as non-transparent bridges
6524	* where the alias may be a proxy for devices in another domain.
6525	*
6526	* IOMMU group creation is performed during device discovery or addition,
6527	* prior to any potential DMA mapping and therefore prior to driver probing
6528	* (especially for userspace assigned devices where IOMMU group definition
6529	* cannot be left as a userspace activity). DMA aliases should therefore
6530	* be configured via quirks, such as the PCI fixup header quirk.
6531	*/
6532	void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from,
6533	unsigned int nr_devfns)
6534	{
6535	int devfn_to;
6536
6537	nr_devfns = min(nr_devfns, (unsigned int)MAX_NR_DEVFNS - devfn_from);
6538	devfn_to = devfn_from + nr_devfns - 1;
6539
6540	if (!dev->dma_alias_mask)
6541	dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL);
6542	if (!dev->dma_alias_mask) {
6543	pci_warn(dev, "Unable to allocate DMA alias mask\n");
6544	return;
6545	}
6546
6547	bitmap_set(map: dev->dma_alias_mask, start: devfn_from, nbits: nr_devfns);
6548
6549	if (nr_devfns == 1)
6550	pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n",
6551	PCI_SLOT(devfn_from), PCI_FUNC(devfn_from));
6552	else if (nr_devfns > 1)
6553	pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n",
6554	PCI_SLOT(devfn_from), PCI_FUNC(devfn_from),
6555	PCI_SLOT(devfn_to), PCI_FUNC(devfn_to));
6556	}
6557
6558	bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2)
6559	{
6560	return (dev1->dma_alias_mask &&
6561	test_bit(dev2->devfn, dev1->dma_alias_mask)) ||
6562	(dev2->dma_alias_mask &&
6563	test_bit(dev1->devfn, dev2->dma_alias_mask)) ||
6564	pci_real_dma_dev(dev: dev1) == dev2 ||
6565	pci_real_dma_dev(dev: dev2) == dev1;
6566	}
6567
6568	bool pci_device_is_present(struct pci_dev *pdev)
6569	{
6570	u32 v;
6571
6572	/* Check PF if pdev is a VF, since VF Vendor/Device IDs are 0xffff */
6573	pdev = pci_physfn(dev: pdev);
6574	if (pci_dev_is_disconnected(dev: pdev))
6575	return false;
6576	return pci_bus_read_dev_vendor_id(bus: pdev->bus, devfn: pdev->devfn, pl: &v, crs_timeout: 0);
6577	}
6578	EXPORT_SYMBOL_GPL(pci_device_is_present);
6579
6580	void pci_ignore_hotplug(struct pci_dev *dev)
6581	{
6582	struct pci_dev *bridge = dev->bus->self;
6583
6584	dev->ignore_hotplug = 1;
6585	/* Propagate the "ignore hotplug" setting to the parent bridge. */
6586	if (bridge)
6587	bridge->ignore_hotplug = 1;
6588	}
6589	EXPORT_SYMBOL_GPL(pci_ignore_hotplug);
6590
6591	/**
6592	* pci_real_dma_dev - Get PCI DMA device for PCI device
6593	* @dev: the PCI device that may have a PCI DMA alias
6594	*
6595	* Permits the platform to provide architecture-specific functionality to
6596	* devices needing to alias DMA to another PCI device on another PCI bus. If
6597	* the PCI device is on the same bus, it is recommended to use
6598	* pci_add_dma_alias(). This is the default implementation. Architecture
6599	* implementations can override this.
6600	*/
6601	struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev)
6602	{
6603	return dev;
6604	}
6605
6606	resource_size_t __weak pcibios_default_alignment(void)
6607	{
6608	return 0;
6609	}
6610
6611	/*
6612	* Arches that don't want to expose struct resource to userland as-is in
6613	* sysfs and /proc can implement their own pci_resource_to_user().
6614	*/
6615	void __weak pci_resource_to_user(const struct pci_dev *dev, int bar,
6616	const struct resource *rsrc,
6617	resource_size_t *start, resource_size_t *end)
6618	{
6619	*start = rsrc->start;
6620	*end = rsrc->end;
6621	}
6622
6623	static char *resource_alignment_param;
6624	static DEFINE_SPINLOCK(resource_alignment_lock);
6625
6626	/**
6627	* pci_specified_resource_alignment - get resource alignment specified by user.
6628	* @dev: the PCI device to get
6629	* @resize: whether or not to change resources' size when reassigning alignment
6630	*
6631	* RETURNS: Resource alignment if it is specified.
6632	* Zero if it is not specified.
6633	*/
6634	static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev,
6635	bool *resize)
6636	{
6637	int align_order, count;
6638	resource_size_t align = pcibios_default_alignment();
6639	const char *p;
6640	int ret;
6641
6642	spin_lock(lock: &resource_alignment_lock);
6643	p = resource_alignment_param;
6644	if (!p || !*p)
6645	goto out;
6646	if (pci_has_flag(flag: PCI_PROBE_ONLY)) {
6647	align = 0;
6648	pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n");
6649	goto out;
6650	}
6651
6652	while (*p) {
6653	count = 0;
6654	if (sscanf(p, "%d%n", &align_order, &count) == 1 &&
6655	p[count] == '@') {
6656	p += count + 1;
6657	if (align_order > 63) {
6658	pr_err("PCI: Invalid requested alignment (order %d)\n",
6659	align_order);
6660	align_order = PAGE_SHIFT;
6661	}
6662	} else {
6663	align_order = PAGE_SHIFT;
6664	}
6665
6666	ret = pci_dev_str_match(dev, p, endptr: &p);
6667	if (ret == 1) {
6668	*resize = true;
6669	align = 1ULL << align_order;
6670	break;
6671	} else if (ret < 0) {
6672	pr_err("PCI: Can't parse resource_alignment parameter: %s\n",
6673	p);
6674	break;
6675	}
6676
6677	if (*p != ';' && *p != ',') {
6678	/* End of param or invalid format */
6679	break;
6680	}
6681	p++;
6682	}
6683	out:
6684	spin_unlock(lock: &resource_alignment_lock);
6685	return align;
6686	}
6687
6688	static void pci_request_resource_alignment(struct pci_dev *dev, int bar,
6689	resource_size_t align, bool resize)
6690	{
6691	struct resource *r = &dev->resource[bar];
6692	resource_size_t size;
6693
6694	if (!(r->flags & IORESOURCE_MEM))
6695	return;
6696
6697	if (r->flags & IORESOURCE_PCI_FIXED) {
6698	pci_info(dev, "BAR%d %pR: ignoring requested alignment %#llx\n",
6699	bar, r, (unsigned long long)align);
6700	return;
6701	}
6702
6703	size = resource_size(res: r);
6704	if (size >= align)
6705	return;
6706
6707	/*
6708	* Increase the alignment of the resource. There are two ways we
6709	* can do this:
6710	*
6711	* 1) Increase the size of the resource. BARs are aligned on their
6712	* size, so when we reallocate space for this resource, we'll
6713	* allocate it with the larger alignment. This also prevents
6714	* assignment of any other BARs inside the alignment region, so
6715	* if we're requesting page alignment, this means no other BARs
6716	* will share the page.
6717	*
6718	* The disadvantage is that this makes the resource larger than
6719	* the hardware BAR, which may break drivers that compute things
6720	* based on the resource size, e.g., to find registers at a
6721	* fixed offset before the end of the BAR.
6722	*
6723	* 2) Retain the resource size, but use IORESOURCE_STARTALIGN and
6724	* set r->start to the desired alignment. By itself this
6725	* doesn't prevent other BARs being put inside the alignment
6726	* region, but if we realign *every* resource of every device in
6727	* the system, none of them will share an alignment region.
6728	*
6729	* When the user has requested alignment for only some devices via
6730	* the "pci=resource_alignment" argument, "resize" is true and we
6731	* use the first method. Otherwise we assume we're aligning all
6732	* devices and we use the second.
6733	*/
6734
6735	pci_info(dev, "BAR%d %pR: requesting alignment to %#llx\n",
6736	bar, r, (unsigned long long)align);
6737
6738	if (resize) {
6739	r->start = 0;
6740	r->end = align - 1;
6741	} else {
6742	r->flags &= ~IORESOURCE_SIZEALIGN;
6743	r->flags |= IORESOURCE_STARTALIGN;
6744	r->start = align;
6745	r->end = r->start + size - 1;
6746	}
6747	r->flags |= IORESOURCE_UNSET;
6748	}
6749
6750	/*
6751	* This function disables memory decoding and releases memory resources
6752	* of the device specified by kernel's boot parameter 'pci=resource_alignment='.
6753	* It also rounds up size to specified alignment.
6754	* Later on, the kernel will assign page-aligned memory resource back
6755	* to the device.
6756	*/
6757	void pci_reassigndev_resource_alignment(struct pci_dev *dev)
6758	{
6759	int i;
6760	struct resource *r;
6761	resource_size_t align;
6762	u16 command;
6763	bool resize = false;
6764
6765	/*
6766	* VF BARs are read-only zero according to SR-IOV spec r1.1, sec
6767	* 3.4.1.11. Their resources are allocated from the space
6768	* described by the VF BARx register in the PF's SR-IOV capability.
6769	* We can't influence their alignment here.
6770	*/
6771	if (dev->is_virtfn)
6772	return;
6773
6774	/* check if specified PCI is target device to reassign */
6775	align = pci_specified_resource_alignment(dev, resize: &resize);
6776	if (!align)
6777	return;
6778
6779	if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
6780	(dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) {
6781	pci_warn(dev, "Can't reassign resources to host bridge\n");
6782	return;
6783	}
6784
6785	pci_read_config_word(dev, PCI_COMMAND, val: &command);
6786	command &= ~PCI_COMMAND_MEMORY;
6787	pci_write_config_word(dev, PCI_COMMAND, val: command);
6788
6789	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
6790	pci_request_resource_alignment(dev, bar: i, align, resize);
6791
6792	/*
6793	* Need to disable bridge's resource window,
6794	* to enable the kernel to reassign new resource
6795	* window later on.
6796	*/
6797	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
6798	for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) {
6799	r = &dev->resource[i];
6800	if (!(r->flags & IORESOURCE_MEM))
6801	continue;
6802	r->flags |= IORESOURCE_UNSET;
6803	r->end = resource_size(res: r) - 1;
6804	r->start = 0;
6805	}
6806	pci_disable_bridge_window(dev);
6807	}
6808	}
6809
6810	static ssize_t resource_alignment_show(const struct bus_type *bus, char *buf)
6811	{
6812	size_t count = 0;
6813
6814	spin_lock(lock: &resource_alignment_lock);
6815	if (resource_alignment_param)
6816	count = sysfs_emit(buf, fmt: "%s\n", resource_alignment_param);
6817	spin_unlock(lock: &resource_alignment_lock);
6818
6819	return count;
6820	}
6821
6822	static ssize_t resource_alignment_store(const struct bus_type *bus,
6823	const char *buf, size_t count)
6824	{
6825	char *param, *old, *end;
6826
6827	if (count >= (PAGE_SIZE - 1))
6828	return -EINVAL;
6829
6830	param = kstrndup(s: buf, len: count, GFP_KERNEL);
6831	if (!param)
6832	return -ENOMEM;
6833
6834	end = strchr(param, '\n');
6835	if (end)
6836	*end = '\0';
6837
6838	spin_lock(lock: &resource_alignment_lock);
6839	old = resource_alignment_param;
6840	if (strlen(param)) {
6841	resource_alignment_param = param;
6842	} else {
6843	kfree(objp: param);
6844	resource_alignment_param = NULL;
6845	}
6846	spin_unlock(lock: &resource_alignment_lock);
6847
6848	kfree(objp: old);
6849
6850	return count;
6851	}
6852
6853	static BUS_ATTR_RW(resource_alignment);
6854
6855	static int __init pci_resource_alignment_sysfs_init(void)
6856	{
6857	return bus_create_file(bus: &pci_bus_type,
6858	attr: &bus_attr_resource_alignment);
6859	}
6860	late_initcall(pci_resource_alignment_sysfs_init);
6861
6862	static void pci_no_domains(void)
6863	{
6864	#ifdef CONFIG_PCI_DOMAINS
6865	pci_domains_supported = 0;
6866	#endif
6867	}
6868
6869	#ifdef CONFIG_PCI_DOMAINS_GENERIC
6870	static DEFINE_IDA(pci_domain_nr_static_ida);
6871	static DEFINE_IDA(pci_domain_nr_dynamic_ida);
6872
6873	static void of_pci_reserve_static_domain_nr(void)
6874	{
6875	struct device_node *np;
6876	int domain_nr;
6877
6878	for_each_node_by_type(np, "pci") {
6879	domain_nr = of_get_pci_domain_nr(np);
6880	if (domain_nr < 0)
6881	continue;
6882	/*
6883	* Permanently allocate domain_nr in dynamic_ida
6884	* to prevent it from dynamic allocation.
6885	*/
6886	ida_alloc_range(&pci_domain_nr_dynamic_ida,
6887	domain_nr, domain_nr, GFP_KERNEL);
6888	}
6889	}
6890
6891	static int of_pci_bus_find_domain_nr(struct device *parent)
6892	{
6893	static bool static_domains_reserved = false;
6894	int domain_nr;
6895
6896	/* On the first call scan device tree for static allocations. */
6897	if (!static_domains_reserved) {
6898	of_pci_reserve_static_domain_nr();
6899	static_domains_reserved = true;
6900	}
6901
6902	if (parent) {
6903	/*
6904	* If domain is in DT, allocate it in static IDA. This
6905	* prevents duplicate static allocations in case of errors
6906	* in DT.
6907	*/
6908	domain_nr = of_get_pci_domain_nr(parent->of_node);
6909	if (domain_nr >= 0)
6910	return ida_alloc_range(&pci_domain_nr_static_ida,
6911	domain_nr, domain_nr,
6912	GFP_KERNEL);
6913	}
6914
6915	/*
6916	* If domain was not specified in DT, choose a free ID from dynamic
6917	* allocations. All domain numbers from DT are permanently in
6918	* dynamic allocations to prevent assigning them to other DT nodes
6919	* without static domain.
6920	*/
6921	return ida_alloc(&pci_domain_nr_dynamic_ida, GFP_KERNEL);
6922	}
6923
6924	static void of_pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent)
6925	{
6926	if (bus->domain_nr < 0)
6927	return;
6928
6929	/* Release domain from IDA where it was allocated. */
6930	if (of_get_pci_domain_nr(parent->of_node) == bus->domain_nr)
6931	ida_free(&pci_domain_nr_static_ida, bus->domain_nr);
6932	else
6933	ida_free(&pci_domain_nr_dynamic_ida, bus->domain_nr);
6934	}
6935
6936	int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
6937	{
6938	return acpi_disabled ? of_pci_bus_find_domain_nr(parent) :
6939	acpi_pci_bus_find_domain_nr(bus);
6940	}
6941
6942	void pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent)
6943	{
6944	if (!acpi_disabled)
6945	return;
6946	of_pci_bus_release_domain_nr(bus, parent);
6947	}
6948	#endif
6949
6950	/**
6951	* pci_ext_cfg_avail - can we access extended PCI config space?
6952	*
6953	* Returns 1 if we can access PCI extended config space (offsets
6954	* greater than 0xff). This is the default implementation. Architecture
6955	* implementations can override this.
6956	*/
6957	int __weak pci_ext_cfg_avail(void)
6958	{
6959	return 1;
6960	}
6961
6962	void __weak pci_fixup_cardbus(struct pci_bus *bus)
6963	{
6964	}
6965	EXPORT_SYMBOL(pci_fixup_cardbus);
6966
6967	static int __init pci_setup(char *str)
6968	{
6969	while (str) {
6970	char *k = strchr(str, ',');
6971	if (k)
6972	*k++ = 0;
6973	if (*str && (str = pcibios_setup(str)) && *str) {
6974	if (!strcmp(str, "nomsi")) {
6975	pci_no_msi();
6976	} else if (!strncmp(str, "noats", 5)) {
6977	pr_info("PCIe: ATS is disabled\n");
6978	pcie_ats_disabled = true;
6979	} else if (!strcmp(str, "noaer")) {
6980	pci_no_aer();
6981	} else if (!strcmp(str, "earlydump")) {
6982	pci_early_dump = true;
6983	} else if (!strncmp(str, "realloc=", 8)) {
6984	pci_realloc_get_opt(str + 8);
6985	} else if (!strncmp(str, "realloc", 7)) {
6986	pci_realloc_get_opt("on");
6987	} else if (!strcmp(str, "nodomains")) {
6988	pci_no_domains();
6989	} else if (!strncmp(str, "noari", 5)) {
6990	pcie_ari_disabled = true;
6991	} else if (!strncmp(str, "cbiosize=", 9)) {
6992	pci_cardbus_io_size = memparse(ptr: str + 9, retptr: &str);
6993	} else if (!strncmp(str, "cbmemsize=", 10)) {
6994	pci_cardbus_mem_size = memparse(ptr: str + 10, retptr: &str);
6995	} else if (!strncmp(str, "resource_alignment=", 19)) {
6996	resource_alignment_param = str + 19;
6997	} else if (!strncmp(str, "ecrc=", 5)) {
6998	pcie_ecrc_get_policy(str: str + 5);
6999	} else if (!strncmp(str, "hpiosize=", 9)) {
7000	pci_hotplug_io_size = memparse(ptr: str + 9, retptr: &str);
7001	} else if (!strncmp(str, "hpmmiosize=", 11)) {
7002	pci_hotplug_mmio_size = memparse(ptr: str + 11, retptr: &str);
7003	} else if (!strncmp(str, "hpmmioprefsize=", 15)) {
7004	pci_hotplug_mmio_pref_size = memparse(ptr: str + 15, retptr: &str);
7005	} else if (!strncmp(str, "hpmemsize=", 10)) {
7006	pci_hotplug_mmio_size = memparse(ptr: str + 10, retptr: &str);
7007	pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size;
7008	} else if (!strncmp(str, "hpbussize=", 10)) {
7009	pci_hotplug_bus_size =
7010	simple_strtoul(str + 10, &str, 0);
7011	if (pci_hotplug_bus_size > 0xff)
7012	pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
7013	} else if (!strncmp(str, "pcie_bus_tune_off", 17)) {
7014	pcie_bus_config = PCIE_BUS_TUNE_OFF;
7015	} else if (!strncmp(str, "pcie_bus_safe", 13)) {
7016	pcie_bus_config = PCIE_BUS_SAFE;
7017	} else if (!strncmp(str, "pcie_bus_perf", 13)) {
7018	pcie_bus_config = PCIE_BUS_PERFORMANCE;
7019	} else if (!strncmp(str, "pcie_bus_peer2peer", 18)) {
7020	pcie_bus_config = PCIE_BUS_PEER2PEER;
7021	} else if (!strncmp(str, "pcie_scan_all", 13)) {
7022	pci_add_flags(flags: PCI_SCAN_ALL_PCIE_DEVS);
7023	} else if (!strncmp(str, "disable_acs_redir=", 18)) {
7024	disable_acs_redir_param = str + 18;
7025	} else {
7026	pr_err("PCI: Unknown option `%s'\n", str);
7027	}
7028	}
7029	str = k;
7030	}
7031	return 0;
7032	}
7033	early_param("pci", pci_setup);
7034
7035	/*
7036	* 'resource_alignment_param' and 'disable_acs_redir_param' are initialized
7037	* in pci_setup(), above, to point to data in the __initdata section which
7038	* will be freed after the init sequence is complete. We can't allocate memory
7039	* in pci_setup() because some architectures do not have any memory allocation
7040	* service available during an early_param() call. So we allocate memory and
7041	* copy the variable here before the init section is freed.
7042	*
7043	*/
7044	static int __init pci_realloc_setup_params(void)
7045	{
7046	resource_alignment_param = kstrdup(s: resource_alignment_param,
7047	GFP_KERNEL);
7048	disable_acs_redir_param = kstrdup(s: disable_acs_redir_param, GFP_KERNEL);
7049
7050	return 0;
7051	}
7052	pure_initcall(pci_realloc_setup_params);
7053