1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* PCI Bus Services, see include/linux/pci.h for further explanation.
4	*
5	* Copyright 1993 -- 1997 Drew Eckhardt, Frederic Potter,
6	* David Mosberger-Tang
7	*
8	* Copyright 1997 -- 2000 Martin Mares <mj@ucw.cz>
9	*/
10
11	#include <linux/acpi.h>
12	#include <linux/kernel.h>
13	#include <linux/delay.h>
14	#include <linux/dmi.h>
15	#include <linux/init.h>
16	#include <linux/msi.h>
17	#include <linux/of.h>
18	#include <linux/pci.h>
19	#include <linux/pm.h>
20	#include <linux/slab.h>
21	#include <linux/module.h>
22	#include <linux/spinlock.h>
23	#include <linux/string.h>
24	#include <linux/log2.h>
25	#include <linux/logic_pio.h>
26	#include <linux/pm_wakeup.h>
27	#include <linux/device.h>
28	#include <linux/pm_runtime.h>
29	#include <linux/pci_hotplug.h>
30	#include <linux/vmalloc.h>
31	#include <asm/dma.h>
32	#include <linux/aer.h>
33	#include <linux/bitfield.h>
34	#include "pci.h"
35
36	DEFINE_MUTEX(pci_slot_mutex);
37
38	const char *pci_power_names[] = {
39	"error", "D0", "D1", "D2", "D3hot", "D3cold", "unknown",
40	};
41	EXPORT_SYMBOL_GPL(pci_power_names);
42
43	#ifdef CONFIG_X86_32
44	int isa_dma_bridge_buggy;
45	EXPORT_SYMBOL(isa_dma_bridge_buggy);
46	#endif
47
48	int pci_pci_problems;
49	EXPORT_SYMBOL(pci_pci_problems);
50
51	unsigned int pci_pm_d3hot_delay;
52
53	static void pci_pme_list_scan(struct work_struct *work);
54
55	static LIST_HEAD(pci_pme_list);
56	static DEFINE_MUTEX(pci_pme_list_mutex);
57	static DECLARE_DELAYED_WORK(pci_pme_work, pci_pme_list_scan);
58
59	struct pci_pme_device {
60	struct list_head list;
61	struct pci_dev *dev;
62	};
63
64	#define PME_TIMEOUT 1000 /* How long between PME checks */
65
66	/*
67	* Following exit from Conventional Reset, devices must be ready within 1 sec
68	* (PCIe r6.0 sec 6.6.1). A D3cold to D0 transition implies a Conventional
69	* Reset (PCIe r6.0 sec 5.8).
70	*/
71	#define PCI_RESET_WAIT 1000 /* msec */
72
73	/*
74	* Devices may extend the 1 sec period through Request Retry Status
75	* completions (PCIe r6.0 sec 2.3.1). The spec does not provide an upper
76	* limit, but 60 sec ought to be enough for any device to become
77	* responsive.
78	*/
79	#define PCIE_RESET_READY_POLL_MS 60000 /* msec */
80
81	static void pci_dev_d3_sleep(struct pci_dev *dev)
82	{
83	unsigned int delay_ms = max(dev->d3hot_delay, pci_pm_d3hot_delay);
84	unsigned int upper;
85
86	if (delay_ms) {
87	/* Use a 20% upper bound, 1ms minimum */
88	upper = max(DIV_ROUND_CLOSEST(delay_ms, 5), 1U);
89	usleep_range(min: delay_ms * USEC_PER_MSEC,
90	max: (delay_ms + upper) * USEC_PER_MSEC);
91	}
92	}
93
94	bool pci_reset_supported(struct pci_dev *dev)
95	{
96	return dev->reset_methods[0] != 0;
97	}
98
99	#ifdef CONFIG_PCI_DOMAINS
100	int pci_domains_supported = 1;
101	#endif
102
103	#define DEFAULT_CARDBUS_IO_SIZE (256)
104	#define DEFAULT_CARDBUS_MEM_SIZE (64*1024*1024)
105	/* pci=cbmemsize=nnM,cbiosize=nn can override this */
106	unsigned long pci_cardbus_io_size = DEFAULT_CARDBUS_IO_SIZE;
107	unsigned long pci_cardbus_mem_size = DEFAULT_CARDBUS_MEM_SIZE;
108
109	#define DEFAULT_HOTPLUG_IO_SIZE (256)
110	#define DEFAULT_HOTPLUG_MMIO_SIZE (2*1024*1024)
111	#define DEFAULT_HOTPLUG_MMIO_PREF_SIZE (2*1024*1024)
112	/* hpiosize=nn can override this */
113	unsigned long pci_hotplug_io_size = DEFAULT_HOTPLUG_IO_SIZE;
114	/*
115	* pci=hpmmiosize=nnM overrides non-prefetchable MMIO size,
116	* pci=hpmmioprefsize=nnM overrides prefetchable MMIO size;
117	* pci=hpmemsize=nnM overrides both
118	*/
119	unsigned long pci_hotplug_mmio_size = DEFAULT_HOTPLUG_MMIO_SIZE;
120	unsigned long pci_hotplug_mmio_pref_size = DEFAULT_HOTPLUG_MMIO_PREF_SIZE;
121
122	#define DEFAULT_HOTPLUG_BUS_SIZE 1
123	unsigned long pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
124
125
126	/* PCIe MPS/MRRS strategy; can be overridden by kernel command-line param */
127	#ifdef CONFIG_PCIE_BUS_TUNE_OFF
128	enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_TUNE_OFF;
129	#elif defined CONFIG_PCIE_BUS_SAFE
130	enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_SAFE;
131	#elif defined CONFIG_PCIE_BUS_PERFORMANCE
132	enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PERFORMANCE;
133	#elif defined CONFIG_PCIE_BUS_PEER2PEER
134	enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_PEER2PEER;
135	#else
136	enum pcie_bus_config_types pcie_bus_config = PCIE_BUS_DEFAULT;
137	#endif
138
139	/*
140	* The default CLS is used if arch didn't set CLS explicitly and not
141	* all pci devices agree on the same value. Arch can override either
142	* the dfl or actual value as it sees fit. Don't forget this is
143	* measured in 32-bit words, not bytes.
144	*/
145	u8 pci_dfl_cache_line_size = L1_CACHE_BYTES >> 2;
146	u8 pci_cache_line_size;
147
148	/*
149	* If we set up a device for bus mastering, we need to check the latency
150	* timer as certain BIOSes forget to set it properly.
151	*/
152	unsigned int pcibios_max_latency = 255;
153
154	/* If set, the PCIe ARI capability will not be used. */
155	static bool pcie_ari_disabled;
156
157	/* If set, the PCIe ATS capability will not be used. */
158	static bool pcie_ats_disabled;
159
160	/* If set, the PCI config space of each device is printed during boot. */
161	bool pci_early_dump;
162
163	bool pci_ats_disabled(void)
164	{
165	return pcie_ats_disabled;
166	}
167	EXPORT_SYMBOL_GPL(pci_ats_disabled);
168
169	/* Disable bridge_d3 for all PCIe ports */
170	static bool pci_bridge_d3_disable;
171	/* Force bridge_d3 for all PCIe ports */
172	static bool pci_bridge_d3_force;
173
174	static int __init pcie_port_pm_setup(char *str)
175	{
176	if (!strcmp(str, "off"))
177	pci_bridge_d3_disable = true;
178	else if (!strcmp(str, "force"))
179	pci_bridge_d3_force = true;
180	return 1;
181	}
182	__setup("pcie_port_pm=", pcie_port_pm_setup);
183
184	/**
185	* pci_bus_max_busnr - returns maximum PCI bus number of given bus' children
186	* @bus: pointer to PCI bus structure to search
187	*
188	* Given a PCI bus, returns the highest PCI bus number present in the set
189	* including the given PCI bus and its list of child PCI buses.
190	*/
191	unsigned char pci_bus_max_busnr(struct pci_bus *bus)
192	{
193	struct pci_bus *tmp;
194	unsigned char max, n;
195
196	max = bus->busn_res.end;
197	list_for_each_entry(tmp, &bus->children, node) {
198	n = pci_bus_max_busnr(bus: tmp);
199	if (n > max)
200	max = n;
201	}
202	return max;
203	}
204	EXPORT_SYMBOL_GPL(pci_bus_max_busnr);
205
206	/**
207	* pci_status_get_and_clear_errors - return and clear error bits in PCI_STATUS
208	* @pdev: the PCI device
209	*
210	* Returns error bits set in PCI_STATUS and clears them.
211	*/
212	int pci_status_get_and_clear_errors(struct pci_dev *pdev)
213	{
214	u16 status;
215	int ret;
216
217	ret = pci_read_config_word(dev: pdev, PCI_STATUS, val: &status);
218	if (ret != PCIBIOS_SUCCESSFUL)
219	return -EIO;
220
221	status &= PCI_STATUS_ERROR_BITS;
222	if (status)
223	pci_write_config_word(dev: pdev, PCI_STATUS, val: status);
224
225	return status;
226	}
227	EXPORT_SYMBOL_GPL(pci_status_get_and_clear_errors);
228
229	#ifdef CONFIG_HAS_IOMEM
230	static void __iomem *__pci_ioremap_resource(struct pci_dev *pdev, int bar,
231	bool write_combine)
232	{
233	struct resource *res = &pdev->resource[bar];
234	resource_size_t start = res->start;
235	resource_size_t size = resource_size(res);
236
237	/*
238	* Make sure the BAR is actually a memory resource, not an IO resource
239	*/
240	if (res->flags & IORESOURCE_UNSET || !(res->flags & IORESOURCE_MEM)) {
241	pci_err(pdev, "can't ioremap BAR %d: %pR\n", bar, res);
242	return NULL;
243	}
244
245	if (write_combine)
246	return ioremap_wc(offset: start, size);
247
248	return ioremap(offset: start, size);
249	}
250
251	void __iomem *pci_ioremap_bar(struct pci_dev *pdev, int bar)
252	{
253	return __pci_ioremap_resource(pdev, bar, write_combine: false);
254	}
255	EXPORT_SYMBOL_GPL(pci_ioremap_bar);
256
257	void __iomem *pci_ioremap_wc_bar(struct pci_dev *pdev, int bar)
258	{
259	return __pci_ioremap_resource(pdev, bar, write_combine: true);
260	}
261	EXPORT_SYMBOL_GPL(pci_ioremap_wc_bar);
262	#endif
263
264	/**
265	* pci_dev_str_match_path - test if a path string matches a device
266	* @dev: the PCI device to test
267	* @path: string to match the device against
268	* @endptr: pointer to the string after the match
269	*
270	* Test if a string (typically from a kernel parameter) formatted as a
271	* path of device/function addresses matches a PCI device. The string must
272	* be of the form:
273	*
274	* [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
275	*
276	* A path for a device can be obtained using 'lspci -t'. Using a path
277	* is more robust against bus renumbering than using only a single bus,
278	* device and function address.
279	*
280	* Returns 1 if the string matches the device, 0 if it does not and
281	* a negative error code if it fails to parse the string.
282	*/
283	static int pci_dev_str_match_path(struct pci_dev *dev, const char *path,
284	const char **endptr)
285	{
286	int ret;
287	unsigned int seg, bus, slot, func;
288	char *wpath, *p;
289	char end;
290
291	*endptr = strchrnul(path, ';');
292
293	wpath = kmemdup_nul(s: path, len: *endptr - path, GFP_ATOMIC);
294	if (!wpath)
295	return -ENOMEM;
296
297	while (1) {
298	p = strrchr(wpath, '/');
299	if (!p)
300	break;
301	ret = sscanf(p, "/%x.%x%c", &slot, &func, &end);
302	if (ret != 2) {
303	ret = -EINVAL;
304	goto free_and_exit;
305	}
306
307	if (dev->devfn != PCI_DEVFN(slot, func)) {
308	ret = 0;
309	goto free_and_exit;
310	}
311
312	/*
313	* Note: we don't need to get a reference to the upstream
314	* bridge because we hold a reference to the top level
315	* device which should hold a reference to the bridge,
316	* and so on.
317	*/
318	dev = pci_upstream_bridge(dev);
319	if (!dev) {
320	ret = 0;
321	goto free_and_exit;
322	}
323
324	*p = 0;
325	}
326
327	ret = sscanf(wpath, "%x:%x:%x.%x%c", &seg, &bus, &slot,
328	&func, &end);
329	if (ret != 4) {
330	seg = 0;
331	ret = sscanf(wpath, "%x:%x.%x%c", &bus, &slot, &func, &end);
332	if (ret != 3) {
333	ret = -EINVAL;
334	goto free_and_exit;
335	}
336	}
337
338	ret = (seg == pci_domain_nr(bus: dev->bus) &&
339	bus == dev->bus->number &&
340	dev->devfn == PCI_DEVFN(slot, func));
341
342	free_and_exit:
343	kfree(objp: wpath);
344	return ret;
345	}
346
347	/**
348	* pci_dev_str_match - test if a string matches a device
349	* @dev: the PCI device to test
350	* @p: string to match the device against
351	* @endptr: pointer to the string after the match
352	*
353	* Test if a string (typically from a kernel parameter) matches a specified
354	* PCI device. The string may be of one of the following formats:
355	*
356	* [<domain>:]<bus>:<device>.<func>[/<device>.<func>]*
357	* pci:<vendor>:<device>[:<subvendor>:<subdevice>]
358	*
359	* The first format specifies a PCI bus/device/function address which
360	* may change if new hardware is inserted, if motherboard firmware changes,
361	* or due to changes caused in kernel parameters. If the domain is
362	* left unspecified, it is taken to be 0. In order to be robust against
363	* bus renumbering issues, a path of PCI device/function numbers may be used
364	* to address the specific device. The path for a device can be determined
365	* through the use of 'lspci -t'.
366	*
367	* The second format matches devices using IDs in the configuration
368	* space which may match multiple devices in the system. A value of 0
369	* for any field will match all devices. (Note: this differs from
370	* in-kernel code that uses PCI_ANY_ID which is ~0; this is for
371	* legacy reasons and convenience so users don't have to specify
372	* FFFFFFFFs on the command line.)
373	*
374	* Returns 1 if the string matches the device, 0 if it does not and
375	* a negative error code if the string cannot be parsed.
376	*/
377	static int pci_dev_str_match(struct pci_dev *dev, const char *p,
378	const char **endptr)
379	{
380	int ret;
381	int count;
382	unsigned short vendor, device, subsystem_vendor, subsystem_device;
383
384	if (strncmp(p, "pci:", 4) == 0) {
385	/* PCI vendor/device (subvendor/subdevice) IDs are specified */
386	p += 4;
387	ret = sscanf(p, "%hx:%hx:%hx:%hx%n", &vendor, &device,
388	&subsystem_vendor, &subsystem_device, &count);
389	if (ret != 4) {
390	ret = sscanf(p, "%hx:%hx%n", &vendor, &device, &count);
391	if (ret != 2)
392	return -EINVAL;
393
394	subsystem_vendor = 0;
395	subsystem_device = 0;
396	}
397
398	p += count;
399
400	if ((!vendor || vendor == dev->vendor) &&
401	(!device || device == dev->device) &&
402	(!subsystem_vendor ||
403	subsystem_vendor == dev->subsystem_vendor) &&
404	(!subsystem_device ||
405	subsystem_device == dev->subsystem_device))
406	goto found;
407	} else {
408	/*
409	* PCI Bus, Device, Function IDs are specified
410	* (optionally, may include a path of devfns following it)
411	*/
412	ret = pci_dev_str_match_path(dev, path: p, endptr: &p);
413	if (ret < 0)
414	return ret;
415	else if (ret)
416	goto found;
417	}
418
419	*endptr = p;
420	return 0;
421
422	found:
423	*endptr = p;
424	return 1;
425	}
426
427	static u8 __pci_find_next_cap_ttl(struct pci_bus *bus, unsigned int devfn,
428	u8 pos, int cap, int *ttl)
429	{
430	u8 id;
431	u16 ent;
432
433	pci_bus_read_config_byte(bus, devfn, where: pos, val: &pos);
434
435	while ((*ttl)--) {
436	if (pos < 0x40)
437	break;
438	pos &= ~3;
439	pci_bus_read_config_word(bus, devfn, where: pos, val: &ent);
440
441	id = ent & 0xff;
442	if (id == 0xff)
443	break;
444	if (id == cap)
445	return pos;
446	pos = (ent >> 8);
447	}
448	return 0;
449	}
450
451	static u8 __pci_find_next_cap(struct pci_bus *bus, unsigned int devfn,
452	u8 pos, int cap)
453	{
454	int ttl = PCI_FIND_CAP_TTL;
455
456	return __pci_find_next_cap_ttl(bus, devfn, pos, cap, ttl: &ttl);
457	}
458
459	u8 pci_find_next_capability(struct pci_dev *dev, u8 pos, int cap)
460	{
461	return __pci_find_next_cap(bus: dev->bus, devfn: dev->devfn,
462	pos: pos + PCI_CAP_LIST_NEXT, cap);
463	}
464	EXPORT_SYMBOL_GPL(pci_find_next_capability);
465
466	static u8 __pci_bus_find_cap_start(struct pci_bus *bus,
467	unsigned int devfn, u8 hdr_type)
468	{
469	u16 status;
470
471	pci_bus_read_config_word(bus, devfn, PCI_STATUS, val: &status);
472	if (!(status & PCI_STATUS_CAP_LIST))
473	return 0;
474
475	switch (hdr_type) {
476	case PCI_HEADER_TYPE_NORMAL:
477	case PCI_HEADER_TYPE_BRIDGE:
478	return PCI_CAPABILITY_LIST;
479	case PCI_HEADER_TYPE_CARDBUS:
480	return PCI_CB_CAPABILITY_LIST;
481	}
482
483	return 0;
484	}
485
486	/**
487	* pci_find_capability - query for devices' capabilities
488	* @dev: PCI device to query
489	* @cap: capability code
490	*
491	* Tell if a device supports a given PCI capability.
492	* Returns the address of the requested capability structure within the
493	* device's PCI configuration space or 0 in case the device does not
494	* support it. Possible values for @cap include:
495	*
496	* %PCI_CAP_ID_PM Power Management
497	* %PCI_CAP_ID_AGP Accelerated Graphics Port
498	* %PCI_CAP_ID_VPD Vital Product Data
499	* %PCI_CAP_ID_SLOTID Slot Identification
500	* %PCI_CAP_ID_MSI Message Signalled Interrupts
501	* %PCI_CAP_ID_CHSWP CompactPCI HotSwap
502	* %PCI_CAP_ID_PCIX PCI-X
503	* %PCI_CAP_ID_EXP PCI Express
504	*/
505	u8 pci_find_capability(struct pci_dev *dev, int cap)
506	{
507	u8 pos;
508
509	pos = __pci_bus_find_cap_start(bus: dev->bus, devfn: dev->devfn, hdr_type: dev->hdr_type);
510	if (pos)
511	pos = __pci_find_next_cap(bus: dev->bus, devfn: dev->devfn, pos, cap);
512
513	return pos;
514	}
515	EXPORT_SYMBOL(pci_find_capability);
516
517	/**
518	* pci_bus_find_capability - query for devices' capabilities
519	* @bus: the PCI bus to query
520	* @devfn: PCI device to query
521	* @cap: capability code
522	*
523	* Like pci_find_capability() but works for PCI devices that do not have a
524	* pci_dev structure set up yet.
525	*
526	* Returns the address of the requested capability structure within the
527	* device's PCI configuration space or 0 in case the device does not
528	* support it.
529	*/
530	u8 pci_bus_find_capability(struct pci_bus *bus, unsigned int devfn, int cap)
531	{
532	u8 hdr_type, pos;
533
534	pci_bus_read_config_byte(bus, devfn, PCI_HEADER_TYPE, val: &hdr_type);
535
536	pos = __pci_bus_find_cap_start(bus, devfn, hdr_type: hdr_type & PCI_HEADER_TYPE_MASK);
537	if (pos)
538	pos = __pci_find_next_cap(bus, devfn, pos, cap);
539
540	return pos;
541	}
542	EXPORT_SYMBOL(pci_bus_find_capability);
543
544	/**
545	* pci_find_next_ext_capability - Find an extended capability
546	* @dev: PCI device to query
547	* @start: address at which to start looking (0 to start at beginning of list)
548	* @cap: capability code
549	*
550	* Returns the address of the next matching extended capability structure
551	* within the device's PCI configuration space or 0 if the device does
552	* not support it. Some capabilities can occur several times, e.g., the
553	* vendor-specific capability, and this provides a way to find them all.
554	*/
555	u16 pci_find_next_ext_capability(struct pci_dev *dev, u16 start, int cap)
556	{
557	u32 header;
558	int ttl;
559	u16 pos = PCI_CFG_SPACE_SIZE;
560
561	/* minimum 8 bytes per capability */
562	ttl = (PCI_CFG_SPACE_EXP_SIZE - PCI_CFG_SPACE_SIZE) / 8;
563
564	if (dev->cfg_size <= PCI_CFG_SPACE_SIZE)
565	return 0;
566
567	if (start)
568	pos = start;
569
570	if (pci_read_config_dword(dev, where: pos, val: &header) != PCIBIOS_SUCCESSFUL)
571	return 0;
572
573	/*
574	* If we have no capabilities, this is indicated by cap ID,
575	* cap version and next pointer all being 0.
576	*/
577	if (header == 0)
578	return 0;
579
580	while (ttl-- > 0) {
581	if (PCI_EXT_CAP_ID(header) == cap && pos != start)
582	return pos;
583
584	pos = PCI_EXT_CAP_NEXT(header);
585	if (pos < PCI_CFG_SPACE_SIZE)
586	break;
587
588	if (pci_read_config_dword(dev, where: pos, val: &header) != PCIBIOS_SUCCESSFUL)
589	break;
590	}
591
592	return 0;
593	}
594	EXPORT_SYMBOL_GPL(pci_find_next_ext_capability);
595
596	/**
597	* pci_find_ext_capability - Find an extended capability
598	* @dev: PCI device to query
599	* @cap: capability code
600	*
601	* Returns the address of the requested extended capability structure
602	* within the device's PCI configuration space or 0 if the device does
603	* not support it. Possible values for @cap include:
604	*
605	* %PCI_EXT_CAP_ID_ERR Advanced Error Reporting
606	* %PCI_EXT_CAP_ID_VC Virtual Channel
607	* %PCI_EXT_CAP_ID_DSN Device Serial Number
608	* %PCI_EXT_CAP_ID_PWR Power Budgeting
609	*/
610	u16 pci_find_ext_capability(struct pci_dev *dev, int cap)
611	{
612	return pci_find_next_ext_capability(dev, 0, cap);
613	}
614	EXPORT_SYMBOL_GPL(pci_find_ext_capability);
615
616	/**
617	* pci_get_dsn - Read and return the 8-byte Device Serial Number
618	* @dev: PCI device to query
619	*
620	* Looks up the PCI_EXT_CAP_ID_DSN and reads the 8 bytes of the Device Serial
621	* Number.
622	*
623	* Returns the DSN, or zero if the capability does not exist.
624	*/
625	u64 pci_get_dsn(struct pci_dev *dev)
626	{
627	u32 dword;
628	u64 dsn;
629	int pos;
630
631	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DSN);
632	if (!pos)
633	return 0;
634
635	/*
636	* The Device Serial Number is two dwords offset 4 bytes from the
637	* capability position. The specification says that the first dword is
638	* the lower half, and the second dword is the upper half.
639	*/
640	pos += 4;
641	pci_read_config_dword(dev, where: pos, val: &dword);
642	dsn = (u64)dword;
643	pci_read_config_dword(dev, where: pos + 4, val: &dword);
644	dsn |= ((u64)dword) << 32;
645
646	return dsn;
647	}
648	EXPORT_SYMBOL_GPL(pci_get_dsn);
649
650	static u8 __pci_find_next_ht_cap(struct pci_dev *dev, u8 pos, int ht_cap)
651	{
652	int rc, ttl = PCI_FIND_CAP_TTL;
653	u8 cap, mask;
654
655	if (ht_cap == HT_CAPTYPE_SLAVE || ht_cap == HT_CAPTYPE_HOST)
656	mask = HT_3BIT_CAP_MASK;
657	else
658	mask = HT_5BIT_CAP_MASK;
659
660	pos = __pci_find_next_cap_ttl(bus: dev->bus, devfn: dev->devfn, pos,
661	PCI_CAP_ID_HT, ttl: &ttl);
662	while (pos) {
663	rc = pci_read_config_byte(dev, where: pos + 3, val: &cap);
664	if (rc != PCIBIOS_SUCCESSFUL)
665	return 0;
666
667	if ((cap & mask) == ht_cap)
668	return pos;
669
670	pos = __pci_find_next_cap_ttl(bus: dev->bus, devfn: dev->devfn,
671	pos: pos + PCI_CAP_LIST_NEXT,
672	PCI_CAP_ID_HT, ttl: &ttl);
673	}
674
675	return 0;
676	}
677
678	/**
679	* pci_find_next_ht_capability - query a device's HyperTransport capabilities
680	* @dev: PCI device to query
681	* @pos: Position from which to continue searching
682	* @ht_cap: HyperTransport capability code
683	*
684	* To be used in conjunction with pci_find_ht_capability() to search for
685	* all capabilities matching @ht_cap. @pos should always be a value returned
686	* from pci_find_ht_capability().
687	*
688	* NB. To be 100% safe against broken PCI devices, the caller should take
689	* steps to avoid an infinite loop.
690	*/
691	u8 pci_find_next_ht_capability(struct pci_dev *dev, u8 pos, int ht_cap)
692	{
693	return __pci_find_next_ht_cap(dev, pos: pos + PCI_CAP_LIST_NEXT, ht_cap);
694	}
695	EXPORT_SYMBOL_GPL(pci_find_next_ht_capability);
696
697	/**
698	* pci_find_ht_capability - query a device's HyperTransport capabilities
699	* @dev: PCI device to query
700	* @ht_cap: HyperTransport capability code
701	*
702	* Tell if a device supports a given HyperTransport capability.
703	* Returns an address within the device's PCI configuration space
704	* or 0 in case the device does not support the request capability.
705	* The address points to the PCI capability, of type PCI_CAP_ID_HT,
706	* which has a HyperTransport capability matching @ht_cap.
707	*/
708	u8 pci_find_ht_capability(struct pci_dev *dev, int ht_cap)
709	{
710	u8 pos;
711
712	pos = __pci_bus_find_cap_start(bus: dev->bus, devfn: dev->devfn, hdr_type: dev->hdr_type);
713	if (pos)
714	pos = __pci_find_next_ht_cap(dev, pos, ht_cap);
715
716	return pos;
717	}
718	EXPORT_SYMBOL_GPL(pci_find_ht_capability);
719
720	/**
721	* pci_find_vsec_capability - Find a vendor-specific extended capability
722	* @dev: PCI device to query
723	* @vendor: Vendor ID for which capability is defined
724	* @cap: Vendor-specific capability ID
725	*
726	* If @dev has Vendor ID @vendor, search for a VSEC capability with
727	* VSEC ID @cap. If found, return the capability offset in
728	* config space; otherwise return 0.
729	*/
730	u16 pci_find_vsec_capability(struct pci_dev *dev, u16 vendor, int cap)
731	{
732	u16 vsec = 0;
733	u32 header;
734	int ret;
735
736	if (vendor != dev->vendor)
737	return 0;
738
739	while ((vsec = pci_find_next_ext_capability(dev, vsec,
740	PCI_EXT_CAP_ID_VNDR))) {
741	ret = pci_read_config_dword(dev, where: vsec + PCI_VNDR_HEADER, val: &header);
742	if (ret != PCIBIOS_SUCCESSFUL)
743	continue;
744
745	if (PCI_VNDR_HEADER_ID(header) == cap)
746	return vsec;
747	}
748
749	return 0;
750	}
751	EXPORT_SYMBOL_GPL(pci_find_vsec_capability);
752
753	/**
754	* pci_find_dvsec_capability - Find DVSEC for vendor
755	* @dev: PCI device to query
756	* @vendor: Vendor ID to match for the DVSEC
757	* @dvsec: Designated Vendor-specific capability ID
758	*
759	* If DVSEC has Vendor ID @vendor and DVSEC ID @dvsec return the capability
760	* offset in config space; otherwise return 0.
761	*/
762	u16 pci_find_dvsec_capability(struct pci_dev *dev, u16 vendor, u16 dvsec)
763	{
764	int pos;
765
766	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_DVSEC);
767	if (!pos)
768	return 0;
769
770	while (pos) {
771	u16 v, id;
772
773	pci_read_config_word(dev, where: pos + PCI_DVSEC_HEADER1, val: &v);
774	pci_read_config_word(dev, where: pos + PCI_DVSEC_HEADER2, val: &id);
775	if (vendor == v && dvsec == id)
776	return pos;
777
778	pos = pci_find_next_ext_capability(dev, pos, PCI_EXT_CAP_ID_DVSEC);
779	}
780
781	return 0;
782	}
783	EXPORT_SYMBOL_GPL(pci_find_dvsec_capability);
784
785	/**
786	* pci_find_parent_resource - return resource region of parent bus of given
787	* region
788	* @dev: PCI device structure contains resources to be searched
789	* @res: child resource record for which parent is sought
790	*
791	* For given resource region of given device, return the resource region of
792	* parent bus the given region is contained in.
793	*/
794	struct resource *pci_find_parent_resource(const struct pci_dev *dev,
795	struct resource *res)
796	{
797	const struct pci_bus *bus = dev->bus;
798	struct resource *r;
799
800	pci_bus_for_each_resource(bus, r) {
801	if (!r)
802	continue;
803	if (resource_contains(r1: r, r2: res)) {
804
805	/*
806	* If the window is prefetchable but the BAR is
807	* not, the allocator made a mistake.
808	*/
809	if (r->flags & IORESOURCE_PREFETCH &&
810	!(res->flags & IORESOURCE_PREFETCH))
811	return NULL;
812
813	/*
814	* If we're below a transparent bridge, there may
815	* be both a positively-decoded aperture and a
816	* subtractively-decoded region that contain the BAR.
817	* We want the positively-decoded one, so this depends
818	* on pci_bus_for_each_resource() giving us those
819	* first.
820	*/
821	return r;
822	}
823	}
824	return NULL;
825	}
826	EXPORT_SYMBOL(pci_find_parent_resource);
827
828	/**
829	* pci_find_resource - Return matching PCI device resource
830	* @dev: PCI device to query
831	* @res: Resource to look for
832	*
833	* Goes over standard PCI resources (BARs) and checks if the given resource
834	* is partially or fully contained in any of them. In that case the
835	* matching resource is returned, %NULL otherwise.
836	*/
837	struct resource *pci_find_resource(struct pci_dev *dev, struct resource *res)
838	{
839	int i;
840
841	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
842	struct resource *r = &dev->resource[i];
843
844	if (r->start && resource_contains(r1: r, r2: res))
845	return r;
846	}
847
848	return NULL;
849	}
850	EXPORT_SYMBOL(pci_find_resource);
851
852	/**
853	* pci_resource_name - Return the name of the PCI resource
854	* @dev: PCI device to query
855	* @i: index of the resource
856	*
857	* Return the standard PCI resource (BAR) name according to their index.
858	*/
859	const char *pci_resource_name(struct pci_dev *dev, unsigned int i)
860	{
861	static const char * const bar_name[] = {
862	"BAR 0",
863	"BAR 1",
864	"BAR 2",
865	"BAR 3",
866	"BAR 4",
867	"BAR 5",
868	"ROM",
869	#ifdef CONFIG_PCI_IOV
870	"VF BAR 0",
871	"VF BAR 1",
872	"VF BAR 2",
873	"VF BAR 3",
874	"VF BAR 4",
875	"VF BAR 5",
876	#endif
877	"bridge window", /* "io" included in %pR */
878	"bridge window", /* "mem" included in %pR */
879	"bridge window", /* "mem pref" included in %pR */
880	};
881	static const char * const cardbus_name[] = {
882	"BAR 1",
883	"unknown",
884	"unknown",
885	"unknown",
886	"unknown",
887	"unknown",
888	#ifdef CONFIG_PCI_IOV
889	"unknown",
890	"unknown",
891	"unknown",
892	"unknown",
893	"unknown",
894	"unknown",
895	#endif
896	"CardBus bridge window 0", /* I/O */
897	"CardBus bridge window 1", /* I/O */
898	"CardBus bridge window 0", /* mem */
899	"CardBus bridge window 1", /* mem */
900	};
901
902	if (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS &&
903	i < ARRAY_SIZE(cardbus_name))
904	return cardbus_name[i];
905
906	if (i < ARRAY_SIZE(bar_name))
907	return bar_name[i];
908
909	return "unknown";
910	}
911
912	/**
913	* pci_wait_for_pending - wait for @mask bit(s) to clear in status word @pos
914	* @dev: the PCI device to operate on
915	* @pos: config space offset of status word
916	* @mask: mask of bit(s) to care about in status word
917	*
918	* Return 1 when mask bit(s) in status word clear, 0 otherwise.
919	*/
920	int pci_wait_for_pending(struct pci_dev *dev, int pos, u16 mask)
921	{
922	int i;
923
924	/* Wait for Transaction Pending bit clean */
925	for (i = 0; i < 4; i++) {
926	u16 status;
927	if (i)
928	msleep(msecs: (1 << (i - 1)) * 100);
929
930	pci_read_config_word(dev, where: pos, val: &status);
931	if (!(status & mask))
932	return 1;
933	}
934
935	return 0;
936	}
937
938	static int pci_acs_enable;
939
940	/**
941	* pci_request_acs - ask for ACS to be enabled if supported
942	*/
943	void pci_request_acs(void)
944	{
945	pci_acs_enable = 1;
946	}
947
948	static const char *disable_acs_redir_param;
949
950	/**
951	* pci_disable_acs_redir - disable ACS redirect capabilities
952	* @dev: the PCI device
953	*
954	* For only devices specified in the disable_acs_redir parameter.
955	*/
956	static void pci_disable_acs_redir(struct pci_dev *dev)
957	{
958	int ret = 0;
959	const char *p;
960	int pos;
961	u16 ctrl;
962
963	if (!disable_acs_redir_param)
964	return;
965
966	p = disable_acs_redir_param;
967	while (*p) {
968	ret = pci_dev_str_match(dev, p, endptr: &p);
969	if (ret < 0) {
970	pr_info_once("PCI: Can't parse disable_acs_redir parameter: %s\n",
971	disable_acs_redir_param);
972
973	break;
974	} else if (ret == 1) {
975	/* Found a match */
976	break;
977	}
978
979	if (*p != ';' && *p != ',') {
980	/* End of param or invalid format */
981	break;
982	}
983	p++;
984	}
985
986	if (ret != 1)
987	return;
988
989	if (!pci_dev_specific_disable_acs_redir(dev))
990	return;
991
992	pos = dev->acs_cap;
993	if (!pos) {
994	pci_warn(dev, "cannot disable ACS redirect for this hardware as it does not have ACS capabilities\n");
995	return;
996	}
997
998	pci_read_config_word(dev, where: pos + PCI_ACS_CTRL, val: &ctrl);
999
1000	/* P2P Request & Completion Redirect */
1001	ctrl &= ~(PCI_ACS_RR | PCI_ACS_CR | PCI_ACS_EC);
1002
1003	pci_write_config_word(dev, where: pos + PCI_ACS_CTRL, val: ctrl);
1004
1005	pci_info(dev, "disabled ACS redirect\n");
1006	}
1007
1008	/**
1009	* pci_std_enable_acs - enable ACS on devices using standard ACS capabilities
1010	* @dev: the PCI device
1011	*/
1012	static void pci_std_enable_acs(struct pci_dev *dev)
1013	{
1014	int pos;
1015	u16 cap;
1016	u16 ctrl;
1017
1018	pos = dev->acs_cap;
1019	if (!pos)
1020	return;
1021
1022	pci_read_config_word(dev, where: pos + PCI_ACS_CAP, val: &cap);
1023	pci_read_config_word(dev, where: pos + PCI_ACS_CTRL, val: &ctrl);
1024
1025	/* Source Validation */
1026	ctrl |= (cap & PCI_ACS_SV);
1027
1028	/* P2P Request Redirect */
1029	ctrl |= (cap & PCI_ACS_RR);
1030
1031	/* P2P Completion Redirect */
1032	ctrl |= (cap & PCI_ACS_CR);
1033
1034	/* Upstream Forwarding */
1035	ctrl |= (cap & PCI_ACS_UF);
1036
1037	/* Enable Translation Blocking for external devices and noats */
1038	if (pci_ats_disabled() || dev->external_facing || dev->untrusted)
1039	ctrl |= (cap & PCI_ACS_TB);
1040
1041	pci_write_config_word(dev, where: pos + PCI_ACS_CTRL, val: ctrl);
1042	}
1043
1044	/**
1045	* pci_enable_acs - enable ACS if hardware support it
1046	* @dev: the PCI device
1047	*/
1048	static void pci_enable_acs(struct pci_dev *dev)
1049	{
1050	if (!pci_acs_enable)
1051	goto disable_acs_redir;
1052
1053	if (!pci_dev_specific_enable_acs(dev))
1054	goto disable_acs_redir;
1055
1056	pci_std_enable_acs(dev);
1057
1058	disable_acs_redir:
1059	/*
1060	* Note: pci_disable_acs_redir() must be called even if ACS was not
1061	* enabled by the kernel because it may have been enabled by
1062	* platform firmware. So if we are told to disable it, we should
1063	* always disable it after setting the kernel's default
1064	* preferences.
1065	*/
1066	pci_disable_acs_redir(dev);
1067	}
1068
1069	/**
1070	* pcie_read_tlp_log - read TLP Header Log
1071	* @dev: PCIe device
1072	* @where: PCI Config offset of TLP Header Log
1073	* @tlp_log: TLP Log structure to fill
1074	*
1075	* Fill @tlp_log from TLP Header Log registers, e.g., AER or DPC.
1076	*
1077	* Return: 0 on success and filled TLP Log structure, <0 on error.
1078	*/
1079	int pcie_read_tlp_log(struct pci_dev *dev, int where,
1080	struct pcie_tlp_log *tlp_log)
1081	{
1082	int i, ret;
1083
1084	memset(tlp_log, 0, sizeof(*tlp_log));
1085
1086	for (i = 0; i < 4; i++) {
1087	ret = pci_read_config_dword(dev, where: where + i * 4,
1088	val: &tlp_log->dw[i]);
1089	if (ret)
1090	return pcibios_err_to_errno(err: ret);
1091	}
1092
1093	return 0;
1094	}
1095	EXPORT_SYMBOL_GPL(pcie_read_tlp_log);
1096
1097	/**
1098	* pci_restore_bars - restore a device's BAR values (e.g. after wake-up)
1099	* @dev: PCI device to have its BARs restored
1100	*
1101	* Restore the BAR values for a given device, so as to make it
1102	* accessible by its driver.
1103	*/
1104	static void pci_restore_bars(struct pci_dev *dev)
1105	{
1106	int i;
1107
1108	for (i = 0; i < PCI_BRIDGE_RESOURCES; i++)
1109	pci_update_resource(dev, resno: i);
1110	}
1111
1112	static inline bool platform_pci_power_manageable(struct pci_dev *dev)
1113	{
1114	if (pci_use_mid_pm())
1115	return true;
1116
1117	return acpi_pci_power_manageable(dev);
1118	}
1119
1120	static inline int platform_pci_set_power_state(struct pci_dev *dev,
1121	pci_power_t t)
1122	{
1123	if (pci_use_mid_pm())
1124	return mid_pci_set_power_state(pdev: dev, state: t);
1125
1126	return acpi_pci_set_power_state(dev, state: t);
1127	}
1128
1129	static inline pci_power_t platform_pci_get_power_state(struct pci_dev *dev)
1130	{
1131	if (pci_use_mid_pm())
1132	return mid_pci_get_power_state(pdev: dev);
1133
1134	return acpi_pci_get_power_state(dev);
1135	}
1136
1137	static inline void platform_pci_refresh_power_state(struct pci_dev *dev)
1138	{
1139	if (!pci_use_mid_pm())
1140	acpi_pci_refresh_power_state(dev);
1141	}
1142
1143	static inline pci_power_t platform_pci_choose_state(struct pci_dev *dev)
1144	{
1145	if (pci_use_mid_pm())
1146	return PCI_POWER_ERROR;
1147
1148	return acpi_pci_choose_state(pdev: dev);
1149	}
1150
1151	static inline int platform_pci_set_wakeup(struct pci_dev *dev, bool enable)
1152	{
1153	if (pci_use_mid_pm())
1154	return PCI_POWER_ERROR;
1155
1156	return acpi_pci_wakeup(dev, enable);
1157	}
1158
1159	static inline bool platform_pci_need_resume(struct pci_dev *dev)
1160	{
1161	if (pci_use_mid_pm())
1162	return false;
1163
1164	return acpi_pci_need_resume(dev);
1165	}
1166
1167	static inline bool platform_pci_bridge_d3(struct pci_dev *dev)
1168	{
1169	if (pci_use_mid_pm())
1170	return false;
1171
1172	return acpi_pci_bridge_d3(dev);
1173	}
1174
1175	/**
1176	* pci_update_current_state - Read power state of given device and cache it
1177	* @dev: PCI device to handle.
1178	* @state: State to cache in case the device doesn't have the PM capability
1179	*
1180	* The power state is read from the PMCSR register, which however is
1181	* inaccessible in D3cold. The platform firmware is therefore queried first
1182	* to detect accessibility of the register. In case the platform firmware
1183	* reports an incorrect state or the device isn't power manageable by the
1184	* platform at all, we try to detect D3cold by testing accessibility of the
1185	* vendor ID in config space.
1186	*/
1187	void pci_update_current_state(struct pci_dev *dev, pci_power_t state)
1188	{
1189	if (platform_pci_get_power_state(dev) == PCI_D3cold) {
1190	dev->current_state = PCI_D3cold;
1191	} else if (dev->pm_cap) {
1192	u16 pmcsr;
1193
1194	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
1195	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1196	dev->current_state = PCI_D3cold;
1197	return;
1198	}
1199	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1200	} else {
1201	dev->current_state = state;
1202	}
1203	}
1204
1205	/**
1206	* pci_refresh_power_state - Refresh the given device's power state data
1207	* @dev: Target PCI device.
1208	*
1209	* Ask the platform to refresh the devices power state information and invoke
1210	* pci_update_current_state() to update its current PCI power state.
1211	*/
1212	void pci_refresh_power_state(struct pci_dev *dev)
1213	{
1214	platform_pci_refresh_power_state(dev);
1215	pci_update_current_state(dev, state: dev->current_state);
1216	}
1217
1218	/**
1219	* pci_platform_power_transition - Use platform to change device power state
1220	* @dev: PCI device to handle.
1221	* @state: State to put the device into.
1222	*/
1223	int pci_platform_power_transition(struct pci_dev *dev, pci_power_t state)
1224	{
1225	int error;
1226
1227	error = platform_pci_set_power_state(dev, t: state);
1228	if (!error)
1229	pci_update_current_state(dev, state);
1230	else if (!dev->pm_cap) /* Fall back to PCI_D0 */
1231	dev->current_state = PCI_D0;
1232
1233	return error;
1234	}
1235	EXPORT_SYMBOL_GPL(pci_platform_power_transition);
1236
1237	static int pci_resume_one(struct pci_dev *pci_dev, void *ign)
1238	{
1239	pm_request_resume(dev: &pci_dev->dev);
1240	return 0;
1241	}
1242
1243	/**
1244	* pci_resume_bus - Walk given bus and runtime resume devices on it
1245	* @bus: Top bus of the subtree to walk.
1246	*/
1247	void pci_resume_bus(struct pci_bus *bus)
1248	{
1249	if (bus)
1250	pci_walk_bus(top: bus, cb: pci_resume_one, NULL);
1251	}
1252
1253	static int pci_dev_wait(struct pci_dev *dev, char *reset_type, int timeout)
1254	{
1255	int delay = 1;
1256	bool retrain = false;
1257	struct pci_dev *bridge;
1258
1259	if (pci_is_pcie(dev)) {
1260	bridge = pci_upstream_bridge(dev);
1261	if (bridge)
1262	retrain = true;
1263	}
1264
1265	/*
1266	* After reset, the device should not silently discard config
1267	* requests, but it may still indicate that it needs more time by
1268	* responding to them with CRS completions. The Root Port will
1269	* generally synthesize ~0 (PCI_ERROR_RESPONSE) data to complete
1270	* the read (except when CRS SV is enabled and the read was for the
1271	* Vendor ID; in that case it synthesizes 0x0001 data).
1272	*
1273	* Wait for the device to return a non-CRS completion. Read the
1274	* Command register instead of Vendor ID so we don't have to
1275	* contend with the CRS SV value.
1276	*/
1277	for (;;) {
1278	u32 id;
1279
1280	pci_read_config_dword(dev, PCI_COMMAND, val: &id);
1281	if (!PCI_POSSIBLE_ERROR(id))
1282	break;
1283
1284	if (delay > timeout) {
1285	pci_warn(dev, "not ready %dms after %s; giving up\n",
1286	delay - 1, reset_type);
1287	return -ENOTTY;
1288	}
1289
1290	if (delay > PCI_RESET_WAIT) {
1291	if (retrain) {
1292	retrain = false;
1293	if (pcie_failed_link_retrain(dev: bridge)) {
1294	delay = 1;
1295	continue;
1296	}
1297	}
1298	pci_info(dev, "not ready %dms after %s; waiting\n",
1299	delay - 1, reset_type);
1300	}
1301
1302	msleep(msecs: delay);
1303	delay *= 2;
1304	}
1305
1306	if (delay > PCI_RESET_WAIT)
1307	pci_info(dev, "ready %dms after %s\n", delay - 1,
1308	reset_type);
1309	else
1310	pci_dbg(dev, "ready %dms after %s\n", delay - 1,
1311	reset_type);
1312
1313	return 0;
1314	}
1315
1316	/**
1317	* pci_power_up - Put the given device into D0
1318	* @dev: PCI device to power up
1319	*
1320	* On success, return 0 or 1, depending on whether or not it is necessary to
1321	* restore the device's BARs subsequently (1 is returned in that case).
1322	*
1323	* On failure, return a negative error code. Always return failure if @dev
1324	* lacks a Power Management Capability, even if the platform was able to
1325	* put the device in D0 via non-PCI means.
1326	*/
1327	int pci_power_up(struct pci_dev *dev)
1328	{
1329	bool need_restore;
1330	pci_power_t state;
1331	u16 pmcsr;
1332
1333	platform_pci_set_power_state(dev, PCI_D0);
1334
1335	if (!dev->pm_cap) {
1336	state = platform_pci_get_power_state(dev);
1337	if (state == PCI_UNKNOWN)
1338	dev->current_state = PCI_D0;
1339	else
1340	dev->current_state = state;
1341
1342	return -EIO;
1343	}
1344
1345	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
1346	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1347	pci_err(dev, "Unable to change power state from %s to D0, device inaccessible\n",
1348	pci_power_name(dev->current_state));
1349	dev->current_state = PCI_D3cold;
1350	return -EIO;
1351	}
1352
1353	state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1354
1355	need_restore = (state == PCI_D3hot || dev->current_state >= PCI_D3hot) &&
1356	!(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
1357
1358	if (state == PCI_D0)
1359	goto end;
1360
1361	/*
1362	* Force the entire word to 0. This doesn't affect PME_Status, disables
1363	* PME_En, and sets PowerState to 0.
1364	*/
1365	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: 0);
1366
1367	/* Mandatory transition delays; see PCI PM 1.2. */
1368	if (state == PCI_D3hot)
1369	pci_dev_d3_sleep(dev);
1370	else if (state == PCI_D2)
1371	udelay(PCI_PM_D2_DELAY);
1372
1373	end:
1374	dev->current_state = PCI_D0;
1375	if (need_restore)
1376	return 1;
1377
1378	return 0;
1379	}
1380
1381	/**
1382	* pci_set_full_power_state - Put a PCI device into D0 and update its state
1383	* @dev: PCI device to power up
1384	* @locked: whether pci_bus_sem is held
1385	*
1386	* Call pci_power_up() to put @dev into D0, read from its PCI_PM_CTRL register
1387	* to confirm the state change, restore its BARs if they might be lost and
1388	* reconfigure ASPM in accordance with the new power state.
1389	*
1390	* If pci_restore_state() is going to be called right after a power state change
1391	* to D0, it is more efficient to use pci_power_up() directly instead of this
1392	* function.
1393	*/
1394	static int pci_set_full_power_state(struct pci_dev *dev, bool locked)
1395	{
1396	u16 pmcsr;
1397	int ret;
1398
1399	ret = pci_power_up(dev);
1400	if (ret < 0) {
1401	if (dev->current_state == PCI_D0)
1402	return 0;
1403
1404	return ret;
1405	}
1406
1407	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
1408	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1409	if (dev->current_state != PCI_D0) {
1410	pci_info_ratelimited(dev, "Refused to change power state from %s to D0\n",
1411	pci_power_name(dev->current_state));
1412	} else if (ret > 0) {
1413	/*
1414	* According to section 5.4.1 of the "PCI BUS POWER MANAGEMENT
1415	* INTERFACE SPECIFICATION, REV. 1.2", a device transitioning
1416	* from D3hot to D0 _may_ perform an internal reset, thereby
1417	* going to "D0 Uninitialized" rather than "D0 Initialized".
1418	* For example, at least some versions of the 3c905B and the
1419	* 3c556B exhibit this behaviour.
1420	*
1421	* At least some laptop BIOSen (e.g. the Thinkpad T21) leave
1422	* devices in a D3hot state at boot. Consequently, we need to
1423	* restore at least the BARs so that the device will be
1424	* accessible to its driver.
1425	*/
1426	pci_restore_bars(dev);
1427	}
1428
1429	if (dev->bus->self)
1430	pcie_aspm_pm_state_change(pdev: dev->bus->self, locked);
1431
1432	return 0;
1433	}
1434
1435	/**
1436	* __pci_dev_set_current_state - Set current state of a PCI device
1437	* @dev: Device to handle
1438	* @data: pointer to state to be set
1439	*/
1440	static int __pci_dev_set_current_state(struct pci_dev *dev, void *data)
1441	{
1442	pci_power_t state = *(pci_power_t *)data;
1443
1444	dev->current_state = state;
1445	return 0;
1446	}
1447
1448	/**
1449	* pci_bus_set_current_state - Walk given bus and set current state of devices
1450	* @bus: Top bus of the subtree to walk.
1451	* @state: state to be set
1452	*/
1453	void pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state)
1454	{
1455	if (bus)
1456	pci_walk_bus(top: bus, cb: __pci_dev_set_current_state, userdata: &state);
1457	}
1458
1459	static void __pci_bus_set_current_state(struct pci_bus *bus, pci_power_t state, bool locked)
1460	{
1461	if (!bus)
1462	return;
1463
1464	if (locked)
1465	pci_walk_bus_locked(top: bus, cb: __pci_dev_set_current_state, userdata: &state);
1466	else
1467	pci_walk_bus(top: bus, cb: __pci_dev_set_current_state, userdata: &state);
1468	}
1469
1470	/**
1471	* pci_set_low_power_state - Put a PCI device into a low-power state.
1472	* @dev: PCI device to handle.
1473	* @state: PCI power state (D1, D2, D3hot) to put the device into.
1474	* @locked: whether pci_bus_sem is held
1475	*
1476	* Use the device's PCI_PM_CTRL register to put it into a low-power state.
1477	*
1478	* RETURN VALUE:
1479	* -EINVAL if the requested state is invalid.
1480	* -EIO if device does not support PCI PM or its PM capabilities register has a
1481	* wrong version, or device doesn't support the requested state.
1482	* 0 if device already is in the requested state.
1483	* 0 if device's power state has been successfully changed.
1484	*/
1485	static int pci_set_low_power_state(struct pci_dev *dev, pci_power_t state, bool locked)
1486	{
1487	u16 pmcsr;
1488
1489	if (!dev->pm_cap)
1490	return -EIO;
1491
1492	/*
1493	* Validate transition: We can enter D0 from any state, but if
1494	* we're already in a low-power state, we can only go deeper. E.g.,
1495	* we can go from D1 to D3, but we can't go directly from D3 to D1;
1496	* we'd have to go from D3 to D0, then to D1.
1497	*/
1498	if (dev->current_state <= PCI_D3cold && dev->current_state > state) {
1499	pci_dbg(dev, "Invalid power transition (from %s to %s)\n",
1500	pci_power_name(dev->current_state),
1501	pci_power_name(state));
1502	return -EINVAL;
1503	}
1504
1505	/* Check if this device supports the desired state */
1506	if ((state == PCI_D1 && !dev->d1_support)
1507	|| (state == PCI_D2 && !dev->d2_support))
1508	return -EIO;
1509
1510	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
1511	if (PCI_POSSIBLE_ERROR(pmcsr)) {
1512	pci_err(dev, "Unable to change power state from %s to %s, device inaccessible\n",
1513	pci_power_name(dev->current_state),
1514	pci_power_name(state));
1515	dev->current_state = PCI_D3cold;
1516	return -EIO;
1517	}
1518
1519	pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
1520	pmcsr |= state;
1521
1522	/* Enter specified state */
1523	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: pmcsr);
1524
1525	/* Mandatory power management transition delays; see PCI PM 1.2. */
1526	if (state == PCI_D3hot)
1527	pci_dev_d3_sleep(dev);
1528	else if (state == PCI_D2)
1529	udelay(PCI_PM_D2_DELAY);
1530
1531	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
1532	dev->current_state = pmcsr & PCI_PM_CTRL_STATE_MASK;
1533	if (dev->current_state != state)
1534	pci_info_ratelimited(dev, "Refused to change power state from %s to %s\n",
1535	pci_power_name(dev->current_state),
1536	pci_power_name(state));
1537
1538	if (dev->bus->self)
1539	pcie_aspm_pm_state_change(pdev: dev->bus->self, locked);
1540
1541	return 0;
1542	}
1543
1544	static int __pci_set_power_state(struct pci_dev *dev, pci_power_t state, bool locked)
1545	{
1546	int error;
1547
1548	/* Bound the state we're entering */
1549	if (state > PCI_D3cold)
1550	state = PCI_D3cold;
1551	else if (state < PCI_D0)
1552	state = PCI_D0;
1553	else if ((state == PCI_D1 || state == PCI_D2) && pci_no_d1d2(dev))
1554
1555	/*
1556	* If the device or the parent bridge do not support PCI
1557	* PM, ignore the request if we're doing anything other
1558	* than putting it into D0 (which would only happen on
1559	* boot).
1560	*/
1561	return 0;
1562
1563	/* Check if we're already there */
1564	if (dev->current_state == state)
1565	return 0;
1566
1567	if (state == PCI_D0)
1568	return pci_set_full_power_state(dev, locked);
1569
1570	/*
1571	* This device is quirked not to be put into D3, so don't put it in
1572	* D3
1573	*/
1574	if (state >= PCI_D3hot && (dev->dev_flags & PCI_DEV_FLAGS_NO_D3))
1575	return 0;
1576
1577	if (state == PCI_D3cold) {
1578	/*
1579	* To put the device in D3cold, put it into D3hot in the native
1580	* way, then put it into D3cold using platform ops.
1581	*/
1582	error = pci_set_low_power_state(dev, PCI_D3hot, locked);
1583
1584	if (pci_platform_power_transition(dev, PCI_D3cold))
1585	return error;
1586
1587	/* Powering off a bridge may power off the whole hierarchy */
1588	if (dev->current_state == PCI_D3cold)
1589	__pci_bus_set_current_state(bus: dev->subordinate, PCI_D3cold, locked);
1590	} else {
1591	error = pci_set_low_power_state(dev, state, locked);
1592
1593	if (pci_platform_power_transition(dev, state))
1594	return error;
1595	}
1596
1597	return 0;
1598	}
1599
1600	/**
1601	* pci_set_power_state - Set the power state of a PCI device
1602	* @dev: PCI device to handle.
1603	* @state: PCI power state (D0, D1, D2, D3hot) to put the device into.
1604	*
1605	* Transition a device to a new power state, using the platform firmware and/or
1606	* the device's PCI PM registers.
1607	*
1608	* RETURN VALUE:
1609	* -EINVAL if the requested state is invalid.
1610	* -EIO if device does not support PCI PM or its PM capabilities register has a
1611	* wrong version, or device doesn't support the requested state.
1612	* 0 if the transition is to D1 or D2 but D1 and D2 are not supported.
1613	* 0 if device already is in the requested state.
1614	* 0 if the transition is to D3 but D3 is not supported.
1615	* 0 if device's power state has been successfully changed.
1616	*/
1617	int pci_set_power_state(struct pci_dev *dev, pci_power_t state)
1618	{
1619	return __pci_set_power_state(dev, state, locked: false);
1620	}
1621	EXPORT_SYMBOL(pci_set_power_state);
1622
1623	int pci_set_power_state_locked(struct pci_dev *dev, pci_power_t state)
1624	{
1625	lockdep_assert_held(&pci_bus_sem);
1626
1627	return __pci_set_power_state(dev, state, locked: true);
1628	}
1629	EXPORT_SYMBOL(pci_set_power_state_locked);
1630
1631	#define PCI_EXP_SAVE_REGS 7
1632
1633	static struct pci_cap_saved_state *_pci_find_saved_cap(struct pci_dev *pci_dev,
1634	u16 cap, bool extended)
1635	{
1636	struct pci_cap_saved_state *tmp;
1637
1638	hlist_for_each_entry(tmp, &pci_dev->saved_cap_space, next) {
1639	if (tmp->cap.cap_extended == extended && tmp->cap.cap_nr == cap)
1640	return tmp;
1641	}
1642	return NULL;
1643	}
1644
1645	struct pci_cap_saved_state *pci_find_saved_cap(struct pci_dev *dev, char cap)
1646	{
1647	return _pci_find_saved_cap(pci_dev: dev, cap, extended: false);
1648	}
1649
1650	struct pci_cap_saved_state *pci_find_saved_ext_cap(struct pci_dev *dev, u16 cap)
1651	{
1652	return _pci_find_saved_cap(pci_dev: dev, cap, extended: true);
1653	}
1654
1655	static int pci_save_pcie_state(struct pci_dev *dev)
1656	{
1657	int i = 0;
1658	struct pci_cap_saved_state *save_state;
1659	u16 *cap;
1660
1661	if (!pci_is_pcie(dev))
1662	return 0;
1663
1664	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1665	if (!save_state) {
1666	pci_err(dev, "buffer not found in %s\n", __func__);
1667	return -ENOMEM;
1668	}
1669
1670	cap = (u16 *)&save_state->cap.data[0];
1671	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &cap[i++]);
1672	pcie_capability_read_word(dev, PCI_EXP_LNKCTL, val: &cap[i++]);
1673	pcie_capability_read_word(dev, PCI_EXP_SLTCTL, val: &cap[i++]);
1674	pcie_capability_read_word(dev, PCI_EXP_RTCTL, val: &cap[i++]);
1675	pcie_capability_read_word(dev, PCI_EXP_DEVCTL2, val: &cap[i++]);
1676	pcie_capability_read_word(dev, PCI_EXP_LNKCTL2, val: &cap[i++]);
1677	pcie_capability_read_word(dev, PCI_EXP_SLTCTL2, val: &cap[i++]);
1678
1679	pci_save_aspm_l1ss_state(dev);
1680	pci_save_ltr_state(dev);
1681
1682	return 0;
1683	}
1684
1685	static void pci_restore_pcie_state(struct pci_dev *dev)
1686	{
1687	int i = 0;
1688	struct pci_cap_saved_state *save_state;
1689	u16 *cap;
1690
1691	/*
1692	* Restore max latencies (in the LTR capability) before enabling
1693	* LTR itself in PCI_EXP_DEVCTL2.
1694	*/
1695	pci_restore_ltr_state(dev);
1696	pci_restore_aspm_l1ss_state(dev);
1697
1698	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
1699	if (!save_state)
1700	return;
1701
1702	/*
1703	* Downstream ports reset the LTR enable bit when link goes down.
1704	* Check and re-configure the bit here before restoring device.
1705	* PCIe r5.0, sec 7.5.3.16.
1706	*/
1707	pci_bridge_reconfigure_ltr(pdev: dev);
1708
1709	cap = (u16 *)&save_state->cap.data[0];
1710	pcie_capability_write_word(dev, PCI_EXP_DEVCTL, val: cap[i++]);
1711	pcie_capability_write_word(dev, PCI_EXP_LNKCTL, val: cap[i++]);
1712	pcie_capability_write_word(dev, PCI_EXP_SLTCTL, val: cap[i++]);
1713	pcie_capability_write_word(dev, PCI_EXP_RTCTL, val: cap[i++]);
1714	pcie_capability_write_word(dev, PCI_EXP_DEVCTL2, val: cap[i++]);
1715	pcie_capability_write_word(dev, PCI_EXP_LNKCTL2, val: cap[i++]);
1716	pcie_capability_write_word(dev, PCI_EXP_SLTCTL2, val: cap[i++]);
1717	}
1718
1719	static int pci_save_pcix_state(struct pci_dev *dev)
1720	{
1721	int pos;
1722	struct pci_cap_saved_state *save_state;
1723
1724	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1725	if (!pos)
1726	return 0;
1727
1728	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1729	if (!save_state) {
1730	pci_err(dev, "buffer not found in %s\n", __func__);
1731	return -ENOMEM;
1732	}
1733
1734	pci_read_config_word(dev, where: pos + PCI_X_CMD,
1735	val: (u16 *)save_state->cap.data);
1736
1737	return 0;
1738	}
1739
1740	static void pci_restore_pcix_state(struct pci_dev *dev)
1741	{
1742	int i = 0, pos;
1743	struct pci_cap_saved_state *save_state;
1744	u16 *cap;
1745
1746	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_PCIX);
1747	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1748	if (!save_state || !pos)
1749	return;
1750	cap = (u16 *)&save_state->cap.data[0];
1751
1752	pci_write_config_word(dev, where: pos + PCI_X_CMD, val: cap[i++]);
1753	}
1754
1755	/**
1756	* pci_save_state - save the PCI configuration space of a device before
1757	* suspending
1758	* @dev: PCI device that we're dealing with
1759	*/
1760	int pci_save_state(struct pci_dev *dev)
1761	{
1762	int i;
1763	/* XXX: 100% dword access ok here? */
1764	for (i = 0; i < 16; i++) {
1765	pci_read_config_dword(dev, where: i * 4, val: &dev->saved_config_space[i]);
1766	pci_dbg(dev, "save config %#04x: %#010x\n",
1767	i * 4, dev->saved_config_space[i]);
1768	}
1769	dev->state_saved = true;
1770
1771	i = pci_save_pcie_state(dev);
1772	if (i != 0)
1773	return i;
1774
1775	i = pci_save_pcix_state(dev);
1776	if (i != 0)
1777	return i;
1778
1779	pci_save_dpc_state(dev);
1780	pci_save_aer_state(dev);
1781	pci_save_ptm_state(dev);
1782	return pci_save_vc_state(dev);
1783	}
1784	EXPORT_SYMBOL(pci_save_state);
1785
1786	static void pci_restore_config_dword(struct pci_dev *pdev, int offset,
1787	u32 saved_val, int retry, bool force)
1788	{
1789	u32 val;
1790
1791	pci_read_config_dword(dev: pdev, where: offset, val: &val);
1792	if (!force && val == saved_val)
1793	return;
1794
1795	for (;;) {
1796	pci_dbg(pdev, "restore config %#04x: %#010x -> %#010x\n",
1797	offset, val, saved_val);
1798	pci_write_config_dword(dev: pdev, where: offset, val: saved_val);
1799	if (retry-- <= 0)
1800	return;
1801
1802	pci_read_config_dword(dev: pdev, where: offset, val: &val);
1803	if (val == saved_val)
1804	return;
1805
1806	mdelay(1);
1807	}
1808	}
1809
1810	static void pci_restore_config_space_range(struct pci_dev *pdev,
1811	int start, int end, int retry,
1812	bool force)
1813	{
1814	int index;
1815
1816	for (index = end; index >= start; index--)
1817	pci_restore_config_dword(pdev, offset: 4 * index,
1818	saved_val: pdev->saved_config_space[index],
1819	retry, force);
1820	}
1821
1822	static void pci_restore_config_space(struct pci_dev *pdev)
1823	{
1824	if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL) {
1825	pci_restore_config_space_range(pdev, start: 10, end: 15, retry: 0, force: false);
1826	/* Restore BARs before the command register. */
1827	pci_restore_config_space_range(pdev, start: 4, end: 9, retry: 10, force: false);
1828	pci_restore_config_space_range(pdev, start: 0, end: 3, retry: 0, force: false);
1829	} else if (pdev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
1830	pci_restore_config_space_range(pdev, start: 12, end: 15, retry: 0, force: false);
1831
1832	/*
1833	* Force rewriting of prefetch registers to avoid S3 resume
1834	* issues on Intel PCI bridges that occur when these
1835	* registers are not explicitly written.
1836	*/
1837	pci_restore_config_space_range(pdev, start: 9, end: 11, retry: 0, force: true);
1838	pci_restore_config_space_range(pdev, start: 0, end: 8, retry: 0, force: false);
1839	} else {
1840	pci_restore_config_space_range(pdev, start: 0, end: 15, retry: 0, force: false);
1841	}
1842	}
1843
1844	static void pci_restore_rebar_state(struct pci_dev *pdev)
1845	{
1846	unsigned int pos, nbars, i;
1847	u32 ctrl;
1848
1849	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
1850	if (!pos)
1851	return;
1852
1853	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
1854	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
1855
1856	for (i = 0; i < nbars; i++, pos += 8) {
1857	struct resource *res;
1858	int bar_idx, size;
1859
1860	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
1861	bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
1862	res = pdev->resource + bar_idx;
1863	size = pci_rebar_bytes_to_size(bytes: resource_size(res));
1864	ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
1865	ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
1866	pci_write_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: ctrl);
1867	}
1868	}
1869
1870	/**
1871	* pci_restore_state - Restore the saved state of a PCI device
1872	* @dev: PCI device that we're dealing with
1873	*/
1874	void pci_restore_state(struct pci_dev *dev)
1875	{
1876	if (!dev->state_saved)
1877	return;
1878
1879	pci_restore_pcie_state(dev);
1880	pci_restore_pasid_state(pdev: dev);
1881	pci_restore_pri_state(pdev: dev);
1882	pci_restore_ats_state(dev);
1883	pci_restore_vc_state(dev);
1884	pci_restore_rebar_state(pdev: dev);
1885	pci_restore_dpc_state(dev);
1886	pci_restore_ptm_state(dev);
1887
1888	pci_aer_clear_status(dev);
1889	pci_restore_aer_state(dev);
1890
1891	pci_restore_config_space(pdev: dev);
1892
1893	pci_restore_pcix_state(dev);
1894	pci_restore_msi_state(dev);
1895
1896	/* Restore ACS and IOV configuration state */
1897	pci_enable_acs(dev);
1898	pci_restore_iov_state(dev);
1899
1900	dev->state_saved = false;
1901	}
1902	EXPORT_SYMBOL(pci_restore_state);
1903
1904	struct pci_saved_state {
1905	u32 config_space[16];
1906	struct pci_cap_saved_data cap[];
1907	};
1908
1909	/**
1910	* pci_store_saved_state - Allocate and return an opaque struct containing
1911	* the device saved state.
1912	* @dev: PCI device that we're dealing with
1913	*
1914	* Return NULL if no state or error.
1915	*/
1916	struct pci_saved_state *pci_store_saved_state(struct pci_dev *dev)
1917	{
1918	struct pci_saved_state *state;
1919	struct pci_cap_saved_state *tmp;
1920	struct pci_cap_saved_data *cap;
1921	size_t size;
1922
1923	if (!dev->state_saved)
1924	return NULL;
1925
1926	size = sizeof(*state) + sizeof(struct pci_cap_saved_data);
1927
1928	hlist_for_each_entry(tmp, &dev->saved_cap_space, next)
1929	size += sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1930
1931	state = kzalloc(size, GFP_KERNEL);
1932	if (!state)
1933	return NULL;
1934
1935	memcpy(state->config_space, dev->saved_config_space,
1936	sizeof(state->config_space));
1937
1938	cap = state->cap;
1939	hlist_for_each_entry(tmp, &dev->saved_cap_space, next) {
1940	size_t len = sizeof(struct pci_cap_saved_data) + tmp->cap.size;
1941	memcpy(cap, &tmp->cap, len);
1942	cap = (struct pci_cap_saved_data *)((u8 *)cap + len);
1943	}
1944	/* Empty cap_save terminates list */
1945
1946	return state;
1947	}
1948	EXPORT_SYMBOL_GPL(pci_store_saved_state);
1949
1950	/**
1951	* pci_load_saved_state - Reload the provided save state into struct pci_dev.
1952	* @dev: PCI device that we're dealing with
1953	* @state: Saved state returned from pci_store_saved_state()
1954	*/
1955	int pci_load_saved_state(struct pci_dev *dev,
1956	struct pci_saved_state *state)
1957	{
1958	struct pci_cap_saved_data *cap;
1959
1960	dev->state_saved = false;
1961
1962	if (!state)
1963	return 0;
1964
1965	memcpy(dev->saved_config_space, state->config_space,
1966	sizeof(state->config_space));
1967
1968	cap = state->cap;
1969	while (cap->size) {
1970	struct pci_cap_saved_state *tmp;
1971
1972	tmp = _pci_find_saved_cap(pci_dev: dev, cap: cap->cap_nr, extended: cap->cap_extended);
1973	if (!tmp || tmp->cap.size != cap->size)
1974	return -EINVAL;
1975
1976	memcpy(tmp->cap.data, cap->data, tmp->cap.size);
1977	cap = (struct pci_cap_saved_data *)((u8 *)cap +
1978	sizeof(struct pci_cap_saved_data) + cap->size);
1979	}
1980
1981	dev->state_saved = true;
1982	return 0;
1983	}
1984	EXPORT_SYMBOL_GPL(pci_load_saved_state);
1985
1986	/**
1987	* pci_load_and_free_saved_state - Reload the save state pointed to by state,
1988	* and free the memory allocated for it.
1989	* @dev: PCI device that we're dealing with
1990	* @state: Pointer to saved state returned from pci_store_saved_state()
1991	*/
1992	int pci_load_and_free_saved_state(struct pci_dev *dev,
1993	struct pci_saved_state **state)
1994	{
1995	int ret = pci_load_saved_state(dev, *state);
1996	kfree(objp: *state);
1997	*state = NULL;
1998	return ret;
1999	}
2000	EXPORT_SYMBOL_GPL(pci_load_and_free_saved_state);
2001
2002	int __weak pcibios_enable_device(struct pci_dev *dev, int bars)
2003	{
2004	return pci_enable_resources(dev, mask: bars);
2005	}
2006
2007	static int do_pci_enable_device(struct pci_dev *dev, int bars)
2008	{
2009	int err;
2010	struct pci_dev *bridge;
2011	u16 cmd;
2012	u8 pin;
2013
2014	err = pci_set_power_state(dev, PCI_D0);
2015	if (err < 0 && err != -EIO)
2016	return err;
2017
2018	bridge = pci_upstream_bridge(dev);
2019	if (bridge)
2020	pcie_aspm_powersave_config_link(pdev: bridge);
2021
2022	err = pcibios_enable_device(dev, bars);
2023	if (err < 0)
2024	return err;
2025	pci_fixup_device(pass: pci_fixup_enable, dev);
2026
2027	if (dev->msi_enabled || dev->msix_enabled)
2028	return 0;
2029
2030	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, val: &pin);
2031	if (pin) {
2032	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
2033	if (cmd & PCI_COMMAND_INTX_DISABLE)
2034	pci_write_config_word(dev, PCI_COMMAND,
2035	val: cmd & ~PCI_COMMAND_INTX_DISABLE);
2036	}
2037
2038	return 0;
2039	}
2040
2041	/**
2042	* pci_reenable_device - Resume abandoned device
2043	* @dev: PCI device to be resumed
2044	*
2045	* NOTE: This function is a backend of pci_default_resume() and is not supposed
2046	* to be called by normal code, write proper resume handler and use it instead.
2047	*/
2048	int pci_reenable_device(struct pci_dev *dev)
2049	{
2050	if (pci_is_enabled(pdev: dev))
2051	return do_pci_enable_device(dev, bars: (1 << PCI_NUM_RESOURCES) - 1);
2052	return 0;
2053	}
2054	EXPORT_SYMBOL(pci_reenable_device);
2055
2056	static void pci_enable_bridge(struct pci_dev *dev)
2057	{
2058	struct pci_dev *bridge;
2059	int retval;
2060
2061	bridge = pci_upstream_bridge(dev);
2062	if (bridge)
2063	pci_enable_bridge(dev: bridge);
2064
2065	if (pci_is_enabled(pdev: dev)) {
2066	if (!dev->is_busmaster)
2067	pci_set_master(dev);
2068	return;
2069	}
2070
2071	retval = pci_enable_device(dev);
2072	if (retval)
2073	pci_err(dev, "Error enabling bridge (%d), continuing\n",
2074	retval);
2075	pci_set_master(dev);
2076	}
2077
2078	static int pci_enable_device_flags(struct pci_dev *dev, unsigned long flags)
2079	{
2080	struct pci_dev *bridge;
2081	int err;
2082	int i, bars = 0;
2083
2084	/*
2085	* Power state could be unknown at this point, either due to a fresh
2086	* boot or a device removal call. So get the current power state
2087	* so that things like MSI message writing will behave as expected
2088	* (e.g. if the device really is in D0 at enable time).
2089	*/
2090	pci_update_current_state(dev, state: dev->current_state);
2091
2092	if (atomic_inc_return(v: &dev->enable_cnt) > 1)
2093	return 0; /* already enabled */
2094
2095	bridge = pci_upstream_bridge(dev);
2096	if (bridge)
2097	pci_enable_bridge(dev: bridge);
2098
2099	/* only skip sriov related */
2100	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
2101	if (dev->resource[i].flags & flags)
2102	bars |= (1 << i);
2103	for (i = PCI_BRIDGE_RESOURCES; i < DEVICE_COUNT_RESOURCE; i++)
2104	if (dev->resource[i].flags & flags)
2105	bars |= (1 << i);
2106
2107	err = do_pci_enable_device(dev, bars);
2108	if (err < 0)
2109	atomic_dec(v: &dev->enable_cnt);
2110	return err;
2111	}
2112
2113	/**
2114	* pci_enable_device_io - Initialize a device for use with IO space
2115	* @dev: PCI device to be initialized
2116	*
2117	* Initialize device before it's used by a driver. Ask low-level code
2118	* to enable I/O resources. Wake up the device if it was suspended.
2119	* Beware, this function can fail.
2120	*/
2121	int pci_enable_device_io(struct pci_dev *dev)
2122	{
2123	return pci_enable_device_flags(dev, IORESOURCE_IO);
2124	}
2125	EXPORT_SYMBOL(pci_enable_device_io);
2126
2127	/**
2128	* pci_enable_device_mem - Initialize a device for use with Memory space
2129	* @dev: PCI device to be initialized
2130	*
2131	* Initialize device before it's used by a driver. Ask low-level code
2132	* to enable Memory resources. Wake up the device if it was suspended.
2133	* Beware, this function can fail.
2134	*/
2135	int pci_enable_device_mem(struct pci_dev *dev)
2136	{
2137	return pci_enable_device_flags(dev, IORESOURCE_MEM);
2138	}
2139	EXPORT_SYMBOL(pci_enable_device_mem);
2140
2141	/**
2142	* pci_enable_device - Initialize device before it's used by a driver.
2143	* @dev: PCI device to be initialized
2144	*
2145	* Initialize device before it's used by a driver. Ask low-level code
2146	* to enable I/O and memory. Wake up the device if it was suspended.
2147	* Beware, this function can fail.
2148	*
2149	* Note we don't actually enable the device many times if we call
2150	* this function repeatedly (we just increment the count).
2151	*/
2152	int pci_enable_device(struct pci_dev *dev)
2153	{
2154	return pci_enable_device_flags(dev, IORESOURCE_MEM | IORESOURCE_IO);
2155	}
2156	EXPORT_SYMBOL(pci_enable_device);
2157
2158	/*
2159	* pcibios_device_add - provide arch specific hooks when adding device dev
2160	* @dev: the PCI device being added
2161	*
2162	* Permits the platform to provide architecture specific functionality when
2163	* devices are added. This is the default implementation. Architecture
2164	* implementations can override this.
2165	*/
2166	int __weak pcibios_device_add(struct pci_dev *dev)
2167	{
2168	return 0;
2169	}
2170
2171	/**
2172	* pcibios_release_device - provide arch specific hooks when releasing
2173	* device dev
2174	* @dev: the PCI device being released
2175	*
2176	* Permits the platform to provide architecture specific functionality when
2177	* devices are released. This is the default implementation. Architecture
2178	* implementations can override this.
2179	*/
2180	void __weak pcibios_release_device(struct pci_dev *dev) {}
2181
2182	/**
2183	* pcibios_disable_device - disable arch specific PCI resources for device dev
2184	* @dev: the PCI device to disable
2185	*
2186	* Disables architecture specific PCI resources for the device. This
2187	* is the default implementation. Architecture implementations can
2188	* override this.
2189	*/
2190	void __weak pcibios_disable_device(struct pci_dev *dev) {}
2191
2192	static void do_pci_disable_device(struct pci_dev *dev)
2193	{
2194	u16 pci_command;
2195
2196	pci_read_config_word(dev, PCI_COMMAND, val: &pci_command);
2197	if (pci_command & PCI_COMMAND_MASTER) {
2198	pci_command &= ~PCI_COMMAND_MASTER;
2199	pci_write_config_word(dev, PCI_COMMAND, val: pci_command);
2200	}
2201
2202	pcibios_disable_device(dev);
2203	}
2204
2205	/**
2206	* pci_disable_enabled_device - Disable device without updating enable_cnt
2207	* @dev: PCI device to disable
2208	*
2209	* NOTE: This function is a backend of PCI power management routines and is
2210	* not supposed to be called drivers.
2211	*/
2212	void pci_disable_enabled_device(struct pci_dev *dev)
2213	{
2214	if (pci_is_enabled(pdev: dev))
2215	do_pci_disable_device(dev);
2216	}
2217
2218	/**
2219	* pci_disable_device - Disable PCI device after use
2220	* @dev: PCI device to be disabled
2221	*
2222	* Signal to the system that the PCI device is not in use by the system
2223	* anymore. This only involves disabling PCI bus-mastering, if active.
2224	*
2225	* Note we don't actually disable the device until all callers of
2226	* pci_enable_device() have called pci_disable_device().
2227	*/
2228	void pci_disable_device(struct pci_dev *dev)
2229	{
2230	struct pci_devres *dr;
2231
2232	dr = find_pci_dr(pdev: dev);
2233	if (dr)
2234	dr->enabled = 0;
2235
2236	dev_WARN_ONCE(&dev->dev, atomic_read(&dev->enable_cnt) <= 0,
2237	"disabling already-disabled device");
2238
2239	if (atomic_dec_return(v: &dev->enable_cnt) != 0)
2240	return;
2241
2242	do_pci_disable_device(dev);
2243
2244	dev->is_busmaster = 0;
2245	}
2246	EXPORT_SYMBOL(pci_disable_device);
2247
2248	/**
2249	* pcibios_set_pcie_reset_state - set reset state for device dev
2250	* @dev: the PCIe device reset
2251	* @state: Reset state to enter into
2252	*
2253	* Set the PCIe reset state for the device. This is the default
2254	* implementation. Architecture implementations can override this.
2255	*/
2256	int __weak pcibios_set_pcie_reset_state(struct pci_dev *dev,
2257	enum pcie_reset_state state)
2258	{
2259	return -EINVAL;
2260	}
2261
2262	/**
2263	* pci_set_pcie_reset_state - set reset state for device dev
2264	* @dev: the PCIe device reset
2265	* @state: Reset state to enter into
2266	*
2267	* Sets the PCI reset state for the device.
2268	*/
2269	int pci_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state)
2270	{
2271	return pcibios_set_pcie_reset_state(dev, state);
2272	}
2273	EXPORT_SYMBOL_GPL(pci_set_pcie_reset_state);
2274
2275	#ifdef CONFIG_PCIEAER
2276	void pcie_clear_device_status(struct pci_dev *dev)
2277	{
2278	u16 sta;
2279
2280	pcie_capability_read_word(dev, PCI_EXP_DEVSTA, val: &sta);
2281	pcie_capability_write_word(dev, PCI_EXP_DEVSTA, val: sta);
2282	}
2283	#endif
2284
2285	/**
2286	* pcie_clear_root_pme_status - Clear root port PME interrupt status.
2287	* @dev: PCIe root port or event collector.
2288	*/
2289	void pcie_clear_root_pme_status(struct pci_dev *dev)
2290	{
2291	pcie_capability_set_dword(dev, PCI_EXP_RTSTA, PCI_EXP_RTSTA_PME);
2292	}
2293
2294	/**
2295	* pci_check_pme_status - Check if given device has generated PME.
2296	* @dev: Device to check.
2297	*
2298	* Check the PME status of the device and if set, clear it and clear PME enable
2299	* (if set). Return 'true' if PME status and PME enable were both set or
2300	* 'false' otherwise.
2301	*/
2302	bool pci_check_pme_status(struct pci_dev *dev)
2303	{
2304	int pmcsr_pos;
2305	u16 pmcsr;
2306	bool ret = false;
2307
2308	if (!dev->pm_cap)
2309	return false;
2310
2311	pmcsr_pos = dev->pm_cap + PCI_PM_CTRL;
2312	pci_read_config_word(dev, where: pmcsr_pos, val: &pmcsr);
2313	if (!(pmcsr & PCI_PM_CTRL_PME_STATUS))
2314	return false;
2315
2316	/* Clear PME status. */
2317	pmcsr |= PCI_PM_CTRL_PME_STATUS;
2318	if (pmcsr & PCI_PM_CTRL_PME_ENABLE) {
2319	/* Disable PME to avoid interrupt flood. */
2320	pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2321	ret = true;
2322	}
2323
2324	pci_write_config_word(dev, where: pmcsr_pos, val: pmcsr);
2325
2326	return ret;
2327	}
2328
2329	/**
2330	* pci_pme_wakeup - Wake up a PCI device if its PME Status bit is set.
2331	* @dev: Device to handle.
2332	* @pme_poll_reset: Whether or not to reset the device's pme_poll flag.
2333	*
2334	* Check if @dev has generated PME and queue a resume request for it in that
2335	* case.
2336	*/
2337	static int pci_pme_wakeup(struct pci_dev *dev, void *pme_poll_reset)
2338	{
2339	if (pme_poll_reset && dev->pme_poll)
2340	dev->pme_poll = false;
2341
2342	if (pci_check_pme_status(dev)) {
2343	pci_wakeup_event(dev);
2344	pm_request_resume(dev: &dev->dev);
2345	}
2346	return 0;
2347	}
2348
2349	/**
2350	* pci_pme_wakeup_bus - Walk given bus and wake up devices on it, if necessary.
2351	* @bus: Top bus of the subtree to walk.
2352	*/
2353	void pci_pme_wakeup_bus(struct pci_bus *bus)
2354	{
2355	if (bus)
2356	pci_walk_bus(top: bus, cb: pci_pme_wakeup, userdata: (void *)true);
2357	}
2358
2359
2360	/**
2361	* pci_pme_capable - check the capability of PCI device to generate PME#
2362	* @dev: PCI device to handle.
2363	* @state: PCI state from which device will issue PME#.
2364	*/
2365	bool pci_pme_capable(struct pci_dev *dev, pci_power_t state)
2366	{
2367	if (!dev->pm_cap)
2368	return false;
2369
2370	return !!(dev->pme_support & (1 << state));
2371	}
2372	EXPORT_SYMBOL(pci_pme_capable);
2373
2374	static void pci_pme_list_scan(struct work_struct *work)
2375	{
2376	struct pci_pme_device *pme_dev, *n;
2377
2378	mutex_lock(&pci_pme_list_mutex);
2379	list_for_each_entry_safe(pme_dev, n, &pci_pme_list, list) {
2380	struct pci_dev *pdev = pme_dev->dev;
2381
2382	if (pdev->pme_poll) {
2383	struct pci_dev *bridge = pdev->bus->self;
2384	struct device *dev = &pdev->dev;
2385	struct device *bdev = bridge ? &bridge->dev : NULL;
2386	int bref = 0;
2387
2388	/*
2389	* If we have a bridge, it should be in an active/D0
2390	* state or the configuration space of subordinate
2391	* devices may not be accessible or stable over the
2392	* course of the call.
2393	*/
2394	if (bdev) {
2395	bref = pm_runtime_get_if_active(dev: bdev);
2396	if (!bref)
2397	continue;
2398
2399	if (bridge->current_state != PCI_D0)
2400	goto put_bridge;
2401	}
2402
2403	/*
2404	* The device itself should be suspended but config
2405	* space must be accessible, therefore it cannot be in
2406	* D3cold.
2407	*/
2408	if (pm_runtime_suspended(dev) &&
2409	pdev->current_state != PCI_D3cold)
2410	pci_pme_wakeup(dev: pdev, NULL);
2411
2412	put_bridge:
2413	if (bref > 0)
2414	pm_runtime_put(dev: bdev);
2415	} else {
2416	list_del(entry: &pme_dev->list);
2417	kfree(objp: pme_dev);
2418	}
2419	}
2420	if (!list_empty(head: &pci_pme_list))
2421	queue_delayed_work(wq: system_freezable_wq, dwork: &pci_pme_work,
2422	delay: msecs_to_jiffies(PME_TIMEOUT));
2423	mutex_unlock(lock: &pci_pme_list_mutex);
2424	}
2425
2426	static void __pci_pme_active(struct pci_dev *dev, bool enable)
2427	{
2428	u16 pmcsr;
2429
2430	if (!dev->pme_support)
2431	return;
2432
2433	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
2434	/* Clear PME_Status by writing 1 to it and enable PME# */
2435	pmcsr |= PCI_PM_CTRL_PME_STATUS | PCI_PM_CTRL_PME_ENABLE;
2436	if (!enable)
2437	pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2438
2439	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: pmcsr);
2440	}
2441
2442	/**
2443	* pci_pme_restore - Restore PME configuration after config space restore.
2444	* @dev: PCI device to update.
2445	*/
2446	void pci_pme_restore(struct pci_dev *dev)
2447	{
2448	u16 pmcsr;
2449
2450	if (!dev->pme_support)
2451	return;
2452
2453	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &pmcsr);
2454	if (dev->wakeup_prepared) {
2455	pmcsr |= PCI_PM_CTRL_PME_ENABLE;
2456	pmcsr &= ~PCI_PM_CTRL_PME_STATUS;
2457	} else {
2458	pmcsr &= ~PCI_PM_CTRL_PME_ENABLE;
2459	pmcsr |= PCI_PM_CTRL_PME_STATUS;
2460	}
2461	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: pmcsr);
2462	}
2463
2464	/**
2465	* pci_pme_active - enable or disable PCI device's PME# function
2466	* @dev: PCI device to handle.
2467	* @enable: 'true' to enable PME# generation; 'false' to disable it.
2468	*
2469	* The caller must verify that the device is capable of generating PME# before
2470	* calling this function with @enable equal to 'true'.
2471	*/
2472	void pci_pme_active(struct pci_dev *dev, bool enable)
2473	{
2474	__pci_pme_active(dev, enable);
2475
2476	/*
2477	* PCI (as opposed to PCIe) PME requires that the device have
2478	* its PME# line hooked up correctly. Not all hardware vendors
2479	* do this, so the PME never gets delivered and the device
2480	* remains asleep. The easiest way around this is to
2481	* periodically walk the list of suspended devices and check
2482	* whether any have their PME flag set. The assumption is that
2483	* we'll wake up often enough anyway that this won't be a huge
2484	* hit, and the power savings from the devices will still be a
2485	* win.
2486	*
2487	* Although PCIe uses in-band PME message instead of PME# line
2488	* to report PME, PME does not work for some PCIe devices in
2489	* reality. For example, there are devices that set their PME
2490	* status bits, but don't really bother to send a PME message;
2491	* there are PCI Express Root Ports that don't bother to
2492	* trigger interrupts when they receive PME messages from the
2493	* devices below. So PME poll is used for PCIe devices too.
2494	*/
2495
2496	if (dev->pme_poll) {
2497	struct pci_pme_device *pme_dev;
2498	if (enable) {
2499	pme_dev = kmalloc(size: sizeof(struct pci_pme_device),
2500	GFP_KERNEL);
2501	if (!pme_dev) {
2502	pci_warn(dev, "can't enable PME#\n");
2503	return;
2504	}
2505	pme_dev->dev = dev;
2506	mutex_lock(&pci_pme_list_mutex);
2507	list_add(new: &pme_dev->list, head: &pci_pme_list);
2508	if (list_is_singular(head: &pci_pme_list))
2509	queue_delayed_work(wq: system_freezable_wq,
2510	dwork: &pci_pme_work,
2511	delay: msecs_to_jiffies(PME_TIMEOUT));
2512	mutex_unlock(lock: &pci_pme_list_mutex);
2513	} else {
2514	mutex_lock(&pci_pme_list_mutex);
2515	list_for_each_entry(pme_dev, &pci_pme_list, list) {
2516	if (pme_dev->dev == dev) {
2517	list_del(entry: &pme_dev->list);
2518	kfree(objp: pme_dev);
2519	break;
2520	}
2521	}
2522	mutex_unlock(lock: &pci_pme_list_mutex);
2523	}
2524	}
2525
2526	pci_dbg(dev, "PME# %s\n", enable ? "enabled" : "disabled");
2527	}
2528	EXPORT_SYMBOL(pci_pme_active);
2529
2530	/**
2531	* __pci_enable_wake - enable PCI device as wakeup event source
2532	* @dev: PCI device affected
2533	* @state: PCI state from which device will issue wakeup events
2534	* @enable: True to enable event generation; false to disable
2535	*
2536	* This enables the device as a wakeup event source, or disables it.
2537	* When such events involves platform-specific hooks, those hooks are
2538	* called automatically by this routine.
2539	*
2540	* Devices with legacy power management (no standard PCI PM capabilities)
2541	* always require such platform hooks.
2542	*
2543	* RETURN VALUE:
2544	* 0 is returned on success
2545	* -EINVAL is returned if device is not supposed to wake up the system
2546	* Error code depending on the platform is returned if both the platform and
2547	* the native mechanism fail to enable the generation of wake-up events
2548	*/
2549	static int __pci_enable_wake(struct pci_dev *dev, pci_power_t state, bool enable)
2550	{
2551	int ret = 0;
2552
2553	/*
2554	* Bridges that are not power-manageable directly only signal
2555	* wakeup on behalf of subordinate devices which is set up
2556	* elsewhere, so skip them. However, bridges that are
2557	* power-manageable may signal wakeup for themselves (for example,
2558	* on a hotplug event) and they need to be covered here.
2559	*/
2560	if (!pci_power_manageable(pci_dev: dev))
2561	return 0;
2562
2563	/* Don't do the same thing twice in a row for one device. */
2564	if (!!enable == !!dev->wakeup_prepared)
2565	return 0;
2566
2567	/*
2568	* According to "PCI System Architecture" 4th ed. by Tom Shanley & Don
2569	* Anderson we should be doing PME# wake enable followed by ACPI wake
2570	* enable. To disable wake-up we call the platform first, for symmetry.
2571	*/
2572
2573	if (enable) {
2574	int error;
2575
2576	/*
2577	* Enable PME signaling if the device can signal PME from
2578	* D3cold regardless of whether or not it can signal PME from
2579	* the current target state, because that will allow it to
2580	* signal PME when the hierarchy above it goes into D3cold and
2581	* the device itself ends up in D3cold as a result of that.
2582	*/
2583	if (pci_pme_capable(dev, state) || pci_pme_capable(dev, PCI_D3cold))
2584	pci_pme_active(dev, true);
2585	else
2586	ret = 1;
2587	error = platform_pci_set_wakeup(dev, enable: true);
2588	if (ret)
2589	ret = error;
2590	if (!ret)
2591	dev->wakeup_prepared = true;
2592	} else {
2593	platform_pci_set_wakeup(dev, enable: false);
2594	pci_pme_active(dev, false);
2595	dev->wakeup_prepared = false;
2596	}
2597
2598	return ret;
2599	}
2600
2601	/**
2602	* pci_enable_wake - change wakeup settings for a PCI device
2603	* @pci_dev: Target device
2604	* @state: PCI state from which device will issue wakeup events
2605	* @enable: Whether or not to enable event generation
2606	*
2607	* If @enable is set, check device_may_wakeup() for the device before calling
2608	* __pci_enable_wake() for it.
2609	*/
2610	int pci_enable_wake(struct pci_dev *pci_dev, pci_power_t state, bool enable)
2611	{
2612	if (enable && !device_may_wakeup(dev: &pci_dev->dev))
2613	return -EINVAL;
2614
2615	return __pci_enable_wake(dev: pci_dev, state, enable);
2616	}
2617	EXPORT_SYMBOL(pci_enable_wake);
2618
2619	/**
2620	* pci_wake_from_d3 - enable/disable device to wake up from D3_hot or D3_cold
2621	* @dev: PCI device to prepare
2622	* @enable: True to enable wake-up event generation; false to disable
2623	*
2624	* Many drivers want the device to wake up the system from D3_hot or D3_cold
2625	* and this function allows them to set that up cleanly - pci_enable_wake()
2626	* should not be called twice in a row to enable wake-up due to PCI PM vs ACPI
2627	* ordering constraints.
2628	*
2629	* This function only returns error code if the device is not allowed to wake
2630	* up the system from sleep or it is not capable of generating PME# from both
2631	* D3_hot and D3_cold and the platform is unable to enable wake-up power for it.
2632	*/
2633	int pci_wake_from_d3(struct pci_dev *dev, bool enable)
2634	{
2635	return pci_pme_capable(dev, PCI_D3cold) ?
2636	pci_enable_wake(dev, PCI_D3cold, enable) :
2637	pci_enable_wake(dev, PCI_D3hot, enable);
2638	}
2639	EXPORT_SYMBOL(pci_wake_from_d3);
2640
2641	/**
2642	* pci_target_state - find an appropriate low power state for a given PCI dev
2643	* @dev: PCI device
2644	* @wakeup: Whether or not wakeup functionality will be enabled for the device.
2645	*
2646	* Use underlying platform code to find a supported low power state for @dev.
2647	* If the platform can't manage @dev, return the deepest state from which it
2648	* can generate wake events, based on any available PME info.
2649	*/
2650	static pci_power_t pci_target_state(struct pci_dev *dev, bool wakeup)
2651	{
2652	if (platform_pci_power_manageable(dev)) {
2653	/*
2654	* Call the platform to find the target state for the device.
2655	*/
2656	pci_power_t state = platform_pci_choose_state(dev);
2657
2658	switch (state) {
2659	case PCI_POWER_ERROR:
2660	case PCI_UNKNOWN:
2661	return PCI_D3hot;
2662
2663	case PCI_D1:
2664	case PCI_D2:
2665	if (pci_no_d1d2(dev))
2666	return PCI_D3hot;
2667	}
2668
2669	return state;
2670	}
2671
2672	/*
2673	* If the device is in D3cold even though it's not power-manageable by
2674	* the platform, it may have been powered down by non-standard means.
2675	* Best to let it slumber.
2676	*/
2677	if (dev->current_state == PCI_D3cold)
2678	return PCI_D3cold;
2679	else if (!dev->pm_cap)
2680	return PCI_D0;
2681
2682	if (wakeup && dev->pme_support) {
2683	pci_power_t state = PCI_D3hot;
2684
2685	/*
2686	* Find the deepest state from which the device can generate
2687	* PME#.
2688	*/
2689	while (state && !(dev->pme_support & (1 << state)))
2690	state--;
2691
2692	if (state)
2693	return state;
2694	else if (dev->pme_support & 1)
2695	return PCI_D0;
2696	}
2697
2698	return PCI_D3hot;
2699	}
2700
2701	/**
2702	* pci_prepare_to_sleep - prepare PCI device for system-wide transition
2703	* into a sleep state
2704	* @dev: Device to handle.
2705	*
2706	* Choose the power state appropriate for the device depending on whether
2707	* it can wake up the system and/or is power manageable by the platform
2708	* (PCI_D3hot is the default) and put the device into that state.
2709	*/
2710	int pci_prepare_to_sleep(struct pci_dev *dev)
2711	{
2712	bool wakeup = device_may_wakeup(dev: &dev->dev);
2713	pci_power_t target_state = pci_target_state(dev, wakeup);
2714	int error;
2715
2716	if (target_state == PCI_POWER_ERROR)
2717	return -EIO;
2718
2719	pci_enable_wake(dev, target_state, wakeup);
2720
2721	error = pci_set_power_state(dev, target_state);
2722
2723	if (error)
2724	pci_enable_wake(dev, target_state, false);
2725
2726	return error;
2727	}
2728	EXPORT_SYMBOL(pci_prepare_to_sleep);
2729
2730	/**
2731	* pci_back_from_sleep - turn PCI device on during system-wide transition
2732	* into working state
2733	* @dev: Device to handle.
2734	*
2735	* Disable device's system wake-up capability and put it into D0.
2736	*/
2737	int pci_back_from_sleep(struct pci_dev *dev)
2738	{
2739	int ret = pci_set_power_state(dev, PCI_D0);
2740
2741	if (ret)
2742	return ret;
2743
2744	pci_enable_wake(dev, PCI_D0, false);
2745	return 0;
2746	}
2747	EXPORT_SYMBOL(pci_back_from_sleep);
2748
2749	/**
2750	* pci_finish_runtime_suspend - Carry out PCI-specific part of runtime suspend.
2751	* @dev: PCI device being suspended.
2752	*
2753	* Prepare @dev to generate wake-up events at run time and put it into a low
2754	* power state.
2755	*/
2756	int pci_finish_runtime_suspend(struct pci_dev *dev)
2757	{
2758	pci_power_t target_state;
2759	int error;
2760
2761	target_state = pci_target_state(dev, wakeup: device_can_wakeup(dev: &dev->dev));
2762	if (target_state == PCI_POWER_ERROR)
2763	return -EIO;
2764
2765	__pci_enable_wake(dev, state: target_state, enable: pci_dev_run_wake(dev));
2766
2767	error = pci_set_power_state(dev, target_state);
2768
2769	if (error)
2770	pci_enable_wake(dev, target_state, false);
2771
2772	return error;
2773	}
2774
2775	/**
2776	* pci_dev_run_wake - Check if device can generate run-time wake-up events.
2777	* @dev: Device to check.
2778	*
2779	* Return true if the device itself is capable of generating wake-up events
2780	* (through the platform or using the native PCIe PME) or if the device supports
2781	* PME and one of its upstream bridges can generate wake-up events.
2782	*/
2783	bool pci_dev_run_wake(struct pci_dev *dev)
2784	{
2785	struct pci_bus *bus = dev->bus;
2786
2787	if (!dev->pme_support)
2788	return false;
2789
2790	/* PME-capable in principle, but not from the target power state */
2791	if (!pci_pme_capable(dev, pci_target_state(dev, wakeup: true)))
2792	return false;
2793
2794	if (device_can_wakeup(dev: &dev->dev))
2795	return true;
2796
2797	while (bus->parent) {
2798	struct pci_dev *bridge = bus->self;
2799
2800	if (device_can_wakeup(dev: &bridge->dev))
2801	return true;
2802
2803	bus = bus->parent;
2804	}
2805
2806	/* We have reached the root bus. */
2807	if (bus->bridge)
2808	return device_can_wakeup(dev: bus->bridge);
2809
2810	return false;
2811	}
2812	EXPORT_SYMBOL_GPL(pci_dev_run_wake);
2813
2814	/**
2815	* pci_dev_need_resume - Check if it is necessary to resume the device.
2816	* @pci_dev: Device to check.
2817	*
2818	* Return 'true' if the device is not runtime-suspended or it has to be
2819	* reconfigured due to wakeup settings difference between system and runtime
2820	* suspend, or the current power state of it is not suitable for the upcoming
2821	* (system-wide) transition.
2822	*/
2823	bool pci_dev_need_resume(struct pci_dev *pci_dev)
2824	{
2825	struct device *dev = &pci_dev->dev;
2826	pci_power_t target_state;
2827
2828	if (!pm_runtime_suspended(dev) || platform_pci_need_resume(dev: pci_dev))
2829	return true;
2830
2831	target_state = pci_target_state(dev: pci_dev, wakeup: device_may_wakeup(dev));
2832
2833	/*
2834	* If the earlier platform check has not triggered, D3cold is just power
2835	* removal on top of D3hot, so no need to resume the device in that
2836	* case.
2837	*/
2838	return target_state != pci_dev->current_state &&
2839	target_state != PCI_D3cold &&
2840	pci_dev->current_state != PCI_D3hot;
2841	}
2842
2843	/**
2844	* pci_dev_adjust_pme - Adjust PME setting for a suspended device.
2845	* @pci_dev: Device to check.
2846	*
2847	* If the device is suspended and it is not configured for system wakeup,
2848	* disable PME for it to prevent it from waking up the system unnecessarily.
2849	*
2850	* Note that if the device's power state is D3cold and the platform check in
2851	* pci_dev_need_resume() has not triggered, the device's configuration need not
2852	* be changed.
2853	*/
2854	void pci_dev_adjust_pme(struct pci_dev *pci_dev)
2855	{
2856	struct device *dev = &pci_dev->dev;
2857
2858	spin_lock_irq(lock: &dev->power.lock);
2859
2860	if (pm_runtime_suspended(dev) && !device_may_wakeup(dev) &&
2861	pci_dev->current_state < PCI_D3cold)
2862	__pci_pme_active(dev: pci_dev, enable: false);
2863
2864	spin_unlock_irq(lock: &dev->power.lock);
2865	}
2866
2867	/**
2868	* pci_dev_complete_resume - Finalize resume from system sleep for a device.
2869	* @pci_dev: Device to handle.
2870	*
2871	* If the device is runtime suspended and wakeup-capable, enable PME for it as
2872	* it might have been disabled during the prepare phase of system suspend if
2873	* the device was not configured for system wakeup.
2874	*/
2875	void pci_dev_complete_resume(struct pci_dev *pci_dev)
2876	{
2877	struct device *dev = &pci_dev->dev;
2878
2879	if (!pci_dev_run_wake(pci_dev))
2880	return;
2881
2882	spin_lock_irq(lock: &dev->power.lock);
2883
2884	if (pm_runtime_suspended(dev) && pci_dev->current_state < PCI_D3cold)
2885	__pci_pme_active(dev: pci_dev, enable: true);
2886
2887	spin_unlock_irq(lock: &dev->power.lock);
2888	}
2889
2890	/**
2891	* pci_choose_state - Choose the power state of a PCI device.
2892	* @dev: Target PCI device.
2893	* @state: Target state for the whole system.
2894	*
2895	* Returns PCI power state suitable for @dev and @state.
2896	*/
2897	pci_power_t pci_choose_state(struct pci_dev *dev, pm_message_t state)
2898	{
2899	if (state.event == PM_EVENT_ON)
2900	return PCI_D0;
2901
2902	return pci_target_state(dev, wakeup: false);
2903	}
2904	EXPORT_SYMBOL(pci_choose_state);
2905
2906	void pci_config_pm_runtime_get(struct pci_dev *pdev)
2907	{
2908	struct device *dev = &pdev->dev;
2909	struct device *parent = dev->parent;
2910
2911	if (parent)
2912	pm_runtime_get_sync(dev: parent);
2913	pm_runtime_get_noresume(dev);
2914	/*
2915	* pdev->current_state is set to PCI_D3cold during suspending,
2916	* so wait until suspending completes
2917	*/
2918	pm_runtime_barrier(dev);
2919	/*
2920	* Only need to resume devices in D3cold, because config
2921	* registers are still accessible for devices suspended but
2922	* not in D3cold.
2923	*/
2924	if (pdev->current_state == PCI_D3cold)
2925	pm_runtime_resume(dev);
2926	}
2927
2928	void pci_config_pm_runtime_put(struct pci_dev *pdev)
2929	{
2930	struct device *dev = &pdev->dev;
2931	struct device *parent = dev->parent;
2932
2933	pm_runtime_put(dev);
2934	if (parent)
2935	pm_runtime_put_sync(dev: parent);
2936	}
2937
2938	static const struct dmi_system_id bridge_d3_blacklist[] = {
2939	#ifdef CONFIG_X86
2940	{
2941	/*
2942	* Gigabyte X299 root port is not marked as hotplug capable
2943	* which allows Linux to power manage it. However, this
2944	* confuses the BIOS SMI handler so don't power manage root
2945	* ports on that system.
2946	*/
2947	.ident = "X299 DESIGNARE EX-CF",
2948	.matches = {
2949	DMI_MATCH(DMI_BOARD_VENDOR, "Gigabyte Technology Co., Ltd."),
2950	DMI_MATCH(DMI_BOARD_NAME, "X299 DESIGNARE EX-CF"),
2951	},
2952	},
2953	{
2954	/*
2955	* Downstream device is not accessible after putting a root port
2956	* into D3cold and back into D0 on Elo Continental Z2 board
2957	*/
2958	.ident = "Elo Continental Z2",
2959	.matches = {
2960	DMI_MATCH(DMI_BOARD_VENDOR, "Elo Touch Solutions"),
2961	DMI_MATCH(DMI_BOARD_NAME, "Geminilake"),
2962	DMI_MATCH(DMI_BOARD_VERSION, "Continental Z2"),
2963	},
2964	},
2965	#endif
2966	{ }
2967	};
2968
2969	/**
2970	* pci_bridge_d3_possible - Is it possible to put the bridge into D3
2971	* @bridge: Bridge to check
2972	*
2973	* This function checks if it is possible to move the bridge to D3.
2974	* Currently we only allow D3 for recent enough PCIe ports and Thunderbolt.
2975	*/
2976	bool pci_bridge_d3_possible(struct pci_dev *bridge)
2977	{
2978	if (!pci_is_pcie(dev: bridge))
2979	return false;
2980
2981	switch (pci_pcie_type(dev: bridge)) {
2982	case PCI_EXP_TYPE_ROOT_PORT:
2983	case PCI_EXP_TYPE_UPSTREAM:
2984	case PCI_EXP_TYPE_DOWNSTREAM:
2985	if (pci_bridge_d3_disable)
2986	return false;
2987
2988	/*
2989	* Hotplug ports handled by firmware in System Management Mode
2990	* may not be put into D3 by the OS (Thunderbolt on non-Macs).
2991	*/
2992	if (bridge->is_hotplug_bridge && !pciehp_is_native(bridge))
2993	return false;
2994
2995	if (pci_bridge_d3_force)
2996	return true;
2997
2998	/* Even the oldest 2010 Thunderbolt controller supports D3. */
2999	if (bridge->is_thunderbolt)
3000	return true;
3001
3002	/* Platform might know better if the bridge supports D3 */
3003	if (platform_pci_bridge_d3(dev: bridge))
3004	return true;
3005
3006	/*
3007	* Hotplug ports handled natively by the OS were not validated
3008	* by vendors for runtime D3 at least until 2018 because there
3009	* was no OS support.
3010	*/
3011	if (bridge->is_hotplug_bridge)
3012	return false;
3013
3014	if (dmi_check_system(list: bridge_d3_blacklist))
3015	return false;
3016
3017	/*
3018	* It should be safe to put PCIe ports from 2015 or newer
3019	* to D3.
3020	*/
3021	if (dmi_get_bios_year() >= 2015)
3022	return true;
3023	break;
3024	}
3025
3026	return false;
3027	}
3028
3029	static int pci_dev_check_d3cold(struct pci_dev *dev, void *data)
3030	{
3031	bool *d3cold_ok = data;
3032
3033	if (/* The device needs to be allowed to go D3cold ... */
3034	dev->no_d3cold || !dev->d3cold_allowed ||
3035
3036	/* ... and if it is wakeup capable to do so from D3cold. */
3037	(device_may_wakeup(dev: &dev->dev) &&
3038	!pci_pme_capable(dev, PCI_D3cold)) ||
3039
3040	/* If it is a bridge it must be allowed to go to D3. */
3041	!pci_power_manageable(pci_dev: dev))
3042
3043	*d3cold_ok = false;
3044
3045	return !*d3cold_ok;
3046	}
3047
3048	/*
3049	* pci_bridge_d3_update - Update bridge D3 capabilities
3050	* @dev: PCI device which is changed
3051	*
3052	* Update upstream bridge PM capabilities accordingly depending on if the
3053	* device PM configuration was changed or the device is being removed. The
3054	* change is also propagated upstream.
3055	*/
3056	void pci_bridge_d3_update(struct pci_dev *dev)
3057	{
3058	bool remove = !device_is_registered(dev: &dev->dev);
3059	struct pci_dev *bridge;
3060	bool d3cold_ok = true;
3061
3062	bridge = pci_upstream_bridge(dev);
3063	if (!bridge || !pci_bridge_d3_possible(bridge))
3064	return;
3065
3066	/*
3067	* If D3 is currently allowed for the bridge, removing one of its
3068	* children won't change that.
3069	*/
3070	if (remove && bridge->bridge_d3)
3071	return;
3072
3073	/*
3074	* If D3 is currently allowed for the bridge and a child is added or
3075	* changed, disallowance of D3 can only be caused by that child, so
3076	* we only need to check that single device, not any of its siblings.
3077	*
3078	* If D3 is currently not allowed for the bridge, checking the device
3079	* first may allow us to skip checking its siblings.
3080	*/
3081	if (!remove)
3082	pci_dev_check_d3cold(dev, data: &d3cold_ok);
3083
3084	/*
3085	* If D3 is currently not allowed for the bridge, this may be caused
3086	* either by the device being changed/removed or any of its siblings,
3087	* so we need to go through all children to find out if one of them
3088	* continues to block D3.
3089	*/
3090	if (d3cold_ok && !bridge->bridge_d3)
3091	pci_walk_bus(top: bridge->subordinate, cb: pci_dev_check_d3cold,
3092	userdata: &d3cold_ok);
3093
3094	if (bridge->bridge_d3 != d3cold_ok) {
3095	bridge->bridge_d3 = d3cold_ok;
3096	/* Propagate change to upstream bridges */
3097	pci_bridge_d3_update(dev: bridge);
3098	}
3099	}
3100
3101	/**
3102	* pci_d3cold_enable - Enable D3cold for device
3103	* @dev: PCI device to handle
3104	*
3105	* This function can be used in drivers to enable D3cold from the device
3106	* they handle. It also updates upstream PCI bridge PM capabilities
3107	* accordingly.
3108	*/
3109	void pci_d3cold_enable(struct pci_dev *dev)
3110	{
3111	if (dev->no_d3cold) {
3112	dev->no_d3cold = false;
3113	pci_bridge_d3_update(dev);
3114	}
3115	}
3116	EXPORT_SYMBOL_GPL(pci_d3cold_enable);
3117
3118	/**
3119	* pci_d3cold_disable - Disable D3cold for device
3120	* @dev: PCI device to handle
3121	*
3122	* This function can be used in drivers to disable D3cold from the device
3123	* they handle. It also updates upstream PCI bridge PM capabilities
3124	* accordingly.
3125	*/
3126	void pci_d3cold_disable(struct pci_dev *dev)
3127	{
3128	if (!dev->no_d3cold) {
3129	dev->no_d3cold = true;
3130	pci_bridge_d3_update(dev);
3131	}
3132	}
3133	EXPORT_SYMBOL_GPL(pci_d3cold_disable);
3134
3135	/**
3136	* pci_pm_init - Initialize PM functions of given PCI device
3137	* @dev: PCI device to handle.
3138	*/
3139	void pci_pm_init(struct pci_dev *dev)
3140	{
3141	int pm;
3142	u16 status;
3143	u16 pmc;
3144
3145	pm_runtime_forbid(dev: &dev->dev);
3146	pm_runtime_set_active(dev: &dev->dev);
3147	pm_runtime_enable(dev: &dev->dev);
3148	device_enable_async_suspend(dev: &dev->dev);
3149	dev->wakeup_prepared = false;
3150
3151	dev->pm_cap = 0;
3152	dev->pme_support = 0;
3153
3154	/* find PCI PM capability in list */
3155	pm = pci_find_capability(dev, PCI_CAP_ID_PM);
3156	if (!pm)
3157	return;
3158	/* Check device's ability to generate PME# */
3159	pci_read_config_word(dev, where: pm + PCI_PM_PMC, val: &pmc);
3160
3161	if ((pmc & PCI_PM_CAP_VER_MASK) > 3) {
3162	pci_err(dev, "unsupported PM cap regs version (%u)\n",
3163	pmc & PCI_PM_CAP_VER_MASK);
3164	return;
3165	}
3166
3167	dev->pm_cap = pm;
3168	dev->d3hot_delay = PCI_PM_D3HOT_WAIT;
3169	dev->d3cold_delay = PCI_PM_D3COLD_WAIT;
3170	dev->bridge_d3 = pci_bridge_d3_possible(bridge: dev);
3171	dev->d3cold_allowed = true;
3172
3173	dev->d1_support = false;
3174	dev->d2_support = false;
3175	if (!pci_no_d1d2(dev)) {
3176	if (pmc & PCI_PM_CAP_D1)
3177	dev->d1_support = true;
3178	if (pmc & PCI_PM_CAP_D2)
3179	dev->d2_support = true;
3180
3181	if (dev->d1_support || dev->d2_support)
3182	pci_info(dev, "supports%s%s\n",
3183	dev->d1_support ? " D1" : "",
3184	dev->d2_support ? " D2" : "");
3185	}
3186
3187	pmc &= PCI_PM_CAP_PME_MASK;
3188	if (pmc) {
3189	pci_info(dev, "PME# supported from%s%s%s%s%s\n",
3190	(pmc & PCI_PM_CAP_PME_D0) ? " D0" : "",
3191	(pmc & PCI_PM_CAP_PME_D1) ? " D1" : "",
3192	(pmc & PCI_PM_CAP_PME_D2) ? " D2" : "",
3193	(pmc & PCI_PM_CAP_PME_D3hot) ? " D3hot" : "",
3194	(pmc & PCI_PM_CAP_PME_D3cold) ? " D3cold" : "");
3195	dev->pme_support = FIELD_GET(PCI_PM_CAP_PME_MASK, pmc);
3196	dev->pme_poll = true;
3197	/*
3198	* Make device's PM flags reflect the wake-up capability, but
3199	* let the user space enable it to wake up the system as needed.
3200	*/
3201	device_set_wakeup_capable(dev: &dev->dev, capable: true);
3202	/* Disable the PME# generation functionality */
3203	pci_pme_active(dev, false);
3204	}
3205
3206	pci_read_config_word(dev, PCI_STATUS, val: &status);
3207	if (status & PCI_STATUS_IMM_READY)
3208	dev->imm_ready = 1;
3209	}
3210
3211	static unsigned long pci_ea_flags(struct pci_dev *dev, u8 prop)
3212	{
3213	unsigned long flags = IORESOURCE_PCI_FIXED | IORESOURCE_PCI_EA_BEI;
3214
3215	switch (prop) {
3216	case PCI_EA_P_MEM:
3217	case PCI_EA_P_VF_MEM:
3218	flags |= IORESOURCE_MEM;
3219	break;
3220	case PCI_EA_P_MEM_PREFETCH:
3221	case PCI_EA_P_VF_MEM_PREFETCH:
3222	flags |= IORESOURCE_MEM | IORESOURCE_PREFETCH;
3223	break;
3224	case PCI_EA_P_IO:
3225	flags |= IORESOURCE_IO;
3226	break;
3227	default:
3228	return 0;
3229	}
3230
3231	return flags;
3232	}
3233
3234	static struct resource *pci_ea_get_resource(struct pci_dev *dev, u8 bei,
3235	u8 prop)
3236	{
3237	if (bei <= PCI_EA_BEI_BAR5 && prop <= PCI_EA_P_IO)
3238	return &dev->resource[bei];
3239	#ifdef CONFIG_PCI_IOV
3240	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5 &&
3241	(prop == PCI_EA_P_VF_MEM || prop == PCI_EA_P_VF_MEM_PREFETCH))
3242	return &dev->resource[PCI_IOV_RESOURCES +
3243	bei - PCI_EA_BEI_VF_BAR0];
3244	#endif
3245	else if (bei == PCI_EA_BEI_ROM)
3246	return &dev->resource[PCI_ROM_RESOURCE];
3247	else
3248	return NULL;
3249	}
3250
3251	/* Read an Enhanced Allocation (EA) entry */
3252	static int pci_ea_read(struct pci_dev *dev, int offset)
3253	{
3254	struct resource *res;
3255	const char *res_name;
3256	int ent_size, ent_offset = offset;
3257	resource_size_t start, end;
3258	unsigned long flags;
3259	u32 dw0, bei, base, max_offset;
3260	u8 prop;
3261	bool support_64 = (sizeof(resource_size_t) >= 8);
3262
3263	pci_read_config_dword(dev, where: ent_offset, val: &dw0);
3264	ent_offset += 4;
3265
3266	/* Entry size field indicates DWORDs after 1st */
3267	ent_size = (FIELD_GET(PCI_EA_ES, dw0) + 1) << 2;
3268
3269	if (!(dw0 & PCI_EA_ENABLE)) /* Entry not enabled */
3270	goto out;
3271
3272	bei = FIELD_GET(PCI_EA_BEI, dw0);
3273	prop = FIELD_GET(PCI_EA_PP, dw0);
3274
3275	/*
3276	* If the Property is in the reserved range, try the Secondary
3277	* Property instead.
3278	*/
3279	if (prop > PCI_EA_P_BRIDGE_IO && prop < PCI_EA_P_MEM_RESERVED)
3280	prop = FIELD_GET(PCI_EA_SP, dw0);
3281	if (prop > PCI_EA_P_BRIDGE_IO)
3282	goto out;
3283
3284	res = pci_ea_get_resource(dev, bei, prop);
3285	res_name = pci_resource_name(dev, i: bei);
3286	if (!res) {
3287	pci_err(dev, "Unsupported EA entry BEI: %u\n", bei);
3288	goto out;
3289	}
3290
3291	flags = pci_ea_flags(dev, prop);
3292	if (!flags) {
3293	pci_err(dev, "Unsupported EA properties: %#x\n", prop);
3294	goto out;
3295	}
3296
3297	/* Read Base */
3298	pci_read_config_dword(dev, where: ent_offset, val: &base);
3299	start = (base & PCI_EA_FIELD_MASK);
3300	ent_offset += 4;
3301
3302	/* Read MaxOffset */
3303	pci_read_config_dword(dev, where: ent_offset, val: &max_offset);
3304	ent_offset += 4;
3305
3306	/* Read Base MSBs (if 64-bit entry) */
3307	if (base & PCI_EA_IS_64) {
3308	u32 base_upper;
3309
3310	pci_read_config_dword(dev, where: ent_offset, val: &base_upper);
3311	ent_offset += 4;
3312
3313	flags |= IORESOURCE_MEM_64;
3314
3315	/* entry starts above 32-bit boundary, can't use */
3316	if (!support_64 && base_upper)
3317	goto out;
3318
3319	if (support_64)
3320	start |= ((u64)base_upper << 32);
3321	}
3322
3323	end = start + (max_offset | 0x03);
3324
3325	/* Read MaxOffset MSBs (if 64-bit entry) */
3326	if (max_offset & PCI_EA_IS_64) {
3327	u32 max_offset_upper;
3328
3329	pci_read_config_dword(dev, where: ent_offset, val: &max_offset_upper);
3330	ent_offset += 4;
3331
3332	flags |= IORESOURCE_MEM_64;
3333
3334	/* entry too big, can't use */
3335	if (!support_64 && max_offset_upper)
3336	goto out;
3337
3338	if (support_64)
3339	end += ((u64)max_offset_upper << 32);
3340	}
3341
3342	if (end < start) {
3343	pci_err(dev, "EA Entry crosses address boundary\n");
3344	goto out;
3345	}
3346
3347	if (ent_size != ent_offset - offset) {
3348	pci_err(dev, "EA Entry Size (%d) does not match length read (%d)\n",
3349	ent_size, ent_offset - offset);
3350	goto out;
3351	}
3352
3353	res->name = pci_name(pdev: dev);
3354	res->start = start;
3355	res->end = end;
3356	res->flags = flags;
3357
3358	if (bei <= PCI_EA_BEI_BAR5)
3359	pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3360	res_name, res, prop);
3361	else if (bei == PCI_EA_BEI_ROM)
3362	pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3363	res_name, res, prop);
3364	else if (bei >= PCI_EA_BEI_VF_BAR0 && bei <= PCI_EA_BEI_VF_BAR5)
3365	pci_info(dev, "%s %pR: from Enhanced Allocation, properties %#02x\n",
3366	res_name, res, prop);
3367	else
3368	pci_info(dev, "BEI %d %pR: from Enhanced Allocation, properties %#02x\n",
3369	bei, res, prop);
3370
3371	out:
3372	return offset + ent_size;
3373	}
3374
3375	/* Enhanced Allocation Initialization */
3376	void pci_ea_init(struct pci_dev *dev)
3377	{
3378	int ea;
3379	u8 num_ent;
3380	int offset;
3381	int i;
3382
3383	/* find PCI EA capability in list */
3384	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
3385	if (!ea)
3386	return;
3387
3388	/* determine the number of entries */
3389	pci_bus_read_config_byte(bus: dev->bus, devfn: dev->devfn, where: ea + PCI_EA_NUM_ENT,
3390	val: &num_ent);
3391	num_ent &= PCI_EA_NUM_ENT_MASK;
3392
3393	offset = ea + PCI_EA_FIRST_ENT;
3394
3395	/* Skip DWORD 2 for type 1 functions */
3396	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE)
3397	offset += 4;
3398
3399	/* parse each EA entry */
3400	for (i = 0; i < num_ent; ++i)
3401	offset = pci_ea_read(dev, offset);
3402	}
3403
3404	static void pci_add_saved_cap(struct pci_dev *pci_dev,
3405	struct pci_cap_saved_state *new_cap)
3406	{
3407	hlist_add_head(n: &new_cap->next, h: &pci_dev->saved_cap_space);
3408	}
3409
3410	/**
3411	* _pci_add_cap_save_buffer - allocate buffer for saving given
3412	* capability registers
3413	* @dev: the PCI device
3414	* @cap: the capability to allocate the buffer for
3415	* @extended: Standard or Extended capability ID
3416	* @size: requested size of the buffer
3417	*/
3418	static int _pci_add_cap_save_buffer(struct pci_dev *dev, u16 cap,
3419	bool extended, unsigned int size)
3420	{
3421	int pos;
3422	struct pci_cap_saved_state *save_state;
3423
3424	if (extended)
3425	pos = pci_find_ext_capability(dev, cap);
3426	else
3427	pos = pci_find_capability(dev, cap);
3428
3429	if (!pos)
3430	return 0;
3431
3432	save_state = kzalloc(size: sizeof(*save_state) + size, GFP_KERNEL);
3433	if (!save_state)
3434	return -ENOMEM;
3435
3436	save_state->cap.cap_nr = cap;
3437	save_state->cap.cap_extended = extended;
3438	save_state->cap.size = size;
3439	pci_add_saved_cap(pci_dev: dev, new_cap: save_state);
3440
3441	return 0;
3442	}
3443
3444	int pci_add_cap_save_buffer(struct pci_dev *dev, char cap, unsigned int size)
3445	{
3446	return _pci_add_cap_save_buffer(dev, cap, extended: false, size);
3447	}
3448
3449	int pci_add_ext_cap_save_buffer(struct pci_dev *dev, u16 cap, unsigned int size)
3450	{
3451	return _pci_add_cap_save_buffer(dev, cap, extended: true, size);
3452	}
3453
3454	/**
3455	* pci_allocate_cap_save_buffers - allocate buffers for saving capabilities
3456	* @dev: the PCI device
3457	*/
3458	void pci_allocate_cap_save_buffers(struct pci_dev *dev)
3459	{
3460	int error;
3461
3462	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_EXP,
3463	PCI_EXP_SAVE_REGS * sizeof(u16));
3464	if (error)
3465	pci_err(dev, "unable to preallocate PCI Express save buffer\n");
3466
3467	error = pci_add_cap_save_buffer(dev, PCI_CAP_ID_PCIX, size: sizeof(u16));
3468	if (error)
3469	pci_err(dev, "unable to preallocate PCI-X save buffer\n");
3470
3471	error = pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_LTR,
3472	size: 2 * sizeof(u16));
3473	if (error)
3474	pci_err(dev, "unable to allocate suspend buffer for LTR\n");
3475
3476	pci_allocate_vc_save_buffers(dev);
3477	}
3478
3479	void pci_free_cap_save_buffers(struct pci_dev *dev)
3480	{
3481	struct pci_cap_saved_state *tmp;
3482	struct hlist_node *n;
3483
3484	hlist_for_each_entry_safe(tmp, n, &dev->saved_cap_space, next)
3485	kfree(objp: tmp);
3486	}
3487
3488	/**
3489	* pci_configure_ari - enable or disable ARI forwarding
3490	* @dev: the PCI device
3491	*
3492	* If @dev and its upstream bridge both support ARI, enable ARI in the
3493	* bridge. Otherwise, disable ARI in the bridge.
3494	*/
3495	void pci_configure_ari(struct pci_dev *dev)
3496	{
3497	u32 cap;
3498	struct pci_dev *bridge;
3499
3500	if (pcie_ari_disabled || !pci_is_pcie(dev) || dev->devfn)
3501	return;
3502
3503	bridge = dev->bus->self;
3504	if (!bridge)
3505	return;
3506
3507	pcie_capability_read_dword(dev: bridge, PCI_EXP_DEVCAP2, val: &cap);
3508	if (!(cap & PCI_EXP_DEVCAP2_ARI))
3509	return;
3510
3511	if (pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI)) {
3512	pcie_capability_set_word(dev: bridge, PCI_EXP_DEVCTL2,
3513	PCI_EXP_DEVCTL2_ARI);
3514	bridge->ari_enabled = 1;
3515	} else {
3516	pcie_capability_clear_word(dev: bridge, PCI_EXP_DEVCTL2,
3517	PCI_EXP_DEVCTL2_ARI);
3518	bridge->ari_enabled = 0;
3519	}
3520	}
3521
3522	static bool pci_acs_flags_enabled(struct pci_dev *pdev, u16 acs_flags)
3523	{
3524	int pos;
3525	u16 cap, ctrl;
3526
3527	pos = pdev->acs_cap;
3528	if (!pos)
3529	return false;
3530
3531	/*
3532	* Except for egress control, capabilities are either required
3533	* or only required if controllable. Features missing from the
3534	* capability field can therefore be assumed as hard-wired enabled.
3535	*/
3536	pci_read_config_word(dev: pdev, where: pos + PCI_ACS_CAP, val: &cap);
3537	acs_flags &= (cap | PCI_ACS_EC);
3538
3539	pci_read_config_word(dev: pdev, where: pos + PCI_ACS_CTRL, val: &ctrl);
3540	return (ctrl & acs_flags) == acs_flags;
3541	}
3542
3543	/**
3544	* pci_acs_enabled - test ACS against required flags for a given device
3545	* @pdev: device to test
3546	* @acs_flags: required PCI ACS flags
3547	*
3548	* Return true if the device supports the provided flags. Automatically
3549	* filters out flags that are not implemented on multifunction devices.
3550	*
3551	* Note that this interface checks the effective ACS capabilities of the
3552	* device rather than the actual capabilities. For instance, most single
3553	* function endpoints are not required to support ACS because they have no
3554	* opportunity for peer-to-peer access. We therefore return 'true'
3555	* regardless of whether the device exposes an ACS capability. This makes
3556	* it much easier for callers of this function to ignore the actual type
3557	* or topology of the device when testing ACS support.
3558	*/
3559	bool pci_acs_enabled(struct pci_dev *pdev, u16 acs_flags)
3560	{
3561	int ret;
3562
3563	ret = pci_dev_specific_acs_enabled(dev: pdev, acs_flags);
3564	if (ret >= 0)
3565	return ret > 0;
3566
3567	/*
3568	* Conventional PCI and PCI-X devices never support ACS, either
3569	* effectively or actually. The shared bus topology implies that
3570	* any device on the bus can receive or snoop DMA.
3571	*/
3572	if (!pci_is_pcie(dev: pdev))
3573	return false;
3574
3575	switch (pci_pcie_type(dev: pdev)) {
3576	/*
3577	* PCI/X-to-PCIe bridges are not specifically mentioned by the spec,
3578	* but since their primary interface is PCI/X, we conservatively
3579	* handle them as we would a non-PCIe device.
3580	*/
3581	case PCI_EXP_TYPE_PCIE_BRIDGE:
3582	/*
3583	* PCIe 3.0, 6.12.1 excludes ACS on these devices. "ACS is never
3584	* applicable... must never implement an ACS Extended Capability...".
3585	* This seems arbitrary, but we take a conservative interpretation
3586	* of this statement.
3587	*/
3588	case PCI_EXP_TYPE_PCI_BRIDGE:
3589	case PCI_EXP_TYPE_RC_EC:
3590	return false;
3591	/*
3592	* PCIe 3.0, 6.12.1.1 specifies that downstream and root ports should
3593	* implement ACS in order to indicate their peer-to-peer capabilities,
3594	* regardless of whether they are single- or multi-function devices.
3595	*/
3596	case PCI_EXP_TYPE_DOWNSTREAM:
3597	case PCI_EXP_TYPE_ROOT_PORT:
3598	return pci_acs_flags_enabled(pdev, acs_flags);
3599	/*
3600	* PCIe 3.0, 6.12.1.2 specifies ACS capabilities that should be
3601	* implemented by the remaining PCIe types to indicate peer-to-peer
3602	* capabilities, but only when they are part of a multifunction
3603	* device. The footnote for section 6.12 indicates the specific
3604	* PCIe types included here.
3605	*/
3606	case PCI_EXP_TYPE_ENDPOINT:
3607	case PCI_EXP_TYPE_UPSTREAM:
3608	case PCI_EXP_TYPE_LEG_END:
3609	case PCI_EXP_TYPE_RC_END:
3610	if (!pdev->multifunction)
3611	break;
3612
3613	return pci_acs_flags_enabled(pdev, acs_flags);
3614	}
3615
3616	/*
3617	* PCIe 3.0, 6.12.1.3 specifies no ACS capabilities are applicable
3618	* to single function devices with the exception of downstream ports.
3619	*/
3620	return true;
3621	}
3622
3623	/**
3624	* pci_acs_path_enabled - test ACS flags from start to end in a hierarchy
3625	* @start: starting downstream device
3626	* @end: ending upstream device or NULL to search to the root bus
3627	* @acs_flags: required flags
3628	*
3629	* Walk up a device tree from start to end testing PCI ACS support. If
3630	* any step along the way does not support the required flags, return false.
3631	*/
3632	bool pci_acs_path_enabled(struct pci_dev *start,
3633	struct pci_dev *end, u16 acs_flags)
3634	{
3635	struct pci_dev *pdev, *parent = start;
3636
3637	do {
3638	pdev = parent;
3639
3640	if (!pci_acs_enabled(pdev, acs_flags))
3641	return false;
3642
3643	if (pci_is_root_bus(pbus: pdev->bus))
3644	return (end == NULL);
3645
3646	parent = pdev->bus->self;
3647	} while (pdev != end);
3648
3649	return true;
3650	}
3651
3652	/**
3653	* pci_acs_init - Initialize ACS if hardware supports it
3654	* @dev: the PCI device
3655	*/
3656	void pci_acs_init(struct pci_dev *dev)
3657	{
3658	dev->acs_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS);
3659
3660	/*
3661	* Attempt to enable ACS regardless of capability because some Root
3662	* Ports (e.g. those quirked with *_intel_pch_acs_*) do not have
3663	* the standard ACS capability but still support ACS via those
3664	* quirks.
3665	*/
3666	pci_enable_acs(dev);
3667	}
3668
3669	/**
3670	* pci_rebar_find_pos - find position of resize ctrl reg for BAR
3671	* @pdev: PCI device
3672	* @bar: BAR to find
3673	*
3674	* Helper to find the position of the ctrl register for a BAR.
3675	* Returns -ENOTSUPP if resizable BARs are not supported at all.
3676	* Returns -ENOENT if no ctrl register for the BAR could be found.
3677	*/
3678	static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
3679	{
3680	unsigned int pos, nbars, i;
3681	u32 ctrl;
3682
3683	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
3684	if (!pos)
3685	return -ENOTSUPP;
3686
3687	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
3688	nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
3689
3690	for (i = 0; i < nbars; i++, pos += 8) {
3691	int bar_idx;
3692
3693	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
3694	bar_idx = FIELD_GET(PCI_REBAR_CTRL_BAR_IDX, ctrl);
3695	if (bar_idx == bar)
3696	return pos;
3697	}
3698
3699	return -ENOENT;
3700	}
3701
3702	/**
3703	* pci_rebar_get_possible_sizes - get possible sizes for BAR
3704	* @pdev: PCI device
3705	* @bar: BAR to query
3706	*
3707	* Get the possible sizes of a resizable BAR as bitmask defined in the spec
3708	* (bit 0=1MB, bit 19=512GB). Returns 0 if BAR isn't resizable.
3709	*/
3710	u32 pci_rebar_get_possible_sizes(struct pci_dev *pdev, int bar)
3711	{
3712	int pos;
3713	u32 cap;
3714
3715	pos = pci_rebar_find_pos(pdev, bar);
3716	if (pos < 0)
3717	return 0;
3718
3719	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CAP, val: &cap);
3720	cap = FIELD_GET(PCI_REBAR_CAP_SIZES, cap);
3721
3722	/* Sapphire RX 5600 XT Pulse has an invalid cap dword for BAR 0 */
3723	if (pdev->vendor == PCI_VENDOR_ID_ATI && pdev->device == 0x731f &&
3724	bar == 0 && cap == 0x700)
3725	return 0x3f00;
3726
3727	return cap;
3728	}
3729	EXPORT_SYMBOL(pci_rebar_get_possible_sizes);
3730
3731	/**
3732	* pci_rebar_get_current_size - get the current size of a BAR
3733	* @pdev: PCI device
3734	* @bar: BAR to set size to
3735	*
3736	* Read the size of a BAR from the resizable BAR config.
3737	* Returns size if found or negative error code.
3738	*/
3739	int pci_rebar_get_current_size(struct pci_dev *pdev, int bar)
3740	{
3741	int pos;
3742	u32 ctrl;
3743
3744	pos = pci_rebar_find_pos(pdev, bar);
3745	if (pos < 0)
3746	return pos;
3747
3748	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
3749	return FIELD_GET(PCI_REBAR_CTRL_BAR_SIZE, ctrl);
3750	}
3751
3752	/**
3753	* pci_rebar_set_size - set a new size for a BAR
3754	* @pdev: PCI device
3755	* @bar: BAR to set size to
3756	* @size: new size as defined in the spec (0=1MB, 19=512GB)
3757	*
3758	* Set the new size of a BAR as defined in the spec.
3759	* Returns zero if resizing was successful, error code otherwise.
3760	*/
3761	int pci_rebar_set_size(struct pci_dev *pdev, int bar, int size)
3762	{
3763	int pos;
3764	u32 ctrl;
3765
3766	pos = pci_rebar_find_pos(pdev, bar);
3767	if (pos < 0)
3768	return pos;
3769
3770	pci_read_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: &ctrl);
3771	ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
3772	ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
3773	pci_write_config_dword(dev: pdev, where: pos + PCI_REBAR_CTRL, val: ctrl);
3774	return 0;
3775	}
3776
3777	/**
3778	* pci_enable_atomic_ops_to_root - enable AtomicOp requests to root port
3779	* @dev: the PCI device
3780	* @cap_mask: mask of desired AtomicOp sizes, including one or more of:
3781	* PCI_EXP_DEVCAP2_ATOMIC_COMP32
3782	* PCI_EXP_DEVCAP2_ATOMIC_COMP64
3783	* PCI_EXP_DEVCAP2_ATOMIC_COMP128
3784	*
3785	* Return 0 if all upstream bridges support AtomicOp routing, egress
3786	* blocking is disabled on all upstream ports, and the root port supports
3787	* the requested completion capabilities (32-bit, 64-bit and/or 128-bit
3788	* AtomicOp completion), or negative otherwise.
3789	*/
3790	int pci_enable_atomic_ops_to_root(struct pci_dev *dev, u32 cap_mask)
3791	{
3792	struct pci_bus *bus = dev->bus;
3793	struct pci_dev *bridge;
3794	u32 cap, ctl2;
3795
3796	/*
3797	* Per PCIe r5.0, sec 9.3.5.10, the AtomicOp Requester Enable bit
3798	* in Device Control 2 is reserved in VFs and the PF value applies
3799	* to all associated VFs.
3800	*/
3801	if (dev->is_virtfn)
3802	return -EINVAL;
3803
3804	if (!pci_is_pcie(dev))
3805	return -EINVAL;
3806
3807	/*
3808	* Per PCIe r4.0, sec 6.15, endpoints and root ports may be
3809	* AtomicOp requesters. For now, we only support endpoints as
3810	* requesters and root ports as completers. No endpoints as
3811	* completers, and no peer-to-peer.
3812	*/
3813
3814	switch (pci_pcie_type(dev)) {
3815	case PCI_EXP_TYPE_ENDPOINT:
3816	case PCI_EXP_TYPE_LEG_END:
3817	case PCI_EXP_TYPE_RC_END:
3818	break;
3819	default:
3820	return -EINVAL;
3821	}
3822
3823	while (bus->parent) {
3824	bridge = bus->self;
3825
3826	pcie_capability_read_dword(dev: bridge, PCI_EXP_DEVCAP2, val: &cap);
3827
3828	switch (pci_pcie_type(dev: bridge)) {
3829	/* Ensure switch ports support AtomicOp routing */
3830	case PCI_EXP_TYPE_UPSTREAM:
3831	case PCI_EXP_TYPE_DOWNSTREAM:
3832	if (!(cap & PCI_EXP_DEVCAP2_ATOMIC_ROUTE))
3833	return -EINVAL;
3834	break;
3835
3836	/* Ensure root port supports all the sizes we care about */
3837	case PCI_EXP_TYPE_ROOT_PORT:
3838	if ((cap & cap_mask) != cap_mask)
3839	return -EINVAL;
3840	break;
3841	}
3842
3843	/* Ensure upstream ports don't block AtomicOps on egress */
3844	if (pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_UPSTREAM) {
3845	pcie_capability_read_dword(dev: bridge, PCI_EXP_DEVCTL2,
3846	val: &ctl2);
3847	if (ctl2 & PCI_EXP_DEVCTL2_ATOMIC_EGRESS_BLOCK)
3848	return -EINVAL;
3849	}
3850
3851	bus = bus->parent;
3852	}
3853
3854	pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
3855	PCI_EXP_DEVCTL2_ATOMIC_REQ);
3856	return 0;
3857	}
3858	EXPORT_SYMBOL(pci_enable_atomic_ops_to_root);
3859
3860	/**
3861	* pci_release_region - Release a PCI bar
3862	* @pdev: PCI device whose resources were previously reserved by
3863	* pci_request_region()
3864	* @bar: BAR to release
3865	*
3866	* Releases the PCI I/O and memory resources previously reserved by a
3867	* successful call to pci_request_region(). Call this function only
3868	* after all use of the PCI regions has ceased.
3869	*/
3870	void pci_release_region(struct pci_dev *pdev, int bar)
3871	{
3872	struct pci_devres *dr;
3873
3874	if (pci_resource_len(pdev, bar) == 0)
3875	return;
3876	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO)
3877	release_region(pci_resource_start(pdev, bar),
3878	pci_resource_len(pdev, bar));
3879	else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM)
3880	release_mem_region(pci_resource_start(pdev, bar),
3881	pci_resource_len(pdev, bar));
3882
3883	dr = find_pci_dr(pdev);
3884	if (dr)
3885	dr->region_mask &= ~(1 << bar);
3886	}
3887	EXPORT_SYMBOL(pci_release_region);
3888
3889	/**
3890	* __pci_request_region - Reserved PCI I/O and memory resource
3891	* @pdev: PCI device whose resources are to be reserved
3892	* @bar: BAR to be reserved
3893	* @res_name: Name to be associated with resource.
3894	* @exclusive: whether the region access is exclusive or not
3895	*
3896	* Mark the PCI region associated with PCI device @pdev BAR @bar as
3897	* being reserved by owner @res_name. Do not access any
3898	* address inside the PCI regions unless this call returns
3899	* successfully.
3900	*
3901	* If @exclusive is set, then the region is marked so that userspace
3902	* is explicitly not allowed to map the resource via /dev/mem or
3903	* sysfs MMIO access.
3904	*
3905	* Returns 0 on success, or %EBUSY on error. A warning
3906	* message is also printed on failure.
3907	*/
3908	static int __pci_request_region(struct pci_dev *pdev, int bar,
3909	const char *res_name, int exclusive)
3910	{
3911	struct pci_devres *dr;
3912
3913	if (pci_resource_len(pdev, bar) == 0)
3914	return 0;
3915
3916	if (pci_resource_flags(pdev, bar) & IORESOURCE_IO) {
3917	if (!request_region(pci_resource_start(pdev, bar),
3918	pci_resource_len(pdev, bar), res_name))
3919	goto err_out;
3920	} else if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
3921	if (!__request_mem_region(pci_resource_start(pdev, bar),
3922	pci_resource_len(pdev, bar), res_name,
3923	exclusive))
3924	goto err_out;
3925	}
3926
3927	dr = find_pci_dr(pdev);
3928	if (dr)
3929	dr->region_mask |= 1 << bar;
3930
3931	return 0;
3932
3933	err_out:
3934	pci_warn(pdev, "BAR %d: can't reserve %pR\n", bar,
3935	&pdev->resource[bar]);
3936	return -EBUSY;
3937	}
3938
3939	/**
3940	* pci_request_region - Reserve PCI I/O and memory resource
3941	* @pdev: PCI device whose resources are to be reserved
3942	* @bar: BAR to be reserved
3943	* @res_name: Name to be associated with resource
3944	*
3945	* Mark the PCI region associated with PCI device @pdev BAR @bar as
3946	* being reserved by owner @res_name. Do not access any
3947	* address inside the PCI regions unless this call returns
3948	* successfully.
3949	*
3950	* Returns 0 on success, or %EBUSY on error. A warning
3951	* message is also printed on failure.
3952	*/
3953	int pci_request_region(struct pci_dev *pdev, int bar, const char *res_name)
3954	{
3955	return __pci_request_region(pdev, bar, res_name, exclusive: 0);
3956	}
3957	EXPORT_SYMBOL(pci_request_region);
3958
3959	/**
3960	* pci_release_selected_regions - Release selected PCI I/O and memory resources
3961	* @pdev: PCI device whose resources were previously reserved
3962	* @bars: Bitmask of BARs to be released
3963	*
3964	* Release selected PCI I/O and memory resources previously reserved.
3965	* Call this function only after all use of the PCI regions has ceased.
3966	*/
3967	void pci_release_selected_regions(struct pci_dev *pdev, int bars)
3968	{
3969	int i;
3970
3971	for (i = 0; i < PCI_STD_NUM_BARS; i++)
3972	if (bars & (1 << i))
3973	pci_release_region(pdev, i);
3974	}
3975	EXPORT_SYMBOL(pci_release_selected_regions);
3976
3977	static int __pci_request_selected_regions(struct pci_dev *pdev, int bars,
3978	const char *res_name, int excl)
3979	{
3980	int i;
3981
3982	for (i = 0; i < PCI_STD_NUM_BARS; i++)
3983	if (bars & (1 << i))
3984	if (__pci_request_region(pdev, bar: i, res_name, exclusive: excl))
3985	goto err_out;
3986	return 0;
3987
3988	err_out:
3989	while (--i >= 0)
3990	if (bars & (1 << i))
3991	pci_release_region(pdev, i);
3992
3993	return -EBUSY;
3994	}
3995
3996
3997	/**
3998	* pci_request_selected_regions - Reserve selected PCI I/O and memory resources
3999	* @pdev: PCI device whose resources are to be reserved
4000	* @bars: Bitmask of BARs to be requested
4001	* @res_name: Name to be associated with resource
4002	*/
4003	int pci_request_selected_regions(struct pci_dev *pdev, int bars,
4004	const char *res_name)
4005	{
4006	return __pci_request_selected_regions(pdev, bars, res_name, excl: 0);
4007	}
4008	EXPORT_SYMBOL(pci_request_selected_regions);
4009
4010	int pci_request_selected_regions_exclusive(struct pci_dev *pdev, int bars,
4011	const char *res_name)
4012	{
4013	return __pci_request_selected_regions(pdev, bars, res_name,
4014	IORESOURCE_EXCLUSIVE);
4015	}
4016	EXPORT_SYMBOL(pci_request_selected_regions_exclusive);
4017
4018	/**
4019	* pci_release_regions - Release reserved PCI I/O and memory resources
4020	* @pdev: PCI device whose resources were previously reserved by
4021	* pci_request_regions()
4022	*
4023	* Releases all PCI I/O and memory resources previously reserved by a
4024	* successful call to pci_request_regions(). Call this function only
4025	* after all use of the PCI regions has ceased.
4026	*/
4027
4028	void pci_release_regions(struct pci_dev *pdev)
4029	{
4030	pci_release_selected_regions(pdev, (1 << PCI_STD_NUM_BARS) - 1);
4031	}
4032	EXPORT_SYMBOL(pci_release_regions);
4033
4034	/**
4035	* pci_request_regions - Reserve PCI I/O and memory resources
4036	* @pdev: PCI device whose resources are to be reserved
4037	* @res_name: Name to be associated with resource.
4038	*
4039	* Mark all PCI regions associated with PCI device @pdev as
4040	* being reserved by owner @res_name. Do not access any
4041	* address inside the PCI regions unless this call returns
4042	* successfully.
4043	*
4044	* Returns 0 on success, or %EBUSY on error. A warning
4045	* message is also printed on failure.
4046	*/
4047	int pci_request_regions(struct pci_dev *pdev, const char *res_name)
4048	{
4049	return pci_request_selected_regions(pdev,
4050	((1 << PCI_STD_NUM_BARS) - 1), res_name);
4051	}
4052	EXPORT_SYMBOL(pci_request_regions);
4053
4054	/**
4055	* pci_request_regions_exclusive - Reserve PCI I/O and memory resources
4056	* @pdev: PCI device whose resources are to be reserved
4057	* @res_name: Name to be associated with resource.
4058	*
4059	* Mark all PCI regions associated with PCI device @pdev as being reserved
4060	* by owner @res_name. Do not access any address inside the PCI regions
4061	* unless this call returns successfully.
4062	*
4063	* pci_request_regions_exclusive() will mark the region so that /dev/mem
4064	* and the sysfs MMIO access will not be allowed.
4065	*
4066	* Returns 0 on success, or %EBUSY on error. A warning message is also
4067	* printed on failure.
4068	*/
4069	int pci_request_regions_exclusive(struct pci_dev *pdev, const char *res_name)
4070	{
4071	return pci_request_selected_regions_exclusive(pdev,
4072	((1 << PCI_STD_NUM_BARS) - 1), res_name);
4073	}
4074	EXPORT_SYMBOL(pci_request_regions_exclusive);
4075
4076	/*
4077	* Record the PCI IO range (expressed as CPU physical address + size).
4078	* Return a negative value if an error has occurred, zero otherwise
4079	*/
4080	int pci_register_io_range(struct fwnode_handle *fwnode, phys_addr_t addr,
4081	resource_size_t size)
4082	{
4083	int ret = 0;
4084	#ifdef PCI_IOBASE
4085	struct logic_pio_hwaddr *range;
4086
4087	if (!size || addr + size < addr)
4088	return -EINVAL;
4089
4090	range = kzalloc(sizeof(*range), GFP_ATOMIC);
4091	if (!range)
4092	return -ENOMEM;
4093
4094	range->fwnode = fwnode;
4095	range->size = size;
4096	range->hw_start = addr;
4097	range->flags = LOGIC_PIO_CPU_MMIO;
4098
4099	ret = logic_pio_register_range(range);
4100	if (ret)
4101	kfree(range);
4102
4103	/* Ignore duplicates due to deferred probing */
4104	if (ret == -EEXIST)
4105	ret = 0;
4106	#endif
4107
4108	return ret;
4109	}
4110
4111	phys_addr_t pci_pio_to_address(unsigned long pio)
4112	{
4113	#ifdef PCI_IOBASE
4114	if (pio < MMIO_UPPER_LIMIT)
4115	return logic_pio_to_hwaddr(pio);
4116	#endif
4117
4118	return (phys_addr_t) OF_BAD_ADDR;
4119	}
4120	EXPORT_SYMBOL_GPL(pci_pio_to_address);
4121
4122	unsigned long __weak pci_address_to_pio(phys_addr_t address)
4123	{
4124	#ifdef PCI_IOBASE
4125	return logic_pio_trans_cpuaddr(address);
4126	#else
4127	if (address > IO_SPACE_LIMIT)
4128	return (unsigned long)-1;
4129
4130	return (unsigned long) address;
4131	#endif
4132	}
4133
4134	/**
4135	* pci_remap_iospace - Remap the memory mapped I/O space
4136	* @res: Resource describing the I/O space
4137	* @phys_addr: physical address of range to be mapped
4138	*
4139	* Remap the memory mapped I/O space described by the @res and the CPU
4140	* physical address @phys_addr into virtual address space. Only
4141	* architectures that have memory mapped IO functions defined (and the
4142	* PCI_IOBASE value defined) should call this function.
4143	*/
4144	#ifndef pci_remap_iospace
4145	int pci_remap_iospace(const struct resource *res, phys_addr_t phys_addr)
4146	{
4147	#if defined(PCI_IOBASE) && defined(CONFIG_MMU)
4148	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
4149
4150	if (!(res->flags & IORESOURCE_IO))
4151	return -EINVAL;
4152
4153	if (res->end > IO_SPACE_LIMIT)
4154	return -EINVAL;
4155
4156	return vmap_page_range(vaddr, vaddr + resource_size(res), phys_addr,
4157	pgprot_device(PAGE_KERNEL));
4158	#else
4159	/*
4160	* This architecture does not have memory mapped I/O space,
4161	* so this function should never be called
4162	*/
4163	WARN_ONCE(1, "This architecture does not support memory mapped I/O\n");
4164	return -ENODEV;
4165	#endif
4166	}
4167	EXPORT_SYMBOL(pci_remap_iospace);
4168	#endif
4169
4170	/**
4171	* pci_unmap_iospace - Unmap the memory mapped I/O space
4172	* @res: resource to be unmapped
4173	*
4174	* Unmap the CPU virtual address @res from virtual address space. Only
4175	* architectures that have memory mapped IO functions defined (and the
4176	* PCI_IOBASE value defined) should call this function.
4177	*/
4178	void pci_unmap_iospace(struct resource *res)
4179	{
4180	#if defined(PCI_IOBASE) && defined(CONFIG_MMU)
4181	unsigned long vaddr = (unsigned long)PCI_IOBASE + res->start;
4182
4183	vunmap_range(vaddr, vaddr + resource_size(res));
4184	#endif
4185	}
4186	EXPORT_SYMBOL(pci_unmap_iospace);
4187
4188	static void __pci_set_master(struct pci_dev *dev, bool enable)
4189	{
4190	u16 old_cmd, cmd;
4191
4192	pci_read_config_word(dev, PCI_COMMAND, val: &old_cmd);
4193	if (enable)
4194	cmd = old_cmd | PCI_COMMAND_MASTER;
4195	else
4196	cmd = old_cmd & ~PCI_COMMAND_MASTER;
4197	if (cmd != old_cmd) {
4198	pci_dbg(dev, "%s bus mastering\n",
4199	enable ? "enabling" : "disabling");
4200	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
4201	}
4202	dev->is_busmaster = enable;
4203	}
4204
4205	/**
4206	* pcibios_setup - process "pci=" kernel boot arguments
4207	* @str: string used to pass in "pci=" kernel boot arguments
4208	*
4209	* Process kernel boot arguments. This is the default implementation.
4210	* Architecture specific implementations can override this as necessary.
4211	*/
4212	char * __weak __init pcibios_setup(char *str)
4213	{
4214	return str;
4215	}
4216
4217	/**
4218	* pcibios_set_master - enable PCI bus-mastering for device dev
4219	* @dev: the PCI device to enable
4220	*
4221	* Enables PCI bus-mastering for the device. This is the default
4222	* implementation. Architecture specific implementations can override
4223	* this if necessary.
4224	*/
4225	void __weak pcibios_set_master(struct pci_dev *dev)
4226	{
4227	u8 lat;
4228
4229	/* The latency timer doesn't apply to PCIe (either Type 0 or Type 1) */
4230	if (pci_is_pcie(dev))
4231	return;
4232
4233	pci_read_config_byte(dev, PCI_LATENCY_TIMER, val: &lat);
4234	if (lat < 16)
4235	lat = (64 <= pcibios_max_latency) ? 64 : pcibios_max_latency;
4236	else if (lat > pcibios_max_latency)
4237	lat = pcibios_max_latency;
4238	else
4239	return;
4240
4241	pci_write_config_byte(dev, PCI_LATENCY_TIMER, val: lat);
4242	}
4243
4244	/**
4245	* pci_set_master - enables bus-mastering for device dev
4246	* @dev: the PCI device to enable
4247	*
4248	* Enables bus-mastering on the device and calls pcibios_set_master()
4249	* to do the needed arch specific settings.
4250	*/
4251	void pci_set_master(struct pci_dev *dev)
4252	{
4253	__pci_set_master(dev, enable: true);
4254	pcibios_set_master(dev);
4255	}
4256	EXPORT_SYMBOL(pci_set_master);
4257
4258	/**
4259	* pci_clear_master - disables bus-mastering for device dev
4260	* @dev: the PCI device to disable
4261	*/
4262	void pci_clear_master(struct pci_dev *dev)
4263	{
4264	__pci_set_master(dev, enable: false);
4265	}
4266	EXPORT_SYMBOL(pci_clear_master);
4267
4268	/**
4269	* pci_set_cacheline_size - ensure the CACHE_LINE_SIZE register is programmed
4270	* @dev: the PCI device for which MWI is to be enabled
4271	*
4272	* Helper function for pci_set_mwi.
4273	* Originally copied from drivers/net/acenic.c.
4274	* Copyright 1998-2001 by Jes Sorensen, <jes@trained-monkey.org>.
4275	*
4276	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4277	*/
4278	int pci_set_cacheline_size(struct pci_dev *dev)
4279	{
4280	u8 cacheline_size;
4281
4282	if (!pci_cache_line_size)
4283	return -EINVAL;
4284
4285	/* Validate current setting: the PCI_CACHE_LINE_SIZE must be
4286	equal to or multiple of the right value. */
4287	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, val: &cacheline_size);
4288	if (cacheline_size >= pci_cache_line_size &&
4289	(cacheline_size % pci_cache_line_size) == 0)
4290	return 0;
4291
4292	/* Write the correct value. */
4293	pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, val: pci_cache_line_size);
4294	/* Read it back. */
4295	pci_read_config_byte(dev, PCI_CACHE_LINE_SIZE, val: &cacheline_size);
4296	if (cacheline_size == pci_cache_line_size)
4297	return 0;
4298
4299	pci_dbg(dev, "cache line size of %d is not supported\n",
4300	pci_cache_line_size << 2);
4301
4302	return -EINVAL;
4303	}
4304	EXPORT_SYMBOL_GPL(pci_set_cacheline_size);
4305
4306	/**
4307	* pci_set_mwi - enables memory-write-invalidate PCI transaction
4308	* @dev: the PCI device for which MWI is enabled
4309	*
4310	* Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4311	*
4312	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4313	*/
4314	int pci_set_mwi(struct pci_dev *dev)
4315	{
4316	#ifdef PCI_DISABLE_MWI
4317	return 0;
4318	#else
4319	int rc;
4320	u16 cmd;
4321
4322	rc = pci_set_cacheline_size(dev);
4323	if (rc)
4324	return rc;
4325
4326	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
4327	if (!(cmd & PCI_COMMAND_INVALIDATE)) {
4328	pci_dbg(dev, "enabling Mem-Wr-Inval\n");
4329	cmd |= PCI_COMMAND_INVALIDATE;
4330	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
4331	}
4332	return 0;
4333	#endif
4334	}
4335	EXPORT_SYMBOL(pci_set_mwi);
4336
4337	/**
4338	* pci_try_set_mwi - enables memory-write-invalidate PCI transaction
4339	* @dev: the PCI device for which MWI is enabled
4340	*
4341	* Enables the Memory-Write-Invalidate transaction in %PCI_COMMAND.
4342	* Callers are not required to check the return value.
4343	*
4344	* RETURNS: An appropriate -ERRNO error value on error, or zero for success.
4345	*/
4346	int pci_try_set_mwi(struct pci_dev *dev)
4347	{
4348	#ifdef PCI_DISABLE_MWI
4349	return 0;
4350	#else
4351	return pci_set_mwi(dev);
4352	#endif
4353	}
4354	EXPORT_SYMBOL(pci_try_set_mwi);
4355
4356	/**
4357	* pci_clear_mwi - disables Memory-Write-Invalidate for device dev
4358	* @dev: the PCI device to disable
4359	*
4360	* Disables PCI Memory-Write-Invalidate transaction on the device
4361	*/
4362	void pci_clear_mwi(struct pci_dev *dev)
4363	{
4364	#ifndef PCI_DISABLE_MWI
4365	u16 cmd;
4366
4367	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
4368	if (cmd & PCI_COMMAND_INVALIDATE) {
4369	cmd &= ~PCI_COMMAND_INVALIDATE;
4370	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
4371	}
4372	#endif
4373	}
4374	EXPORT_SYMBOL(pci_clear_mwi);
4375
4376	/**
4377	* pci_disable_parity - disable parity checking for device
4378	* @dev: the PCI device to operate on
4379	*
4380	* Disable parity checking for device @dev
4381	*/
4382	void pci_disable_parity(struct pci_dev *dev)
4383	{
4384	u16 cmd;
4385
4386	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
4387	if (cmd & PCI_COMMAND_PARITY) {
4388	cmd &= ~PCI_COMMAND_PARITY;
4389	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
4390	}
4391	}
4392
4393	/**
4394	* pci_intx - enables/disables PCI INTx for device dev
4395	* @pdev: the PCI device to operate on
4396	* @enable: boolean: whether to enable or disable PCI INTx
4397	*
4398	* Enables/disables PCI INTx for device @pdev
4399	*/
4400	void pci_intx(struct pci_dev *pdev, int enable)
4401	{
4402	u16 pci_command, new;
4403
4404	pci_read_config_word(dev: pdev, PCI_COMMAND, val: &pci_command);
4405
4406	if (enable)
4407	new = pci_command & ~PCI_COMMAND_INTX_DISABLE;
4408	else
4409	new = pci_command | PCI_COMMAND_INTX_DISABLE;
4410
4411	if (new != pci_command) {
4412	struct pci_devres *dr;
4413
4414	pci_write_config_word(dev: pdev, PCI_COMMAND, val: new);
4415
4416	dr = find_pci_dr(pdev);
4417	if (dr && !dr->restore_intx) {
4418	dr->restore_intx = 1;
4419	dr->orig_intx = !enable;
4420	}
4421	}
4422	}
4423	EXPORT_SYMBOL_GPL(pci_intx);
4424
4425	/**
4426	* pci_wait_for_pending_transaction - wait for pending transaction
4427	* @dev: the PCI device to operate on
4428	*
4429	* Return 0 if transaction is pending 1 otherwise.
4430	*/
4431	int pci_wait_for_pending_transaction(struct pci_dev *dev)
4432	{
4433	if (!pci_is_pcie(dev))
4434	return 1;
4435
4436	return pci_wait_for_pending(dev, pos: pci_pcie_cap(dev) + PCI_EXP_DEVSTA,
4437	PCI_EXP_DEVSTA_TRPND);
4438	}
4439	EXPORT_SYMBOL(pci_wait_for_pending_transaction);
4440
4441	/**
4442	* pcie_flr - initiate a PCIe function level reset
4443	* @dev: device to reset
4444	*
4445	* Initiate a function level reset unconditionally on @dev without
4446	* checking any flags and DEVCAP
4447	*/
4448	int pcie_flr(struct pci_dev *dev)
4449	{
4450	if (!pci_wait_for_pending_transaction(dev))
4451	pci_err(dev, "timed out waiting for pending transaction; performing function level reset anyway\n");
4452
4453	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR);
4454
4455	if (dev->imm_ready)
4456	return 0;
4457
4458	/*
4459	* Per PCIe r4.0, sec 6.6.2, a device must complete an FLR within
4460	* 100ms, but may silently discard requests while the FLR is in
4461	* progress. Wait 100ms before trying to access the device.
4462	*/
4463	msleep(msecs: 100);
4464
4465	return pci_dev_wait(dev, reset_type: "FLR", PCIE_RESET_READY_POLL_MS);
4466	}
4467	EXPORT_SYMBOL_GPL(pcie_flr);
4468
4469	/**
4470	* pcie_reset_flr - initiate a PCIe function level reset
4471	* @dev: device to reset
4472	* @probe: if true, return 0 if device can be reset this way
4473	*
4474	* Initiate a function level reset on @dev.
4475	*/
4476	int pcie_reset_flr(struct pci_dev *dev, bool probe)
4477	{
4478	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4479	return -ENOTTY;
4480
4481	if (!(dev->devcap & PCI_EXP_DEVCAP_FLR))
4482	return -ENOTTY;
4483
4484	if (probe)
4485	return 0;
4486
4487	return pcie_flr(dev);
4488	}
4489	EXPORT_SYMBOL_GPL(pcie_reset_flr);
4490
4491	static int pci_af_flr(struct pci_dev *dev, bool probe)
4492	{
4493	int pos;
4494	u8 cap;
4495
4496	pos = pci_find_capability(dev, PCI_CAP_ID_AF);
4497	if (!pos)
4498	return -ENOTTY;
4499
4500	if (dev->dev_flags & PCI_DEV_FLAGS_NO_FLR_RESET)
4501	return -ENOTTY;
4502
4503	pci_read_config_byte(dev, where: pos + PCI_AF_CAP, val: &cap);
4504	if (!(cap & PCI_AF_CAP_TP) || !(cap & PCI_AF_CAP_FLR))
4505	return -ENOTTY;
4506
4507	if (probe)
4508	return 0;
4509
4510	/*
4511	* Wait for Transaction Pending bit to clear. A word-aligned test
4512	* is used, so we use the control offset rather than status and shift
4513	* the test bit to match.
4514	*/
4515	if (!pci_wait_for_pending(dev, pos: pos + PCI_AF_CTRL,
4516	PCI_AF_STATUS_TP << 8))
4517	pci_err(dev, "timed out waiting for pending transaction; performing AF function level reset anyway\n");
4518
4519	pci_write_config_byte(dev, where: pos + PCI_AF_CTRL, PCI_AF_CTRL_FLR);
4520
4521	if (dev->imm_ready)
4522	return 0;
4523
4524	/*
4525	* Per Advanced Capabilities for Conventional PCI ECN, 13 April 2006,
4526	* updated 27 July 2006; a device must complete an FLR within
4527	* 100ms, but may silently discard requests while the FLR is in
4528	* progress. Wait 100ms before trying to access the device.
4529	*/
4530	msleep(msecs: 100);
4531
4532	return pci_dev_wait(dev, reset_type: "AF_FLR", PCIE_RESET_READY_POLL_MS);
4533	}
4534
4535	/**
4536	* pci_pm_reset - Put device into PCI_D3 and back into PCI_D0.
4537	* @dev: Device to reset.
4538	* @probe: if true, return 0 if the device can be reset this way.
4539	*
4540	* If @dev supports native PCI PM and its PCI_PM_CTRL_NO_SOFT_RESET flag is
4541	* unset, it will be reinitialized internally when going from PCI_D3hot to
4542	* PCI_D0. If that's the case and the device is not in a low-power state
4543	* already, force it into PCI_D3hot and back to PCI_D0, causing it to be reset.
4544	*
4545	* NOTE: This causes the caller to sleep for twice the device power transition
4546	* cooldown period, which for the D0->D3hot and D3hot->D0 transitions is 10 ms
4547	* by default (i.e. unless the @dev's d3hot_delay field has a different value).
4548	* Moreover, only devices in D0 can be reset by this function.
4549	*/
4550	static int pci_pm_reset(struct pci_dev *dev, bool probe)
4551	{
4552	u16 csr;
4553
4554	if (!dev->pm_cap || dev->dev_flags & PCI_DEV_FLAGS_NO_PM_RESET)
4555	return -ENOTTY;
4556
4557	pci_read_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: &csr);
4558	if (csr & PCI_PM_CTRL_NO_SOFT_RESET)
4559	return -ENOTTY;
4560
4561	if (probe)
4562	return 0;
4563
4564	if (dev->current_state != PCI_D0)
4565	return -EINVAL;
4566
4567	csr &= ~PCI_PM_CTRL_STATE_MASK;
4568	csr |= PCI_D3hot;
4569	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: csr);
4570	pci_dev_d3_sleep(dev);
4571
4572	csr &= ~PCI_PM_CTRL_STATE_MASK;
4573	csr |= PCI_D0;
4574	pci_write_config_word(dev, where: dev->pm_cap + PCI_PM_CTRL, val: csr);
4575	pci_dev_d3_sleep(dev);
4576
4577	return pci_dev_wait(dev, reset_type: "PM D3hot->D0", PCIE_RESET_READY_POLL_MS);
4578	}
4579
4580	/**
4581	* pcie_wait_for_link_status - Wait for link status change
4582	* @pdev: Device whose link to wait for.
4583	* @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE.
4584	* @active: Waiting for active or inactive?
4585	*
4586	* Return 0 if successful, or -ETIMEDOUT if status has not changed within
4587	* PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds.
4588	*/
4589	static int pcie_wait_for_link_status(struct pci_dev *pdev,
4590	bool use_lt, bool active)
4591	{
4592	u16 lnksta_mask, lnksta_match;
4593	unsigned long end_jiffies;
4594	u16 lnksta;
4595
4596	lnksta_mask = use_lt ? PCI_EXP_LNKSTA_LT : PCI_EXP_LNKSTA_DLLLA;
4597	lnksta_match = active ? lnksta_mask : 0;
4598
4599	end_jiffies = jiffies + msecs_to_jiffies(PCIE_LINK_RETRAIN_TIMEOUT_MS);
4600	do {
4601	pcie_capability_read_word(dev: pdev, PCI_EXP_LNKSTA, val: &lnksta);
4602	if ((lnksta & lnksta_mask) == lnksta_match)
4603	return 0;
4604	msleep(msecs: 1);
4605	} while (time_before(jiffies, end_jiffies));
4606
4607	return -ETIMEDOUT;
4608	}
4609
4610	/**
4611	* pcie_retrain_link - Request a link retrain and wait for it to complete
4612	* @pdev: Device whose link to retrain.
4613	* @use_lt: Use the LT bit if TRUE, or the DLLLA bit if FALSE, for status.
4614	*
4615	* Retrain completion status is retrieved from the Link Status Register
4616	* according to @use_lt. It is not verified whether the use of the DLLLA
4617	* bit is valid.
4618	*
4619	* Return 0 if successful, or -ETIMEDOUT if training has not completed
4620	* within PCIE_LINK_RETRAIN_TIMEOUT_MS milliseconds.
4621	*/
4622	int pcie_retrain_link(struct pci_dev *pdev, bool use_lt)
4623	{
4624	int rc;
4625
4626	/*
4627	* Ensure the updated LNKCTL parameters are used during link
4628	* training by checking that there is no ongoing link training to
4629	* avoid LTSSM race as recommended in Implementation Note at the
4630	* end of PCIe r6.0.1 sec 7.5.3.7.
4631	*/
4632	rc = pcie_wait_for_link_status(pdev, use_lt, active: !use_lt);
4633	if (rc)
4634	return rc;
4635
4636	pcie_capability_set_word(dev: pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL);
4637	if (pdev->clear_retrain_link) {
4638	/*
4639	* Due to an erratum in some devices the Retrain Link bit
4640	* needs to be cleared again manually to allow the link
4641	* training to succeed.
4642	*/
4643	pcie_capability_clear_word(dev: pdev, PCI_EXP_LNKCTL, PCI_EXP_LNKCTL_RL);
4644	}
4645
4646	return pcie_wait_for_link_status(pdev, use_lt, active: !use_lt);
4647	}
4648
4649	/**
4650	* pcie_wait_for_link_delay - Wait until link is active or inactive
4651	* @pdev: Bridge device
4652	* @active: waiting for active or inactive?
4653	* @delay: Delay to wait after link has become active (in ms)
4654	*
4655	* Use this to wait till link becomes active or inactive.
4656	*/
4657	static bool pcie_wait_for_link_delay(struct pci_dev *pdev, bool active,
4658	int delay)
4659	{
4660	int rc;
4661
4662	/*
4663	* Some controllers might not implement link active reporting. In this
4664	* case, we wait for 1000 ms + any delay requested by the caller.
4665	*/
4666	if (!pdev->link_active_reporting) {
4667	msleep(PCIE_LINK_RETRAIN_TIMEOUT_MS + delay);
4668	return true;
4669	}
4670
4671	/*
4672	* PCIe r4.0 sec 6.6.1, a component must enter LTSSM Detect within 20ms,
4673	* after which we should expect an link active if the reset was
4674	* successful. If so, software must wait a minimum 100ms before sending
4675	* configuration requests to devices downstream this port.
4676	*
4677	* If the link fails to activate, either the device was physically
4678	* removed or the link is permanently failed.
4679	*/
4680	if (active)
4681	msleep(msecs: 20);
4682	rc = pcie_wait_for_link_status(pdev, use_lt: false, active);
4683	if (active) {
4684	if (rc)
4685	rc = pcie_failed_link_retrain(dev: pdev);
4686	if (rc)
4687	return false;
4688
4689	msleep(msecs: delay);
4690	return true;
4691	}
4692
4693	if (rc)
4694	return false;
4695
4696	return true;
4697	}
4698
4699	/**
4700	* pcie_wait_for_link - Wait until link is active or inactive
4701	* @pdev: Bridge device
4702	* @active: waiting for active or inactive?
4703	*
4704	* Use this to wait till link becomes active or inactive.
4705	*/
4706	bool pcie_wait_for_link(struct pci_dev *pdev, bool active)
4707	{
4708	return pcie_wait_for_link_delay(pdev, active, delay: 100);
4709	}
4710
4711	/*
4712	* Find maximum D3cold delay required by all the devices on the bus. The
4713	* spec says 100 ms, but firmware can lower it and we allow drivers to
4714	* increase it as well.
4715	*
4716	* Called with @pci_bus_sem locked for reading.
4717	*/
4718	static int pci_bus_max_d3cold_delay(const struct pci_bus *bus)
4719	{
4720	const struct pci_dev *pdev;
4721	int min_delay = 100;
4722	int max_delay = 0;
4723
4724	list_for_each_entry(pdev, &bus->devices, bus_list) {
4725	if (pdev->d3cold_delay < min_delay)
4726	min_delay = pdev->d3cold_delay;
4727	if (pdev->d3cold_delay > max_delay)
4728	max_delay = pdev->d3cold_delay;
4729	}
4730
4731	return max(min_delay, max_delay);
4732	}
4733
4734	/**
4735	* pci_bridge_wait_for_secondary_bus - Wait for secondary bus to be accessible
4736	* @dev: PCI bridge
4737	* @reset_type: reset type in human-readable form
4738	*
4739	* Handle necessary delays before access to the devices on the secondary
4740	* side of the bridge are permitted after D3cold to D0 transition
4741	* or Conventional Reset.
4742	*
4743	* For PCIe this means the delays in PCIe 5.0 section 6.6.1. For
4744	* conventional PCI it means Tpvrh + Trhfa specified in PCI 3.0 section
4745	* 4.3.2.
4746	*
4747	* Return 0 on success or -ENOTTY if the first device on the secondary bus
4748	* failed to become accessible.
4749	*/
4750	int pci_bridge_wait_for_secondary_bus(struct pci_dev *dev, char *reset_type)
4751	{
4752	struct pci_dev *child;
4753	int delay;
4754
4755	if (pci_dev_is_disconnected(dev))
4756	return 0;
4757
4758	if (!pci_is_bridge(dev))
4759	return 0;
4760
4761	down_read(sem: &pci_bus_sem);
4762
4763	/*
4764	* We only deal with devices that are present currently on the bus.
4765	* For any hot-added devices the access delay is handled in pciehp
4766	* board_added(). In case of ACPI hotplug the firmware is expected
4767	* to configure the devices before OS is notified.
4768	*/
4769	if (!dev->subordinate || list_empty(head: &dev->subordinate->devices)) {
4770	up_read(sem: &pci_bus_sem);
4771	return 0;
4772	}
4773
4774	/* Take d3cold_delay requirements into account */
4775	delay = pci_bus_max_d3cold_delay(bus: dev->subordinate);
4776	if (!delay) {
4777	up_read(sem: &pci_bus_sem);
4778	return 0;
4779	}
4780
4781	child = list_first_entry(&dev->subordinate->devices, struct pci_dev,
4782	bus_list);
4783	up_read(sem: &pci_bus_sem);
4784
4785	/*
4786	* Conventional PCI and PCI-X we need to wait Tpvrh + Trhfa before
4787	* accessing the device after reset (that is 1000 ms + 100 ms).
4788	*/
4789	if (!pci_is_pcie(dev)) {
4790	pci_dbg(dev, "waiting %d ms for secondary bus\n", 1000 + delay);
4791	msleep(msecs: 1000 + delay);
4792	return 0;
4793	}
4794
4795	/*
4796	* For PCIe downstream and root ports that do not support speeds
4797	* greater than 5 GT/s need to wait minimum 100 ms. For higher
4798	* speeds (gen3) we need to wait first for the data link layer to
4799	* become active.
4800	*
4801	* However, 100 ms is the minimum and the PCIe spec says the
4802	* software must allow at least 1s before it can determine that the
4803	* device that did not respond is a broken device. Also device can
4804	* take longer than that to respond if it indicates so through Request
4805	* Retry Status completions.
4806	*
4807	* Therefore we wait for 100 ms and check for the device presence
4808	* until the timeout expires.
4809	*/
4810	if (!pcie_downstream_port(dev))
4811	return 0;
4812
4813	if (pcie_get_speed_cap(dev) <= PCIE_SPEED_5_0GT) {
4814	u16 status;
4815
4816	pci_dbg(dev, "waiting %d ms for downstream link\n", delay);
4817	msleep(msecs: delay);
4818
4819	if (!pci_dev_wait(dev: child, reset_type, PCI_RESET_WAIT - delay))
4820	return 0;
4821
4822	/*
4823	* If the port supports active link reporting we now check
4824	* whether the link is active and if not bail out early with
4825	* the assumption that the device is not present anymore.
4826	*/
4827	if (!dev->link_active_reporting)
4828	return -ENOTTY;
4829
4830	pcie_capability_read_word(dev, PCI_EXP_LNKSTA, val: &status);
4831	if (!(status & PCI_EXP_LNKSTA_DLLLA))
4832	return -ENOTTY;
4833
4834	return pci_dev_wait(dev: child, reset_type,
4835	PCIE_RESET_READY_POLL_MS - PCI_RESET_WAIT);
4836	}
4837
4838	pci_dbg(dev, "waiting %d ms for downstream link, after activation\n",
4839	delay);
4840	if (!pcie_wait_for_link_delay(pdev: dev, active: true, delay)) {
4841	/* Did not train, no need to wait any further */
4842	pci_info(dev, "Data Link Layer Link Active not set in 1000 msec\n");
4843	return -ENOTTY;
4844	}
4845
4846	return pci_dev_wait(dev: child, reset_type,
4847	PCIE_RESET_READY_POLL_MS - delay);
4848	}
4849
4850	void pci_reset_secondary_bus(struct pci_dev *dev)
4851	{
4852	u16 ctrl;
4853
4854	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, val: &ctrl);
4855	ctrl |= PCI_BRIDGE_CTL_BUS_RESET;
4856	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, val: ctrl);
4857
4858	/*
4859	* PCI spec v3.0 7.6.4.2 requires minimum Trst of 1ms. Double
4860	* this to 2ms to ensure that we meet the minimum requirement.
4861	*/
4862	msleep(msecs: 2);
4863
4864	ctrl &= ~PCI_BRIDGE_CTL_BUS_RESET;
4865	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, val: ctrl);
4866	}
4867
4868	void __weak pcibios_reset_secondary_bus(struct pci_dev *dev)
4869	{
4870	pci_reset_secondary_bus(dev);
4871	}
4872
4873	/**
4874	* pci_bridge_secondary_bus_reset - Reset the secondary bus on a PCI bridge.
4875	* @dev: Bridge device
4876	*
4877	* Use the bridge control register to assert reset on the secondary bus.
4878	* Devices on the secondary bus are left in power-on state.
4879	*/
4880	int pci_bridge_secondary_bus_reset(struct pci_dev *dev)
4881	{
4882	pcibios_reset_secondary_bus(dev);
4883
4884	return pci_bridge_wait_for_secondary_bus(dev, reset_type: "bus reset");
4885	}
4886	EXPORT_SYMBOL_GPL(pci_bridge_secondary_bus_reset);
4887
4888	static int pci_parent_bus_reset(struct pci_dev *dev, bool probe)
4889	{
4890	struct pci_dev *pdev;
4891
4892	if (pci_is_root_bus(pbus: dev->bus) || dev->subordinate ||
4893	!dev->bus->self || dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
4894	return -ENOTTY;
4895
4896	list_for_each_entry(pdev, &dev->bus->devices, bus_list)
4897	if (pdev != dev)
4898	return -ENOTTY;
4899
4900	if (probe)
4901	return 0;
4902
4903	return pci_bridge_secondary_bus_reset(dev->bus->self);
4904	}
4905
4906	static int pci_reset_hotplug_slot(struct hotplug_slot *hotplug, bool probe)
4907	{
4908	int rc = -ENOTTY;
4909
4910	if (!hotplug || !try_module_get(module: hotplug->owner))
4911	return rc;
4912
4913	if (hotplug->ops->reset_slot)
4914	rc = hotplug->ops->reset_slot(hotplug, probe);
4915
4916	module_put(module: hotplug->owner);
4917
4918	return rc;
4919	}
4920
4921	static int pci_dev_reset_slot_function(struct pci_dev *dev, bool probe)
4922	{
4923	if (dev->multifunction || dev->subordinate || !dev->slot ||
4924	dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET)
4925	return -ENOTTY;
4926
4927	return pci_reset_hotplug_slot(hotplug: dev->slot->hotplug, probe);
4928	}
4929
4930	static int pci_reset_bus_function(struct pci_dev *dev, bool probe)
4931	{
4932	int rc;
4933
4934	rc = pci_dev_reset_slot_function(dev, probe);
4935	if (rc != -ENOTTY)
4936	return rc;
4937	return pci_parent_bus_reset(dev, probe);
4938	}
4939
4940	void pci_dev_lock(struct pci_dev *dev)
4941	{
4942	/* block PM suspend, driver probe, etc. */
4943	device_lock(dev: &dev->dev);
4944	pci_cfg_access_lock(dev);
4945	}
4946	EXPORT_SYMBOL_GPL(pci_dev_lock);
4947
4948	/* Return 1 on successful lock, 0 on contention */
4949	int pci_dev_trylock(struct pci_dev *dev)
4950	{
4951	if (device_trylock(dev: &dev->dev)) {
4952	if (pci_cfg_access_trylock(dev))
4953	return 1;
4954	device_unlock(dev: &dev->dev);
4955	}
4956
4957	return 0;
4958	}
4959	EXPORT_SYMBOL_GPL(pci_dev_trylock);
4960
4961	void pci_dev_unlock(struct pci_dev *dev)
4962	{
4963	pci_cfg_access_unlock(dev);
4964	device_unlock(dev: &dev->dev);
4965	}
4966	EXPORT_SYMBOL_GPL(pci_dev_unlock);
4967
4968	static void pci_dev_save_and_disable(struct pci_dev *dev)
4969	{
4970	const struct pci_error_handlers *err_handler =
4971	dev->driver ? dev->driver->err_handler : NULL;
4972
4973	/*
4974	* dev->driver->err_handler->reset_prepare() is protected against
4975	* races with ->remove() by the device lock, which must be held by
4976	* the caller.
4977	*/
4978	if (err_handler && err_handler->reset_prepare)
4979	err_handler->reset_prepare(dev);
4980
4981	/*
4982	* Wake-up device prior to save. PM registers default to D0 after
4983	* reset and a simple register restore doesn't reliably return
4984	* to a non-D0 state anyway.
4985	*/
4986	pci_set_power_state(dev, PCI_D0);
4987
4988	pci_save_state(dev);
4989	/*
4990	* Disable the device by clearing the Command register, except for
4991	* INTx-disable which is set. This not only disables MMIO and I/O port
4992	* BARs, but also prevents the device from being Bus Master, preventing
4993	* DMA from the device including MSI/MSI-X interrupts. For PCI 2.3
4994	* compliant devices, INTx-disable prevents legacy interrupts.
4995	*/
4996	pci_write_config_word(dev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE);
4997	}
4998
4999	static void pci_dev_restore(struct pci_dev *dev)
5000	{
5001	const struct pci_error_handlers *err_handler =
5002	dev->driver ? dev->driver->err_handler : NULL;
5003
5004	pci_restore_state(dev);
5005
5006	/*
5007	* dev->driver->err_handler->reset_done() is protected against
5008	* races with ->remove() by the device lock, which must be held by
5009	* the caller.
5010	*/
5011	if (err_handler && err_handler->reset_done)
5012	err_handler->reset_done(dev);
5013	}
5014
5015	/* dev->reset_methods[] is a 0-terminated list of indices into this array */
5016	static const struct pci_reset_fn_method pci_reset_fn_methods[] = {
5017	{ },
5018	{ pci_dev_specific_reset, .name = "device_specific" },
5019	{ pci_dev_acpi_reset, .name = "acpi" },
5020	{ pcie_reset_flr, .name = "flr" },
5021	{ pci_af_flr, .name = "af_flr" },
5022	{ pci_pm_reset, .name = "pm" },
5023	{ pci_reset_bus_function, .name = "bus" },
5024	};
5025
5026	static ssize_t reset_method_show(struct device *dev,
5027	struct device_attribute *attr, char *buf)
5028	{
5029	struct pci_dev *pdev = to_pci_dev(dev);
5030	ssize_t len = 0;
5031	int i, m;
5032
5033	for (i = 0; i < PCI_NUM_RESET_METHODS; i++) {
5034	m = pdev->reset_methods[i];
5035	if (!m)
5036	break;
5037
5038	len += sysfs_emit_at(buf, at: len, fmt: "%s%s", len ? " " : "",
5039	pci_reset_fn_methods[m].name);
5040	}
5041
5042	if (len)
5043	len += sysfs_emit_at(buf, at: len, fmt: "\n");
5044
5045	return len;
5046	}
5047
5048	static int reset_method_lookup(const char *name)
5049	{
5050	int m;
5051
5052	for (m = 1; m < PCI_NUM_RESET_METHODS; m++) {
5053	if (sysfs_streq(s1: name, s2: pci_reset_fn_methods[m].name))
5054	return m;
5055	}
5056
5057	return 0; /* not found */
5058	}
5059
5060	static ssize_t reset_method_store(struct device *dev,
5061	struct device_attribute *attr,
5062	const char *buf, size_t count)
5063	{
5064	struct pci_dev *pdev = to_pci_dev(dev);
5065	char *options, *name;
5066	int m, n;
5067	u8 reset_methods[PCI_NUM_RESET_METHODS] = { 0 };
5068
5069	if (sysfs_streq(s1: buf, s2: "")) {
5070	pdev->reset_methods[0] = 0;
5071	pci_warn(pdev, "All device reset methods disabled by user");
5072	return count;
5073	}
5074
5075	if (sysfs_streq(s1: buf, s2: "default")) {
5076	pci_init_reset_methods(dev: pdev);
5077	return count;
5078	}
5079
5080	options = kstrndup(s: buf, len: count, GFP_KERNEL);
5081	if (!options)
5082	return -ENOMEM;
5083
5084	n = 0;
5085	while ((name = strsep(&options, " ")) != NULL) {
5086	if (sysfs_streq(s1: name, s2: ""))
5087	continue;
5088
5089	name = strim(name);
5090
5091	m = reset_method_lookup(name);
5092	if (!m) {
5093	pci_err(pdev, "Invalid reset method '%s'", name);
5094	goto error;
5095	}
5096
5097	if (pci_reset_fn_methods[m].reset_fn(pdev, PCI_RESET_PROBE)) {
5098	pci_err(pdev, "Unsupported reset method '%s'", name);
5099	goto error;
5100	}
5101
5102	if (n == PCI_NUM_RESET_METHODS - 1) {
5103	pci_err(pdev, "Too many reset methods\n");
5104	goto error;
5105	}
5106
5107	reset_methods[n++] = m;
5108	}
5109
5110	reset_methods[n] = 0;
5111
5112	/* Warn if dev-specific supported but not highest priority */
5113	if (pci_reset_fn_methods[1].reset_fn(pdev, PCI_RESET_PROBE) == 0 &&
5114	reset_methods[0] != 1)
5115	pci_warn(pdev, "Device-specific reset disabled/de-prioritized by user");
5116	memcpy(pdev->reset_methods, reset_methods, sizeof(pdev->reset_methods));
5117	kfree(objp: options);
5118	return count;
5119
5120	error:
5121	/* Leave previous methods unchanged */
5122	kfree(objp: options);
5123	return -EINVAL;
5124	}
5125	static DEVICE_ATTR_RW(reset_method);
5126
5127	static struct attribute *pci_dev_reset_method_attrs[] = {
5128	&dev_attr_reset_method.attr,
5129	NULL,
5130	};
5131
5132	static umode_t pci_dev_reset_method_attr_is_visible(struct kobject *kobj,
5133	struct attribute *a, int n)
5134	{
5135	struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj));
5136
5137	if (!pci_reset_supported(dev: pdev))
5138	return 0;
5139
5140	return a->mode;
5141	}
5142
5143	const struct attribute_group pci_dev_reset_method_attr_group = {
5144	.attrs = pci_dev_reset_method_attrs,
5145	.is_visible = pci_dev_reset_method_attr_is_visible,
5146	};
5147
5148	/**
5149	* __pci_reset_function_locked - reset a PCI device function while holding
5150	* the @dev mutex lock.
5151	* @dev: PCI device to reset
5152	*
5153	* Some devices allow an individual function to be reset without affecting
5154	* other functions in the same device. The PCI device must be responsive
5155	* to PCI config space in order to use this function.
5156	*
5157	* The device function is presumed to be unused and the caller is holding
5158	* the device mutex lock when this function is called.
5159	*
5160	* Resetting the device will make the contents of PCI configuration space
5161	* random, so any caller of this must be prepared to reinitialise the
5162	* device including MSI, bus mastering, BARs, decoding IO and memory spaces,
5163	* etc.
5164	*
5165	* Returns 0 if the device function was successfully reset or negative if the
5166	* device doesn't support resetting a single function.
5167	*/
5168	int __pci_reset_function_locked(struct pci_dev *dev)
5169	{
5170	int i, m, rc;
5171
5172	might_sleep();
5173
5174	/*
5175	* A reset method returns -ENOTTY if it doesn't support this device and
5176	* we should try the next method.
5177	*
5178	* If it returns 0 (success), we're finished. If it returns any other
5179	* error, we're also finished: this indicates that further reset
5180	* mechanisms might be broken on the device.
5181	*/
5182	for (i = 0; i < PCI_NUM_RESET_METHODS; i++) {
5183	m = dev->reset_methods[i];
5184	if (!m)
5185	return -ENOTTY;
5186
5187	rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_DO_RESET);
5188	if (!rc)
5189	return 0;
5190	if (rc != -ENOTTY)
5191	return rc;
5192	}
5193
5194	return -ENOTTY;
5195	}
5196	EXPORT_SYMBOL_GPL(__pci_reset_function_locked);
5197
5198	/**
5199	* pci_init_reset_methods - check whether device can be safely reset
5200	* and store supported reset mechanisms.
5201	* @dev: PCI device to check for reset mechanisms
5202	*
5203	* Some devices allow an individual function to be reset without affecting
5204	* other functions in the same device. The PCI device must be in D0-D3hot
5205	* state.
5206	*
5207	* Stores reset mechanisms supported by device in reset_methods byte array
5208	* which is a member of struct pci_dev.
5209	*/
5210	void pci_init_reset_methods(struct pci_dev *dev)
5211	{
5212	int m, i, rc;
5213
5214	BUILD_BUG_ON(ARRAY_SIZE(pci_reset_fn_methods) != PCI_NUM_RESET_METHODS);
5215
5216	might_sleep();
5217
5218	i = 0;
5219	for (m = 1; m < PCI_NUM_RESET_METHODS; m++) {
5220	rc = pci_reset_fn_methods[m].reset_fn(dev, PCI_RESET_PROBE);
5221	if (!rc)
5222	dev->reset_methods[i++] = m;
5223	else if (rc != -ENOTTY)
5224	break;
5225	}
5226
5227	dev->reset_methods[i] = 0;
5228	}
5229
5230	/**
5231	* pci_reset_function - quiesce and reset a PCI device function
5232	* @dev: PCI device to reset
5233	*
5234	* Some devices allow an individual function to be reset without affecting
5235	* other functions in the same device. The PCI device must be responsive
5236	* to PCI config space in order to use this function.
5237	*
5238	* This function does not just reset the PCI portion of a device, but
5239	* clears all the state associated with the device. This function differs
5240	* from __pci_reset_function_locked() in that it saves and restores device state
5241	* over the reset and takes the PCI device lock.
5242	*
5243	* Returns 0 if the device function was successfully reset or negative if the
5244	* device doesn't support resetting a single function.
5245	*/
5246	int pci_reset_function(struct pci_dev *dev)
5247	{
5248	int rc;
5249
5250	if (!pci_reset_supported(dev))
5251	return -ENOTTY;
5252
5253	pci_dev_lock(dev);
5254	pci_dev_save_and_disable(dev);
5255
5256	rc = __pci_reset_function_locked(dev);
5257
5258	pci_dev_restore(dev);
5259	pci_dev_unlock(dev);
5260
5261	return rc;
5262	}
5263	EXPORT_SYMBOL_GPL(pci_reset_function);
5264
5265	/**
5266	* pci_reset_function_locked - quiesce and reset a PCI device function
5267	* @dev: PCI device to reset
5268	*
5269	* Some devices allow an individual function to be reset without affecting
5270	* other functions in the same device. The PCI device must be responsive
5271	* to PCI config space in order to use this function.
5272	*
5273	* This function does not just reset the PCI portion of a device, but
5274	* clears all the state associated with the device. This function differs
5275	* from __pci_reset_function_locked() in that it saves and restores device state
5276	* over the reset. It also differs from pci_reset_function() in that it
5277	* requires the PCI device lock to be held.
5278	*
5279	* Returns 0 if the device function was successfully reset or negative if the
5280	* device doesn't support resetting a single function.
5281	*/
5282	int pci_reset_function_locked(struct pci_dev *dev)
5283	{
5284	int rc;
5285
5286	if (!pci_reset_supported(dev))
5287	return -ENOTTY;
5288
5289	pci_dev_save_and_disable(dev);
5290
5291	rc = __pci_reset_function_locked(dev);
5292
5293	pci_dev_restore(dev);
5294
5295	return rc;
5296	}
5297	EXPORT_SYMBOL_GPL(pci_reset_function_locked);
5298
5299	/**
5300	* pci_try_reset_function - quiesce and reset a PCI device function
5301	* @dev: PCI device to reset
5302	*
5303	* Same as above, except return -EAGAIN if unable to lock device.
5304	*/
5305	int pci_try_reset_function(struct pci_dev *dev)
5306	{
5307	int rc;
5308
5309	if (!pci_reset_supported(dev))
5310	return -ENOTTY;
5311
5312	if (!pci_dev_trylock(dev))
5313	return -EAGAIN;
5314
5315	pci_dev_save_and_disable(dev);
5316	rc = __pci_reset_function_locked(dev);
5317	pci_dev_restore(dev);
5318	pci_dev_unlock(dev);
5319
5320	return rc;
5321	}
5322	EXPORT_SYMBOL_GPL(pci_try_reset_function);
5323
5324	/* Do any devices on or below this bus prevent a bus reset? */
5325	static bool pci_bus_resettable(struct pci_bus *bus)
5326	{
5327	struct pci_dev *dev;
5328
5329
5330	if (bus->self && (bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5331	return false;
5332
5333	list_for_each_entry(dev, &bus->devices, bus_list) {
5334	if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5335	(dev->subordinate && !pci_bus_resettable(bus: dev->subordinate)))
5336	return false;
5337	}
5338
5339	return true;
5340	}
5341
5342	/* Lock devices from the top of the tree down */
5343	static void pci_bus_lock(struct pci_bus *bus)
5344	{
5345	struct pci_dev *dev;
5346
5347	list_for_each_entry(dev, &bus->devices, bus_list) {
5348	pci_dev_lock(dev);
5349	if (dev->subordinate)
5350	pci_bus_lock(bus: dev->subordinate);
5351	}
5352	}
5353
5354	/* Unlock devices from the bottom of the tree up */
5355	static void pci_bus_unlock(struct pci_bus *bus)
5356	{
5357	struct pci_dev *dev;
5358
5359	list_for_each_entry(dev, &bus->devices, bus_list) {
5360	if (dev->subordinate)
5361	pci_bus_unlock(bus: dev->subordinate);
5362	pci_dev_unlock(dev);
5363	}
5364	}
5365
5366	/* Return 1 on successful lock, 0 on contention */
5367	static int pci_bus_trylock(struct pci_bus *bus)
5368	{
5369	struct pci_dev *dev;
5370
5371	list_for_each_entry(dev, &bus->devices, bus_list) {
5372	if (!pci_dev_trylock(dev))
5373	goto unlock;
5374	if (dev->subordinate) {
5375	if (!pci_bus_trylock(bus: dev->subordinate)) {
5376	pci_dev_unlock(dev);
5377	goto unlock;
5378	}
5379	}
5380	}
5381	return 1;
5382
5383	unlock:
5384	list_for_each_entry_continue_reverse(dev, &bus->devices, bus_list) {
5385	if (dev->subordinate)
5386	pci_bus_unlock(bus: dev->subordinate);
5387	pci_dev_unlock(dev);
5388	}
5389	return 0;
5390	}
5391
5392	/* Do any devices on or below this slot prevent a bus reset? */
5393	static bool pci_slot_resettable(struct pci_slot *slot)
5394	{
5395	struct pci_dev *dev;
5396
5397	if (slot->bus->self &&
5398	(slot->bus->self->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET))
5399	return false;
5400
5401	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5402	if (!dev->slot || dev->slot != slot)
5403	continue;
5404	if (dev->dev_flags & PCI_DEV_FLAGS_NO_BUS_RESET ||
5405	(dev->subordinate && !pci_bus_resettable(bus: dev->subordinate)))
5406	return false;
5407	}
5408
5409	return true;
5410	}
5411
5412	/* Lock devices from the top of the tree down */
5413	static void pci_slot_lock(struct pci_slot *slot)
5414	{
5415	struct pci_dev *dev;
5416
5417	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5418	if (!dev->slot || dev->slot != slot)
5419	continue;
5420	pci_dev_lock(dev);
5421	if (dev->subordinate)
5422	pci_bus_lock(bus: dev->subordinate);
5423	}
5424	}
5425
5426	/* Unlock devices from the bottom of the tree up */
5427	static void pci_slot_unlock(struct pci_slot *slot)
5428	{
5429	struct pci_dev *dev;
5430
5431	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5432	if (!dev->slot || dev->slot != slot)
5433	continue;
5434	if (dev->subordinate)
5435	pci_bus_unlock(bus: dev->subordinate);
5436	pci_dev_unlock(dev);
5437	}
5438	}
5439
5440	/* Return 1 on successful lock, 0 on contention */
5441	static int pci_slot_trylock(struct pci_slot *slot)
5442	{
5443	struct pci_dev *dev;
5444
5445	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5446	if (!dev->slot || dev->slot != slot)
5447	continue;
5448	if (!pci_dev_trylock(dev))
5449	goto unlock;
5450	if (dev->subordinate) {
5451	if (!pci_bus_trylock(bus: dev->subordinate)) {
5452	pci_dev_unlock(dev);
5453	goto unlock;
5454	}
5455	}
5456	}
5457	return 1;
5458
5459	unlock:
5460	list_for_each_entry_continue_reverse(dev,
5461	&slot->bus->devices, bus_list) {
5462	if (!dev->slot || dev->slot != slot)
5463	continue;
5464	if (dev->subordinate)
5465	pci_bus_unlock(bus: dev->subordinate);
5466	pci_dev_unlock(dev);
5467	}
5468	return 0;
5469	}
5470
5471	/*
5472	* Save and disable devices from the top of the tree down while holding
5473	* the @dev mutex lock for the entire tree.
5474	*/
5475	static void pci_bus_save_and_disable_locked(struct pci_bus *bus)
5476	{
5477	struct pci_dev *dev;
5478
5479	list_for_each_entry(dev, &bus->devices, bus_list) {
5480	pci_dev_save_and_disable(dev);
5481	if (dev->subordinate)
5482	pci_bus_save_and_disable_locked(bus: dev->subordinate);
5483	}
5484	}
5485
5486	/*
5487	* Restore devices from top of the tree down while holding @dev mutex lock
5488	* for the entire tree. Parent bridges need to be restored before we can
5489	* get to subordinate devices.
5490	*/
5491	static void pci_bus_restore_locked(struct pci_bus *bus)
5492	{
5493	struct pci_dev *dev;
5494
5495	list_for_each_entry(dev, &bus->devices, bus_list) {
5496	pci_dev_restore(dev);
5497	if (dev->subordinate)
5498	pci_bus_restore_locked(bus: dev->subordinate);
5499	}
5500	}
5501
5502	/*
5503	* Save and disable devices from the top of the tree down while holding
5504	* the @dev mutex lock for the entire tree.
5505	*/
5506	static void pci_slot_save_and_disable_locked(struct pci_slot *slot)
5507	{
5508	struct pci_dev *dev;
5509
5510	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5511	if (!dev->slot || dev->slot != slot)
5512	continue;
5513	pci_dev_save_and_disable(dev);
5514	if (dev->subordinate)
5515	pci_bus_save_and_disable_locked(bus: dev->subordinate);
5516	}
5517	}
5518
5519	/*
5520	* Restore devices from top of the tree down while holding @dev mutex lock
5521	* for the entire tree. Parent bridges need to be restored before we can
5522	* get to subordinate devices.
5523	*/
5524	static void pci_slot_restore_locked(struct pci_slot *slot)
5525	{
5526	struct pci_dev *dev;
5527
5528	list_for_each_entry(dev, &slot->bus->devices, bus_list) {
5529	if (!dev->slot || dev->slot != slot)
5530	continue;
5531	pci_dev_restore(dev);
5532	if (dev->subordinate)
5533	pci_bus_restore_locked(bus: dev->subordinate);
5534	}
5535	}
5536
5537	static int pci_slot_reset(struct pci_slot *slot, bool probe)
5538	{
5539	int rc;
5540
5541	if (!slot || !pci_slot_resettable(slot))
5542	return -ENOTTY;
5543
5544	if (!probe)
5545	pci_slot_lock(slot);
5546
5547	might_sleep();
5548
5549	rc = pci_reset_hotplug_slot(hotplug: slot->hotplug, probe);
5550
5551	if (!probe)
5552	pci_slot_unlock(slot);
5553
5554	return rc;
5555	}
5556
5557	/**
5558	* pci_probe_reset_slot - probe whether a PCI slot can be reset
5559	* @slot: PCI slot to probe
5560	*
5561	* Return 0 if slot can be reset, negative if a slot reset is not supported.
5562	*/
5563	int pci_probe_reset_slot(struct pci_slot *slot)
5564	{
5565	return pci_slot_reset(slot, PCI_RESET_PROBE);
5566	}
5567	EXPORT_SYMBOL_GPL(pci_probe_reset_slot);
5568
5569	/**
5570	* __pci_reset_slot - Try to reset a PCI slot
5571	* @slot: PCI slot to reset
5572	*
5573	* A PCI bus may host multiple slots, each slot may support a reset mechanism
5574	* independent of other slots. For instance, some slots may support slot power
5575	* control. In the case of a 1:1 bus to slot architecture, this function may
5576	* wrap the bus reset to avoid spurious slot related events such as hotplug.
5577	* Generally a slot reset should be attempted before a bus reset. All of the
5578	* function of the slot and any subordinate buses behind the slot are reset
5579	* through this function. PCI config space of all devices in the slot and
5580	* behind the slot is saved before and restored after reset.
5581	*
5582	* Same as above except return -EAGAIN if the slot cannot be locked
5583	*/
5584	static int __pci_reset_slot(struct pci_slot *slot)
5585	{
5586	int rc;
5587
5588	rc = pci_slot_reset(slot, PCI_RESET_PROBE);
5589	if (rc)
5590	return rc;
5591
5592	if (pci_slot_trylock(slot)) {
5593	pci_slot_save_and_disable_locked(slot);
5594	might_sleep();
5595	rc = pci_reset_hotplug_slot(hotplug: slot->hotplug, PCI_RESET_DO_RESET);
5596	pci_slot_restore_locked(slot);
5597	pci_slot_unlock(slot);
5598	} else
5599	rc = -EAGAIN;
5600
5601	return rc;
5602	}
5603
5604	static int pci_bus_reset(struct pci_bus *bus, bool probe)
5605	{
5606	int ret;
5607
5608	if (!bus->self || !pci_bus_resettable(bus))
5609	return -ENOTTY;
5610
5611	if (probe)
5612	return 0;
5613
5614	pci_bus_lock(bus);
5615
5616	might_sleep();
5617
5618	ret = pci_bridge_secondary_bus_reset(bus->self);
5619
5620	pci_bus_unlock(bus);
5621
5622	return ret;
5623	}
5624
5625	/**
5626	* pci_bus_error_reset - reset the bridge's subordinate bus
5627	* @bridge: The parent device that connects to the bus to reset
5628	*
5629	* This function will first try to reset the slots on this bus if the method is
5630	* available. If slot reset fails or is not available, this will fall back to a
5631	* secondary bus reset.
5632	*/
5633	int pci_bus_error_reset(struct pci_dev *bridge)
5634	{
5635	struct pci_bus *bus = bridge->subordinate;
5636	struct pci_slot *slot;
5637
5638	if (!bus)
5639	return -ENOTTY;
5640
5641	mutex_lock(&pci_slot_mutex);
5642	if (list_empty(head: &bus->slots))
5643	goto bus_reset;
5644
5645	list_for_each_entry(slot, &bus->slots, list)
5646	if (pci_probe_reset_slot(slot))
5647	goto bus_reset;
5648
5649	list_for_each_entry(slot, &bus->slots, list)
5650	if (pci_slot_reset(slot, PCI_RESET_DO_RESET))
5651	goto bus_reset;
5652
5653	mutex_unlock(lock: &pci_slot_mutex);
5654	return 0;
5655	bus_reset:
5656	mutex_unlock(lock: &pci_slot_mutex);
5657	return pci_bus_reset(bus: bridge->subordinate, PCI_RESET_DO_RESET);
5658	}
5659
5660	/**
5661	* pci_probe_reset_bus - probe whether a PCI bus can be reset
5662	* @bus: PCI bus to probe
5663	*
5664	* Return 0 if bus can be reset, negative if a bus reset is not supported.
5665	*/
5666	int pci_probe_reset_bus(struct pci_bus *bus)
5667	{
5668	return pci_bus_reset(bus, PCI_RESET_PROBE);
5669	}
5670	EXPORT_SYMBOL_GPL(pci_probe_reset_bus);
5671
5672	/**
5673	* __pci_reset_bus - Try to reset a PCI bus
5674	* @bus: top level PCI bus to reset
5675	*
5676	* Same as above except return -EAGAIN if the bus cannot be locked
5677	*/
5678	static int __pci_reset_bus(struct pci_bus *bus)
5679	{
5680	int rc;
5681
5682	rc = pci_bus_reset(bus, PCI_RESET_PROBE);
5683	if (rc)
5684	return rc;
5685
5686	if (pci_bus_trylock(bus)) {
5687	pci_bus_save_and_disable_locked(bus);
5688	might_sleep();
5689	rc = pci_bridge_secondary_bus_reset(bus->self);
5690	pci_bus_restore_locked(bus);
5691	pci_bus_unlock(bus);
5692	} else
5693	rc = -EAGAIN;
5694
5695	return rc;
5696	}
5697
5698	/**
5699	* pci_reset_bus - Try to reset a PCI bus
5700	* @pdev: top level PCI device to reset via slot/bus
5701	*
5702	* Same as above except return -EAGAIN if the bus cannot be locked
5703	*/
5704	int pci_reset_bus(struct pci_dev *pdev)
5705	{
5706	return (!pci_probe_reset_slot(pdev->slot)) ?
5707	__pci_reset_slot(slot: pdev->slot) : __pci_reset_bus(bus: pdev->bus);
5708	}
5709	EXPORT_SYMBOL_GPL(pci_reset_bus);
5710
5711	/**
5712	* pcix_get_max_mmrbc - get PCI-X maximum designed memory read byte count
5713	* @dev: PCI device to query
5714	*
5715	* Returns mmrbc: maximum designed memory read count in bytes or
5716	* appropriate error value.
5717	*/
5718	int pcix_get_max_mmrbc(struct pci_dev *dev)
5719	{
5720	int cap;
5721	u32 stat;
5722
5723	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5724	if (!cap)
5725	return -EINVAL;
5726
5727	if (pci_read_config_dword(dev, where: cap + PCI_X_STATUS, val: &stat))
5728	return -EINVAL;
5729
5730	return 512 << FIELD_GET(PCI_X_STATUS_MAX_READ, stat);
5731	}
5732	EXPORT_SYMBOL(pcix_get_max_mmrbc);
5733
5734	/**
5735	* pcix_get_mmrbc - get PCI-X maximum memory read byte count
5736	* @dev: PCI device to query
5737	*
5738	* Returns mmrbc: maximum memory read count in bytes or appropriate error
5739	* value.
5740	*/
5741	int pcix_get_mmrbc(struct pci_dev *dev)
5742	{
5743	int cap;
5744	u16 cmd;
5745
5746	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5747	if (!cap)
5748	return -EINVAL;
5749
5750	if (pci_read_config_word(dev, where: cap + PCI_X_CMD, val: &cmd))
5751	return -EINVAL;
5752
5753	return 512 << FIELD_GET(PCI_X_CMD_MAX_READ, cmd);
5754	}
5755	EXPORT_SYMBOL(pcix_get_mmrbc);
5756
5757	/**
5758	* pcix_set_mmrbc - set PCI-X maximum memory read byte count
5759	* @dev: PCI device to query
5760	* @mmrbc: maximum memory read count in bytes
5761	* valid values are 512, 1024, 2048, 4096
5762	*
5763	* If possible sets maximum memory read byte count, some bridges have errata
5764	* that prevent this.
5765	*/
5766	int pcix_set_mmrbc(struct pci_dev *dev, int mmrbc)
5767	{
5768	int cap;
5769	u32 stat, v, o;
5770	u16 cmd;
5771
5772	if (mmrbc < 512 || mmrbc > 4096 || !is_power_of_2(n: mmrbc))
5773	return -EINVAL;
5774
5775	v = ffs(mmrbc) - 10;
5776
5777	cap = pci_find_capability(dev, PCI_CAP_ID_PCIX);
5778	if (!cap)
5779	return -EINVAL;
5780
5781	if (pci_read_config_dword(dev, where: cap + PCI_X_STATUS, val: &stat))
5782	return -EINVAL;
5783
5784	if (v > FIELD_GET(PCI_X_STATUS_MAX_READ, stat))
5785	return -E2BIG;
5786
5787	if (pci_read_config_word(dev, where: cap + PCI_X_CMD, val: &cmd))
5788	return -EINVAL;
5789
5790	o = FIELD_GET(PCI_X_CMD_MAX_READ, cmd);
5791	if (o != v) {
5792	if (v > o && (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_MMRBC))
5793	return -EIO;
5794
5795	cmd &= ~PCI_X_CMD_MAX_READ;
5796	cmd |= FIELD_PREP(PCI_X_CMD_MAX_READ, v);
5797	if (pci_write_config_word(dev, where: cap + PCI_X_CMD, val: cmd))
5798	return -EIO;
5799	}
5800	return 0;
5801	}
5802	EXPORT_SYMBOL(pcix_set_mmrbc);
5803
5804	/**
5805	* pcie_get_readrq - get PCI Express read request size
5806	* @dev: PCI device to query
5807	*
5808	* Returns maximum memory read request in bytes or appropriate error value.
5809	*/
5810	int pcie_get_readrq(struct pci_dev *dev)
5811	{
5812	u16 ctl;
5813
5814	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &ctl);
5815
5816	return 128 << FIELD_GET(PCI_EXP_DEVCTL_READRQ, ctl);
5817	}
5818	EXPORT_SYMBOL(pcie_get_readrq);
5819
5820	/**
5821	* pcie_set_readrq - set PCI Express maximum memory read request
5822	* @dev: PCI device to query
5823	* @rq: maximum memory read count in bytes
5824	* valid values are 128, 256, 512, 1024, 2048, 4096
5825	*
5826	* If possible sets maximum memory read request in bytes
5827	*/
5828	int pcie_set_readrq(struct pci_dev *dev, int rq)
5829	{
5830	u16 v;
5831	int ret;
5832	struct pci_host_bridge *bridge = pci_find_host_bridge(bus: dev->bus);
5833
5834	if (rq < 128 || rq > 4096 || !is_power_of_2(n: rq))
5835	return -EINVAL;
5836
5837	/*
5838	* If using the "performance" PCIe config, we clamp the read rq
5839	* size to the max packet size to keep the host bridge from
5840	* generating requests larger than we can cope with.
5841	*/
5842	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
5843	int mps = pcie_get_mps(dev);
5844
5845	if (mps < rq)
5846	rq = mps;
5847	}
5848
5849	v = FIELD_PREP(PCI_EXP_DEVCTL_READRQ, ffs(rq) - 8);
5850
5851	if (bridge->no_inc_mrrs) {
5852	int max_mrrs = pcie_get_readrq(dev);
5853
5854	if (rq > max_mrrs) {
5855	pci_info(dev, "can't set Max_Read_Request_Size to %d; max is %d\n", rq, max_mrrs);
5856	return -EINVAL;
5857	}
5858	}
5859
5860	ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
5861	PCI_EXP_DEVCTL_READRQ, set: v);
5862
5863	return pcibios_err_to_errno(err: ret);
5864	}
5865	EXPORT_SYMBOL(pcie_set_readrq);
5866
5867	/**
5868	* pcie_get_mps - get PCI Express maximum payload size
5869	* @dev: PCI device to query
5870	*
5871	* Returns maximum payload size in bytes
5872	*/
5873	int pcie_get_mps(struct pci_dev *dev)
5874	{
5875	u16 ctl;
5876
5877	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &ctl);
5878
5879	return 128 << FIELD_GET(PCI_EXP_DEVCTL_PAYLOAD, ctl);
5880	}
5881	EXPORT_SYMBOL(pcie_get_mps);
5882
5883	/**
5884	* pcie_set_mps - set PCI Express maximum payload size
5885	* @dev: PCI device to query
5886	* @mps: maximum payload size in bytes
5887	* valid values are 128, 256, 512, 1024, 2048, 4096
5888	*
5889	* If possible sets maximum payload size
5890	*/
5891	int pcie_set_mps(struct pci_dev *dev, int mps)
5892	{
5893	u16 v;
5894	int ret;
5895
5896	if (mps < 128 || mps > 4096 || !is_power_of_2(n: mps))
5897	return -EINVAL;
5898
5899	v = ffs(mps) - 8;
5900	if (v > dev->pcie_mpss)
5901	return -EINVAL;
5902	v = FIELD_PREP(PCI_EXP_DEVCTL_PAYLOAD, v);
5903
5904	ret = pcie_capability_clear_and_set_word(dev, PCI_EXP_DEVCTL,
5905	PCI_EXP_DEVCTL_PAYLOAD, set: v);
5906
5907	return pcibios_err_to_errno(err: ret);
5908	}
5909	EXPORT_SYMBOL(pcie_set_mps);
5910
5911	static enum pci_bus_speed to_pcie_link_speed(u16 lnksta)
5912	{
5913	return pcie_link_speed[FIELD_GET(PCI_EXP_LNKSTA_CLS, lnksta)];
5914	}
5915
5916	int pcie_link_speed_mbps(struct pci_dev *pdev)
5917	{
5918	u16 lnksta;
5919	int err;
5920
5921	err = pcie_capability_read_word(dev: pdev, PCI_EXP_LNKSTA, val: &lnksta);
5922	if (err)
5923	return err;
5924
5925	switch (to_pcie_link_speed(lnksta)) {
5926	case PCIE_SPEED_2_5GT:
5927	return 2500;
5928	case PCIE_SPEED_5_0GT:
5929	return 5000;
5930	case PCIE_SPEED_8_0GT:
5931	return 8000;
5932	case PCIE_SPEED_16_0GT:
5933	return 16000;
5934	case PCIE_SPEED_32_0GT:
5935	return 32000;
5936	case PCIE_SPEED_64_0GT:
5937	return 64000;
5938	default:
5939	break;
5940	}
5941
5942	return -EINVAL;
5943	}
5944	EXPORT_SYMBOL(pcie_link_speed_mbps);
5945
5946	/**
5947	* pcie_bandwidth_available - determine minimum link settings of a PCIe
5948	* device and its bandwidth limitation
5949	* @dev: PCI device to query
5950	* @limiting_dev: storage for device causing the bandwidth limitation
5951	* @speed: storage for speed of limiting device
5952	* @width: storage for width of limiting device
5953	*
5954	* Walk up the PCI device chain and find the point where the minimum
5955	* bandwidth is available. Return the bandwidth available there and (if
5956	* limiting_dev, speed, and width pointers are supplied) information about
5957	* that point. The bandwidth returned is in Mb/s, i.e., megabits/second of
5958	* raw bandwidth.
5959	*/
5960	u32 pcie_bandwidth_available(struct pci_dev *dev, struct pci_dev **limiting_dev,
5961	enum pci_bus_speed *speed,
5962	enum pcie_link_width *width)
5963	{
5964	u16 lnksta;
5965	enum pci_bus_speed next_speed;
5966	enum pcie_link_width next_width;
5967	u32 bw, next_bw;
5968
5969	if (speed)
5970	*speed = PCI_SPEED_UNKNOWN;
5971	if (width)
5972	*width = PCIE_LNK_WIDTH_UNKNOWN;
5973
5974	bw = 0;
5975
5976	while (dev) {
5977	pcie_capability_read_word(dev, PCI_EXP_LNKSTA, val: &lnksta);
5978
5979	next_speed = to_pcie_link_speed(lnksta);
5980	next_width = FIELD_GET(PCI_EXP_LNKSTA_NLW, lnksta);
5981
5982	next_bw = next_width * PCIE_SPEED2MBS_ENC(next_speed);
5983
5984	/* Check if current device limits the total bandwidth */
5985	if (!bw || next_bw <= bw) {
5986	bw = next_bw;
5987
5988	if (limiting_dev)
5989	*limiting_dev = dev;
5990	if (speed)
5991	*speed = next_speed;
5992	if (width)
5993	*width = next_width;
5994	}
5995
5996	dev = pci_upstream_bridge(dev);
5997	}
5998
5999	return bw;
6000	}
6001	EXPORT_SYMBOL(pcie_bandwidth_available);
6002
6003	/**
6004	* pcie_get_speed_cap - query for the PCI device's link speed capability
6005	* @dev: PCI device to query
6006	*
6007	* Query the PCI device speed capability. Return the maximum link speed
6008	* supported by the device.
6009	*/
6010	enum pci_bus_speed pcie_get_speed_cap(struct pci_dev *dev)
6011	{
6012	u32 lnkcap2, lnkcap;
6013
6014	/*
6015	* Link Capabilities 2 was added in PCIe r3.0, sec 7.8.18. The
6016	* implementation note there recommends using the Supported Link
6017	* Speeds Vector in Link Capabilities 2 when supported.
6018	*
6019	* Without Link Capabilities 2, i.e., prior to PCIe r3.0, software
6020	* should use the Supported Link Speeds field in Link Capabilities,
6021	* where only 2.5 GT/s and 5.0 GT/s speeds were defined.
6022	*/
6023	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP2, val: &lnkcap2);
6024
6025	/* PCIe r3.0-compliant */
6026	if (lnkcap2)
6027	return PCIE_LNKCAP2_SLS2SPEED(lnkcap2);
6028
6029	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, val: &lnkcap);
6030	if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_5_0GB)
6031	return PCIE_SPEED_5_0GT;
6032	else if ((lnkcap & PCI_EXP_LNKCAP_SLS) == PCI_EXP_LNKCAP_SLS_2_5GB)
6033	return PCIE_SPEED_2_5GT;
6034
6035	return PCI_SPEED_UNKNOWN;
6036	}
6037	EXPORT_SYMBOL(pcie_get_speed_cap);
6038
6039	/**
6040	* pcie_get_width_cap - query for the PCI device's link width capability
6041	* @dev: PCI device to query
6042	*
6043	* Query the PCI device width capability. Return the maximum link width
6044	* supported by the device.
6045	*/
6046	enum pcie_link_width pcie_get_width_cap(struct pci_dev *dev)
6047	{
6048	u32 lnkcap;
6049
6050	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP, val: &lnkcap);
6051	if (lnkcap)
6052	return FIELD_GET(PCI_EXP_LNKCAP_MLW, lnkcap);
6053
6054	return PCIE_LNK_WIDTH_UNKNOWN;
6055	}
6056	EXPORT_SYMBOL(pcie_get_width_cap);
6057
6058	/**
6059	* pcie_bandwidth_capable - calculate a PCI device's link bandwidth capability
6060	* @dev: PCI device
6061	* @speed: storage for link speed
6062	* @width: storage for link width
6063	*
6064	* Calculate a PCI device's link bandwidth by querying for its link speed
6065	* and width, multiplying them, and applying encoding overhead. The result
6066	* is in Mb/s, i.e., megabits/second of raw bandwidth.
6067	*/
6068	u32 pcie_bandwidth_capable(struct pci_dev *dev, enum pci_bus_speed *speed,
6069	enum pcie_link_width *width)
6070	{
6071	*speed = pcie_get_speed_cap(dev);
6072	*width = pcie_get_width_cap(dev);
6073
6074	if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN)
6075	return 0;
6076
6077	return *width * PCIE_SPEED2MBS_ENC(*speed);
6078	}
6079
6080	/**
6081	* __pcie_print_link_status - Report the PCI device's link speed and width
6082	* @dev: PCI device to query
6083	* @verbose: Print info even when enough bandwidth is available
6084	*
6085	* If the available bandwidth at the device is less than the device is
6086	* capable of, report the device's maximum possible bandwidth and the
6087	* upstream link that limits its performance. If @verbose, always print
6088	* the available bandwidth, even if the device isn't constrained.
6089	*/
6090	void __pcie_print_link_status(struct pci_dev *dev, bool verbose)
6091	{
6092	enum pcie_link_width width, width_cap;
6093	enum pci_bus_speed speed, speed_cap;
6094	struct pci_dev *limiting_dev = NULL;
6095	u32 bw_avail, bw_cap;
6096
6097	bw_cap = pcie_bandwidth_capable(dev, speed: &speed_cap, width: &width_cap);
6098	bw_avail = pcie_bandwidth_available(dev, &limiting_dev, &speed, &width);
6099
6100	if (bw_avail >= bw_cap && verbose)
6101	pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth (%s x%d link)\n",
6102	bw_cap / 1000, bw_cap % 1000,
6103	pci_speed_string(speed_cap), width_cap);
6104	else if (bw_avail < bw_cap)
6105	pci_info(dev, "%u.%03u Gb/s available PCIe bandwidth, limited by %s x%d link at %s (capable of %u.%03u Gb/s with %s x%d link)\n",
6106	bw_avail / 1000, bw_avail % 1000,
6107	pci_speed_string(speed), width,
6108	limiting_dev ? pci_name(limiting_dev) : "<unknown>",
6109	bw_cap / 1000, bw_cap % 1000,
6110	pci_speed_string(speed_cap), width_cap);
6111	}
6112
6113	/**
6114	* pcie_print_link_status - Report the PCI device's link speed and width
6115	* @dev: PCI device to query
6116	*
6117	* Report the available bandwidth at the device.
6118	*/
6119	void pcie_print_link_status(struct pci_dev *dev)
6120	{
6121	__pcie_print_link_status(dev, verbose: true);
6122	}
6123	EXPORT_SYMBOL(pcie_print_link_status);
6124
6125	/**
6126	* pci_select_bars - Make BAR mask from the type of resource
6127	* @dev: the PCI device for which BAR mask is made
6128	* @flags: resource type mask to be selected
6129	*
6130	* This helper routine makes bar mask from the type of resource.
6131	*/
6132	int pci_select_bars(struct pci_dev *dev, unsigned long flags)
6133	{
6134	int i, bars = 0;
6135	for (i = 0; i < PCI_NUM_RESOURCES; i++)
6136	if (pci_resource_flags(dev, i) & flags)
6137	bars |= (1 << i);
6138	return bars;
6139	}
6140	EXPORT_SYMBOL(pci_select_bars);
6141
6142	/* Some architectures require additional programming to enable VGA */
6143	static arch_set_vga_state_t arch_set_vga_state;
6144
6145	void __init pci_register_set_vga_state(arch_set_vga_state_t func)
6146	{
6147	arch_set_vga_state = func; /* NULL disables */
6148	}
6149
6150	static int pci_set_vga_state_arch(struct pci_dev *dev, bool decode,
6151	unsigned int command_bits, u32 flags)
6152	{
6153	if (arch_set_vga_state)
6154	return arch_set_vga_state(dev, decode, command_bits,
6155	flags);
6156	return 0;
6157	}
6158
6159	/**
6160	* pci_set_vga_state - set VGA decode state on device and parents if requested
6161	* @dev: the PCI device
6162	* @decode: true = enable decoding, false = disable decoding
6163	* @command_bits: PCI_COMMAND_IO and/or PCI_COMMAND_MEMORY
6164	* @flags: traverse ancestors and change bridges
6165	* CHANGE_BRIDGE_ONLY / CHANGE_BRIDGE
6166	*/
6167	int pci_set_vga_state(struct pci_dev *dev, bool decode,
6168	unsigned int command_bits, u32 flags)
6169	{
6170	struct pci_bus *bus;
6171	struct pci_dev *bridge;
6172	u16 cmd;
6173	int rc;
6174
6175	WARN_ON((flags & PCI_VGA_STATE_CHANGE_DECODES) && (command_bits & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY)));
6176
6177	/* ARCH specific VGA enables */
6178	rc = pci_set_vga_state_arch(dev, decode, command_bits, flags);
6179	if (rc)
6180	return rc;
6181
6182	if (flags & PCI_VGA_STATE_CHANGE_DECODES) {
6183	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
6184	if (decode)
6185	cmd |= command_bits;
6186	else
6187	cmd &= ~command_bits;
6188	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
6189	}
6190
6191	if (!(flags & PCI_VGA_STATE_CHANGE_BRIDGE))
6192	return 0;
6193
6194	bus = dev->bus;
6195	while (bus) {
6196	bridge = bus->self;
6197	if (bridge) {
6198	pci_read_config_word(dev: bridge, PCI_BRIDGE_CONTROL,
6199	val: &cmd);
6200	if (decode)
6201	cmd |= PCI_BRIDGE_CTL_VGA;
6202	else
6203	cmd &= ~PCI_BRIDGE_CTL_VGA;
6204	pci_write_config_word(dev: bridge, PCI_BRIDGE_CONTROL,
6205	val: cmd);
6206	}
6207	bus = bus->parent;
6208	}
6209	return 0;
6210	}
6211
6212	#ifdef CONFIG_ACPI
6213	bool pci_pr3_present(struct pci_dev *pdev)
6214	{
6215	struct acpi_device *adev;
6216
6217	if (acpi_disabled)
6218	return false;
6219
6220	adev = ACPI_COMPANION(&pdev->dev);
6221	if (!adev)
6222	return false;
6223
6224	return adev->power.flags.power_resources &&
6225	acpi_has_method(handle: adev->handle, name: "_PR3");
6226	}
6227	EXPORT_SYMBOL_GPL(pci_pr3_present);
6228	#endif
6229
6230	/**
6231	* pci_add_dma_alias - Add a DMA devfn alias for a device
6232	* @dev: the PCI device for which alias is added
6233	* @devfn_from: alias slot and function
6234	* @nr_devfns: number of subsequent devfns to alias
6235	*
6236	* This helper encodes an 8-bit devfn as a bit number in dma_alias_mask
6237	* which is used to program permissible bus-devfn source addresses for DMA
6238	* requests in an IOMMU. These aliases factor into IOMMU group creation
6239	* and are useful for devices generating DMA requests beyond or different
6240	* from their logical bus-devfn. Examples include device quirks where the
6241	* device simply uses the wrong devfn, as well as non-transparent bridges
6242	* where the alias may be a proxy for devices in another domain.
6243	*
6244	* IOMMU group creation is performed during device discovery or addition,
6245	* prior to any potential DMA mapping and therefore prior to driver probing
6246	* (especially for userspace assigned devices where IOMMU group definition
6247	* cannot be left as a userspace activity). DMA aliases should therefore
6248	* be configured via quirks, such as the PCI fixup header quirk.
6249	*/
6250	void pci_add_dma_alias(struct pci_dev *dev, u8 devfn_from,
6251	unsigned int nr_devfns)
6252	{
6253	int devfn_to;
6254
6255	nr_devfns = min(nr_devfns, (unsigned int)MAX_NR_DEVFNS - devfn_from);
6256	devfn_to = devfn_from + nr_devfns - 1;
6257
6258	if (!dev->dma_alias_mask)
6259	dev->dma_alias_mask = bitmap_zalloc(MAX_NR_DEVFNS, GFP_KERNEL);
6260	if (!dev->dma_alias_mask) {
6261	pci_warn(dev, "Unable to allocate DMA alias mask\n");
6262	return;
6263	}
6264
6265	bitmap_set(map: dev->dma_alias_mask, start: devfn_from, nbits: nr_devfns);
6266
6267	if (nr_devfns == 1)
6268	pci_info(dev, "Enabling fixed DMA alias to %02x.%d\n",
6269	PCI_SLOT(devfn_from), PCI_FUNC(devfn_from));
6270	else if (nr_devfns > 1)
6271	pci_info(dev, "Enabling fixed DMA alias for devfn range from %02x.%d to %02x.%d\n",
6272	PCI_SLOT(devfn_from), PCI_FUNC(devfn_from),
6273	PCI_SLOT(devfn_to), PCI_FUNC(devfn_to));
6274	}
6275
6276	bool pci_devs_are_dma_aliases(struct pci_dev *dev1, struct pci_dev *dev2)
6277	{
6278	return (dev1->dma_alias_mask &&
6279	test_bit(dev2->devfn, dev1->dma_alias_mask)) ||
6280	(dev2->dma_alias_mask &&
6281	test_bit(dev1->devfn, dev2->dma_alias_mask)) ||
6282	pci_real_dma_dev(dev: dev1) == dev2 ||
6283	pci_real_dma_dev(dev: dev2) == dev1;
6284	}
6285
6286	bool pci_device_is_present(struct pci_dev *pdev)
6287	{
6288	u32 v;
6289
6290	/* Check PF if pdev is a VF, since VF Vendor/Device IDs are 0xffff */
6291	pdev = pci_physfn(dev: pdev);
6292	if (pci_dev_is_disconnected(dev: pdev))
6293	return false;
6294	return pci_bus_read_dev_vendor_id(bus: pdev->bus, devfn: pdev->devfn, pl: &v, crs_timeout: 0);
6295	}
6296	EXPORT_SYMBOL_GPL(pci_device_is_present);
6297
6298	void pci_ignore_hotplug(struct pci_dev *dev)
6299	{
6300	struct pci_dev *bridge = dev->bus->self;
6301
6302	dev->ignore_hotplug = 1;
6303	/* Propagate the "ignore hotplug" setting to the parent bridge. */
6304	if (bridge)
6305	bridge->ignore_hotplug = 1;
6306	}
6307	EXPORT_SYMBOL_GPL(pci_ignore_hotplug);
6308
6309	/**
6310	* pci_real_dma_dev - Get PCI DMA device for PCI device
6311	* @dev: the PCI device that may have a PCI DMA alias
6312	*
6313	* Permits the platform to provide architecture-specific functionality to
6314	* devices needing to alias DMA to another PCI device on another PCI bus. If
6315	* the PCI device is on the same bus, it is recommended to use
6316	* pci_add_dma_alias(). This is the default implementation. Architecture
6317	* implementations can override this.
6318	*/
6319	struct pci_dev __weak *pci_real_dma_dev(struct pci_dev *dev)
6320	{
6321	return dev;
6322	}
6323
6324	resource_size_t __weak pcibios_default_alignment(void)
6325	{
6326	return 0;
6327	}
6328
6329	/*
6330	* Arches that don't want to expose struct resource to userland as-is in
6331	* sysfs and /proc can implement their own pci_resource_to_user().
6332	*/
6333	void __weak pci_resource_to_user(const struct pci_dev *dev, int bar,
6334	const struct resource *rsrc,
6335	resource_size_t *start, resource_size_t *end)
6336	{
6337	*start = rsrc->start;
6338	*end = rsrc->end;
6339	}
6340
6341	static char *resource_alignment_param;
6342	static DEFINE_SPINLOCK(resource_alignment_lock);
6343
6344	/**
6345	* pci_specified_resource_alignment - get resource alignment specified by user.
6346	* @dev: the PCI device to get
6347	* @resize: whether or not to change resources' size when reassigning alignment
6348	*
6349	* RETURNS: Resource alignment if it is specified.
6350	* Zero if it is not specified.
6351	*/
6352	static resource_size_t pci_specified_resource_alignment(struct pci_dev *dev,
6353	bool *resize)
6354	{
6355	int align_order, count;
6356	resource_size_t align = pcibios_default_alignment();
6357	const char *p;
6358	int ret;
6359
6360	spin_lock(lock: &resource_alignment_lock);
6361	p = resource_alignment_param;
6362	if (!p || !*p)
6363	goto out;
6364	if (pci_has_flag(flag: PCI_PROBE_ONLY)) {
6365	align = 0;
6366	pr_info_once("PCI: Ignoring requested alignments (PCI_PROBE_ONLY)\n");
6367	goto out;
6368	}
6369
6370	while (*p) {
6371	count = 0;
6372	if (sscanf(p, "%d%n", &align_order, &count) == 1 &&
6373	p[count] == '@') {
6374	p += count + 1;
6375	if (align_order > 63) {
6376	pr_err("PCI: Invalid requested alignment (order %d)\n",
6377	align_order);
6378	align_order = PAGE_SHIFT;
6379	}
6380	} else {
6381	align_order = PAGE_SHIFT;
6382	}
6383
6384	ret = pci_dev_str_match(dev, p, endptr: &p);
6385	if (ret == 1) {
6386	*resize = true;
6387	align = 1ULL << align_order;
6388	break;
6389	} else if (ret < 0) {
6390	pr_err("PCI: Can't parse resource_alignment parameter: %s\n",
6391	p);
6392	break;
6393	}
6394
6395	if (*p != ';' && *p != ',') {
6396	/* End of param or invalid format */
6397	break;
6398	}
6399	p++;
6400	}
6401	out:
6402	spin_unlock(lock: &resource_alignment_lock);
6403	return align;
6404	}
6405
6406	static void pci_request_resource_alignment(struct pci_dev *dev, int bar,
6407	resource_size_t align, bool resize)
6408	{
6409	struct resource *r = &dev->resource[bar];
6410	const char *r_name = pci_resource_name(dev, i: bar);
6411	resource_size_t size;
6412
6413	if (!(r->flags & IORESOURCE_MEM))
6414	return;
6415
6416	if (r->flags & IORESOURCE_PCI_FIXED) {
6417	pci_info(dev, "%s %pR: ignoring requested alignment %#llx\n",
6418	r_name, r, (unsigned long long)align);
6419	return;
6420	}
6421
6422	size = resource_size(res: r);
6423	if (size >= align)
6424	return;
6425
6426	/*
6427	* Increase the alignment of the resource. There are two ways we
6428	* can do this:
6429	*
6430	* 1) Increase the size of the resource. BARs are aligned on their
6431	* size, so when we reallocate space for this resource, we'll
6432	* allocate it with the larger alignment. This also prevents
6433	* assignment of any other BARs inside the alignment region, so
6434	* if we're requesting page alignment, this means no other BARs
6435	* will share the page.
6436	*
6437	* The disadvantage is that this makes the resource larger than
6438	* the hardware BAR, which may break drivers that compute things
6439	* based on the resource size, e.g., to find registers at a
6440	* fixed offset before the end of the BAR.
6441	*
6442	* 2) Retain the resource size, but use IORESOURCE_STARTALIGN and
6443	* set r->start to the desired alignment. By itself this
6444	* doesn't prevent other BARs being put inside the alignment
6445	* region, but if we realign *every* resource of every device in
6446	* the system, none of them will share an alignment region.
6447	*
6448	* When the user has requested alignment for only some devices via
6449	* the "pci=resource_alignment" argument, "resize" is true and we
6450	* use the first method. Otherwise we assume we're aligning all
6451	* devices and we use the second.
6452	*/
6453
6454	pci_info(dev, "%s %pR: requesting alignment to %#llx\n",
6455	r_name, r, (unsigned long long)align);
6456
6457	if (resize) {
6458	r->start = 0;
6459	r->end = align - 1;
6460	} else {
6461	r->flags &= ~IORESOURCE_SIZEALIGN;
6462	r->flags |= IORESOURCE_STARTALIGN;
6463	r->start = align;
6464	r->end = r->start + size - 1;
6465	}
6466	r->flags |= IORESOURCE_UNSET;
6467	}
6468
6469	/*
6470	* This function disables memory decoding and releases memory resources
6471	* of the device specified by kernel's boot parameter 'pci=resource_alignment='.
6472	* It also rounds up size to specified alignment.
6473	* Later on, the kernel will assign page-aligned memory resource back
6474	* to the device.
6475	*/
6476	void pci_reassigndev_resource_alignment(struct pci_dev *dev)
6477	{
6478	int i;
6479	struct resource *r;
6480	resource_size_t align;
6481	u16 command;
6482	bool resize = false;
6483
6484	/*
6485	* VF BARs are read-only zero according to SR-IOV spec r1.1, sec
6486	* 3.4.1.11. Their resources are allocated from the space
6487	* described by the VF BARx register in the PF's SR-IOV capability.
6488	* We can't influence their alignment here.
6489	*/
6490	if (dev->is_virtfn)
6491	return;
6492
6493	/* check if specified PCI is target device to reassign */
6494	align = pci_specified_resource_alignment(dev, resize: &resize);
6495	if (!align)
6496	return;
6497
6498	if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
6499	(dev->class >> 8) == PCI_CLASS_BRIDGE_HOST) {
6500	pci_warn(dev, "Can't reassign resources to host bridge\n");
6501	return;
6502	}
6503
6504	pci_read_config_word(dev, PCI_COMMAND, val: &command);
6505	command &= ~PCI_COMMAND_MEMORY;
6506	pci_write_config_word(dev, PCI_COMMAND, val: command);
6507
6508	for (i = 0; i <= PCI_ROM_RESOURCE; i++)
6509	pci_request_resource_alignment(dev, bar: i, align, resize);
6510
6511	/*
6512	* Need to disable bridge's resource window,
6513	* to enable the kernel to reassign new resource
6514	* window later on.
6515	*/
6516	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
6517	for (i = PCI_BRIDGE_RESOURCES; i < PCI_NUM_RESOURCES; i++) {
6518	r = &dev->resource[i];
6519	if (!(r->flags & IORESOURCE_MEM))
6520	continue;
6521	r->flags |= IORESOURCE_UNSET;
6522	r->end = resource_size(res: r) - 1;
6523	r->start = 0;
6524	}
6525	pci_disable_bridge_window(dev);
6526	}
6527	}
6528
6529	static ssize_t resource_alignment_show(const struct bus_type *bus, char *buf)
6530	{
6531	size_t count = 0;
6532
6533	spin_lock(lock: &resource_alignment_lock);
6534	if (resource_alignment_param)
6535	count = sysfs_emit(buf, fmt: "%s\n", resource_alignment_param);
6536	spin_unlock(lock: &resource_alignment_lock);
6537
6538	return count;
6539	}
6540
6541	static ssize_t resource_alignment_store(const struct bus_type *bus,
6542	const char *buf, size_t count)
6543	{
6544	char *param, *old, *end;
6545
6546	if (count >= (PAGE_SIZE - 1))
6547	return -EINVAL;
6548
6549	param = kstrndup(s: buf, len: count, GFP_KERNEL);
6550	if (!param)
6551	return -ENOMEM;
6552
6553	end = strchr(param, '\n');
6554	if (end)
6555	*end = '\0';
6556
6557	spin_lock(lock: &resource_alignment_lock);
6558	old = resource_alignment_param;
6559	if (strlen(param)) {
6560	resource_alignment_param = param;
6561	} else {
6562	kfree(objp: param);
6563	resource_alignment_param = NULL;
6564	}
6565	spin_unlock(lock: &resource_alignment_lock);
6566
6567	kfree(objp: old);
6568
6569	return count;
6570	}
6571
6572	static BUS_ATTR_RW(resource_alignment);
6573
6574	static int __init pci_resource_alignment_sysfs_init(void)
6575	{
6576	return bus_create_file(bus: &pci_bus_type,
6577	attr: &bus_attr_resource_alignment);
6578	}
6579	late_initcall(pci_resource_alignment_sysfs_init);
6580
6581	static void pci_no_domains(void)
6582	{
6583	#ifdef CONFIG_PCI_DOMAINS
6584	pci_domains_supported = 0;
6585	#endif
6586	}
6587
6588	#ifdef CONFIG_PCI_DOMAINS_GENERIC
6589	static DEFINE_IDA(pci_domain_nr_static_ida);
6590	static DEFINE_IDA(pci_domain_nr_dynamic_ida);
6591
6592	static void of_pci_reserve_static_domain_nr(void)
6593	{
6594	struct device_node *np;
6595	int domain_nr;
6596
6597	for_each_node_by_type(np, "pci") {
6598	domain_nr = of_get_pci_domain_nr(np);
6599	if (domain_nr < 0)
6600	continue;
6601	/*
6602	* Permanently allocate domain_nr in dynamic_ida
6603	* to prevent it from dynamic allocation.
6604	*/
6605	ida_alloc_range(&pci_domain_nr_dynamic_ida,
6606	domain_nr, domain_nr, GFP_KERNEL);
6607	}
6608	}
6609
6610	static int of_pci_bus_find_domain_nr(struct device *parent)
6611	{
6612	static bool static_domains_reserved = false;
6613	int domain_nr;
6614
6615	/* On the first call scan device tree for static allocations. */
6616	if (!static_domains_reserved) {
6617	of_pci_reserve_static_domain_nr();
6618	static_domains_reserved = true;
6619	}
6620
6621	if (parent) {
6622	/*
6623	* If domain is in DT, allocate it in static IDA. This
6624	* prevents duplicate static allocations in case of errors
6625	* in DT.
6626	*/
6627	domain_nr = of_get_pci_domain_nr(parent->of_node);
6628	if (domain_nr >= 0)
6629	return ida_alloc_range(&pci_domain_nr_static_ida,
6630	domain_nr, domain_nr,
6631	GFP_KERNEL);
6632	}
6633
6634	/*
6635	* If domain was not specified in DT, choose a free ID from dynamic
6636	* allocations. All domain numbers from DT are permanently in
6637	* dynamic allocations to prevent assigning them to other DT nodes
6638	* without static domain.
6639	*/
6640	return ida_alloc(&pci_domain_nr_dynamic_ida, GFP_KERNEL);
6641	}
6642
6643	static void of_pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent)
6644	{
6645	if (bus->domain_nr < 0)
6646	return;
6647
6648	/* Release domain from IDA where it was allocated. */
6649	if (of_get_pci_domain_nr(parent->of_node) == bus->domain_nr)
6650	ida_free(&pci_domain_nr_static_ida, bus->domain_nr);
6651	else
6652	ida_free(&pci_domain_nr_dynamic_ida, bus->domain_nr);
6653	}
6654
6655	int pci_bus_find_domain_nr(struct pci_bus *bus, struct device *parent)
6656	{
6657	return acpi_disabled ? of_pci_bus_find_domain_nr(parent) :
6658	acpi_pci_bus_find_domain_nr(bus);
6659	}
6660
6661	void pci_bus_release_domain_nr(struct pci_bus *bus, struct device *parent)
6662	{
6663	if (!acpi_disabled)
6664	return;
6665	of_pci_bus_release_domain_nr(bus, parent);
6666	}
6667	#endif
6668
6669	/**
6670	* pci_ext_cfg_avail - can we access extended PCI config space?
6671	*
6672	* Returns 1 if we can access PCI extended config space (offsets
6673	* greater than 0xff). This is the default implementation. Architecture
6674	* implementations can override this.
6675	*/
6676	int __weak pci_ext_cfg_avail(void)
6677	{
6678	return 1;
6679	}
6680
6681	void __weak pci_fixup_cardbus(struct pci_bus *bus)
6682	{
6683	}
6684	EXPORT_SYMBOL(pci_fixup_cardbus);
6685
6686	static int __init pci_setup(char *str)
6687	{
6688	while (str) {
6689	char *k = strchr(str, ',');
6690	if (k)
6691	*k++ = 0;
6692	if (*str && (str = pcibios_setup(str)) && *str) {
6693	if (!strcmp(str, "nomsi")) {
6694	pci_no_msi();
6695	} else if (!strncmp(str, "noats", 5)) {
6696	pr_info("PCIe: ATS is disabled\n");
6697	pcie_ats_disabled = true;
6698	} else if (!strcmp(str, "noaer")) {
6699	pci_no_aer();
6700	} else if (!strcmp(str, "earlydump")) {
6701	pci_early_dump = true;
6702	} else if (!strncmp(str, "realloc=", 8)) {
6703	pci_realloc_get_opt(str + 8);
6704	} else if (!strncmp(str, "realloc", 7)) {
6705	pci_realloc_get_opt("on");
6706	} else if (!strcmp(str, "nodomains")) {
6707	pci_no_domains();
6708	} else if (!strncmp(str, "noari", 5)) {
6709	pcie_ari_disabled = true;
6710	} else if (!strncmp(str, "cbiosize=", 9)) {
6711	pci_cardbus_io_size = memparse(ptr: str + 9, retptr: &str);
6712	} else if (!strncmp(str, "cbmemsize=", 10)) {
6713	pci_cardbus_mem_size = memparse(ptr: str + 10, retptr: &str);
6714	} else if (!strncmp(str, "resource_alignment=", 19)) {
6715	resource_alignment_param = str + 19;
6716	} else if (!strncmp(str, "ecrc=", 5)) {
6717	pcie_ecrc_get_policy(str: str + 5);
6718	} else if (!strncmp(str, "hpiosize=", 9)) {
6719	pci_hotplug_io_size = memparse(ptr: str + 9, retptr: &str);
6720	} else if (!strncmp(str, "hpmmiosize=", 11)) {
6721	pci_hotplug_mmio_size = memparse(ptr: str + 11, retptr: &str);
6722	} else if (!strncmp(str, "hpmmioprefsize=", 15)) {
6723	pci_hotplug_mmio_pref_size = memparse(ptr: str + 15, retptr: &str);
6724	} else if (!strncmp(str, "hpmemsize=", 10)) {
6725	pci_hotplug_mmio_size = memparse(ptr: str + 10, retptr: &str);
6726	pci_hotplug_mmio_pref_size = pci_hotplug_mmio_size;
6727	} else if (!strncmp(str, "hpbussize=", 10)) {
6728	pci_hotplug_bus_size =
6729	simple_strtoul(str + 10, &str, 0);
6730	if (pci_hotplug_bus_size > 0xff)
6731	pci_hotplug_bus_size = DEFAULT_HOTPLUG_BUS_SIZE;
6732	} else if (!strncmp(str, "pcie_bus_tune_off", 17)) {
6733	pcie_bus_config = PCIE_BUS_TUNE_OFF;
6734	} else if (!strncmp(str, "pcie_bus_safe", 13)) {
6735	pcie_bus_config = PCIE_BUS_SAFE;
6736	} else if (!strncmp(str, "pcie_bus_perf", 13)) {
6737	pcie_bus_config = PCIE_BUS_PERFORMANCE;
6738	} else if (!strncmp(str, "pcie_bus_peer2peer", 18)) {
6739	pcie_bus_config = PCIE_BUS_PEER2PEER;
6740	} else if (!strncmp(str, "pcie_scan_all", 13)) {
6741	pci_add_flags(flags: PCI_SCAN_ALL_PCIE_DEVS);
6742	} else if (!strncmp(str, "disable_acs_redir=", 18)) {
6743	disable_acs_redir_param = str + 18;
6744	} else {
6745	pr_err("PCI: Unknown option `%s'\n", str);
6746	}
6747	}
6748	str = k;
6749	}
6750	return 0;
6751	}
6752	early_param("pci", pci_setup);
6753
6754	/*
6755	* 'resource_alignment_param' and 'disable_acs_redir_param' are initialized
6756	* in pci_setup(), above, to point to data in the __initdata section which
6757	* will be freed after the init sequence is complete. We can't allocate memory
6758	* in pci_setup() because some architectures do not have any memory allocation
6759	* service available during an early_param() call. So we allocate memory and
6760	* copy the variable here before the init section is freed.
6761	*
6762	*/
6763	static int __init pci_realloc_setup_params(void)
6764	{
6765	resource_alignment_param = kstrdup(s: resource_alignment_param,
6766	GFP_KERNEL);
6767	disable_acs_redir_param = kstrdup(s: disable_acs_redir_param, GFP_KERNEL);
6768
6769	return 0;
6770	}
6771	pure_initcall(pci_realloc_setup_params);
6772