1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Enable PCIe link L0s/L1 state and Clock Power Management
4	*
5	* Copyright (C) 2007 Intel
6	* Copyright (C) Zhang Yanmin (yanmin.zhang@intel.com)
7	* Copyright (C) Shaohua Li (shaohua.li@intel.com)
8	*/
9
10	#include <linux/bitfield.h>
11	#include <linux/kernel.h>
12	#include <linux/limits.h>
13	#include <linux/math.h>
14	#include <linux/module.h>
15	#include <linux/moduleparam.h>
16	#include <linux/pci.h>
17	#include <linux/pci_regs.h>
18	#include <linux/errno.h>
19	#include <linux/pm.h>
20	#include <linux/init.h>
21	#include <linux/printk.h>
22	#include <linux/slab.h>
23	#include <linux/time.h>
24
25	#include "../pci.h"
26
27	void pci_save_ltr_state(struct pci_dev *dev)
28	{
29	int ltr;
30	struct pci_cap_saved_state *save_state;
31	u32 *cap;
32
33	if (!pci_is_pcie(dev))
34	return;
35
36	ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);
37	if (!ltr)
38	return;
39
40	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);
41	if (!save_state) {
42	pci_err(dev, "no suspend buffer for LTR; ASPM issues possible after resume\n");
43	return;
44	}
45
46	/* Some broken devices only support dword access to LTR */
47	cap = &save_state->cap.data[0];
48	pci_read_config_dword(dev, where: ltr + PCI_LTR_MAX_SNOOP_LAT, val: cap);
49	}
50
51	void pci_restore_ltr_state(struct pci_dev *dev)
52	{
53	struct pci_cap_saved_state *save_state;
54	int ltr;
55	u32 *cap;
56
57	save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_LTR);
58	ltr = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_LTR);
59	if (!save_state || !ltr)
60	return;
61
62	/* Some broken devices only support dword access to LTR */
63	cap = &save_state->cap.data[0];
64	pci_write_config_dword(dev, where: ltr + PCI_LTR_MAX_SNOOP_LAT, val: *cap);
65	}
66
67	void pci_configure_aspm_l1ss(struct pci_dev *pdev)
68	{
69	int rc;
70
71	pdev->l1ss = pci_find_ext_capability(dev: pdev, PCI_EXT_CAP_ID_L1SS);
72
73	rc = pci_add_ext_cap_save_buffer(dev: pdev, PCI_EXT_CAP_ID_L1SS,
74	size: 2 * sizeof(u32));
75	if (rc)
76	pci_err(pdev, "unable to allocate ASPM L1SS save buffer (%pe)\n",
77	ERR_PTR(rc));
78	}
79
80	void pci_save_aspm_l1ss_state(struct pci_dev *pdev)
81	{
82	struct pci_cap_saved_state *save_state;
83	u16 l1ss = pdev->l1ss;
84	u32 *cap;
85
86	/*
87	* Save L1 substate configuration. The ASPM L0s/L1 configuration
88	* in PCI_EXP_LNKCTL_ASPMC is saved by pci_save_pcie_state().
89	*/
90	if (!l1ss)
91	return;
92
93	save_state = pci_find_saved_ext_cap(dev: pdev, PCI_EXT_CAP_ID_L1SS);
94	if (!save_state)
95	return;
96
97	cap = &save_state->cap.data[0];
98	pci_read_config_dword(dev: pdev, where: l1ss + PCI_L1SS_CTL2, val: cap++);
99	pci_read_config_dword(dev: pdev, where: l1ss + PCI_L1SS_CTL1, val: cap++);
100	}
101
102	void pci_restore_aspm_l1ss_state(struct pci_dev *pdev)
103	{
104	struct pci_cap_saved_state *pl_save_state, *cl_save_state;
105	struct pci_dev *parent = pdev->bus->self;
106	u32 *cap, pl_ctl1, pl_ctl2, pl_l1_2_enable;
107	u32 cl_ctl1, cl_ctl2, cl_l1_2_enable;
108	u16 clnkctl, plnkctl;
109
110	/*
111	* In case BIOS enabled L1.2 when resuming, we need to disable it first
112	* on the downstream component before the upstream. So, don't attempt to
113	* restore either until we are at the downstream component.
114	*/
115	if (pcie_downstream_port(dev: pdev) || !parent)
116	return;
117
118	if (!pdev->l1ss || !parent->l1ss)
119	return;
120
121	cl_save_state = pci_find_saved_ext_cap(dev: pdev, PCI_EXT_CAP_ID_L1SS);
122	pl_save_state = pci_find_saved_ext_cap(dev: parent, PCI_EXT_CAP_ID_L1SS);
123	if (!cl_save_state || !pl_save_state)
124	return;
125
126	cap = &cl_save_state->cap.data[0];
127	cl_ctl2 = *cap++;
128	cl_ctl1 = *cap;
129	cap = &pl_save_state->cap.data[0];
130	pl_ctl2 = *cap++;
131	pl_ctl1 = *cap;
132
133	/* Make sure L0s/L1 are disabled before updating L1SS config */
134	pcie_capability_read_word(dev: pdev, PCI_EXP_LNKCTL, val: &clnkctl);
135	pcie_capability_read_word(dev: parent, PCI_EXP_LNKCTL, val: &plnkctl);
136	if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, clnkctl) ||
137	FIELD_GET(PCI_EXP_LNKCTL_ASPMC, plnkctl)) {
138	pcie_capability_write_word(dev: pdev, PCI_EXP_LNKCTL,
139	val: clnkctl & ~PCI_EXP_LNKCTL_ASPMC);
140	pcie_capability_write_word(dev: parent, PCI_EXP_LNKCTL,
141	val: plnkctl & ~PCI_EXP_LNKCTL_ASPMC);
142	}
143
144	/*
145	* Disable L1.2 on this downstream endpoint device first, followed
146	* by the upstream
147	*/
148	pci_clear_and_set_config_dword(dev: pdev, pos: pdev->l1ss + PCI_L1SS_CTL1,
149	PCI_L1SS_CTL1_L1_2_MASK, set: 0);
150	pci_clear_and_set_config_dword(dev: parent, pos: parent->l1ss + PCI_L1SS_CTL1,
151	PCI_L1SS_CTL1_L1_2_MASK, set: 0);
152
153	/*
154	* In addition, Common_Mode_Restore_Time and LTR_L1.2_THRESHOLD
155	* in PCI_L1SS_CTL1 must be programmed *before* setting the L1.2
156	* enable bits, even though they're all in PCI_L1SS_CTL1.
157	*/
158	pl_l1_2_enable = pl_ctl1 & PCI_L1SS_CTL1_L1_2_MASK;
159	pl_ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK;
160	cl_l1_2_enable = cl_ctl1 & PCI_L1SS_CTL1_L1_2_MASK;
161	cl_ctl1 &= ~PCI_L1SS_CTL1_L1_2_MASK;
162
163	/* Write back without enables first (above we cleared them in ctl1) */
164	pci_write_config_dword(dev: parent, where: parent->l1ss + PCI_L1SS_CTL2, val: pl_ctl2);
165	pci_write_config_dword(dev: pdev, where: pdev->l1ss + PCI_L1SS_CTL2, val: cl_ctl2);
166	pci_write_config_dword(dev: parent, where: parent->l1ss + PCI_L1SS_CTL1, val: pl_ctl1);
167	pci_write_config_dword(dev: pdev, where: pdev->l1ss + PCI_L1SS_CTL1, val: cl_ctl1);
168
169	/* Then write back the enables */
170	if (pl_l1_2_enable || cl_l1_2_enable) {
171	pci_write_config_dword(dev: parent, where: parent->l1ss + PCI_L1SS_CTL1,
172	val: pl_ctl1 | pl_l1_2_enable);
173	pci_write_config_dword(dev: pdev, where: pdev->l1ss + PCI_L1SS_CTL1,
174	val: cl_ctl1 | cl_l1_2_enable);
175	}
176
177	/* Restore L0s/L1 if they were enabled */
178	if (FIELD_GET(PCI_EXP_LNKCTL_ASPMC, clnkctl) ||
179	FIELD_GET(PCI_EXP_LNKCTL_ASPMC, plnkctl)) {
180	pcie_capability_write_word(dev: parent, PCI_EXP_LNKCTL, val: clnkctl);
181	pcie_capability_write_word(dev: pdev, PCI_EXP_LNKCTL, val: plnkctl);
182	}
183	}
184
185	#ifdef CONFIG_PCIEASPM
186
187	#ifdef MODULE_PARAM_PREFIX
188	#undef MODULE_PARAM_PREFIX
189	#endif
190	#define MODULE_PARAM_PREFIX "pcie_aspm."
191
192	/* Note: those are not register definitions */
193	#define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
194	#define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
195	#define ASPM_STATE_L1 (4) /* L1 state */
196	#define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */
197	#define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */
198	#define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */
199	#define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */
200	#define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM)
201	#define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM)
202	#define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\
203	ASPM_STATE_L1_2_MASK)
204	#define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
205	#define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \
206	ASPM_STATE_L1SS)
207
208	struct pcie_link_state {
209	struct pci_dev *pdev; /* Upstream component of the Link */
210	struct pci_dev *downstream; /* Downstream component, function 0 */
211	struct pcie_link_state *root; /* pointer to the root port link */
212	struct pcie_link_state *parent; /* pointer to the parent Link state */
213	struct list_head sibling; /* node in link_list */
214
215	/* ASPM state */
216	u32 aspm_support:7; /* Supported ASPM state */
217	u32 aspm_enabled:7; /* Enabled ASPM state */
218	u32 aspm_capable:7; /* Capable ASPM state with latency */
219	u32 aspm_default:7; /* Default ASPM state by BIOS */
220	u32 aspm_disable:7; /* Disabled ASPM state */
221
222	/* Clock PM state */
223	u32 clkpm_capable:1; /* Clock PM capable? */
224	u32 clkpm_enabled:1; /* Current Clock PM state */
225	u32 clkpm_default:1; /* Default Clock PM state by BIOS */
226	u32 clkpm_disable:1; /* Clock PM disabled */
227	};
228
229	static int aspm_disabled, aspm_force;
230	static bool aspm_support_enabled = true;
231	static DEFINE_MUTEX(aspm_lock);
232	static LIST_HEAD(link_list);
233
234	#define POLICY_DEFAULT 0 /* BIOS default setting */
235	#define POLICY_PERFORMANCE 1 /* high performance */
236	#define POLICY_POWERSAVE 2 /* high power saving */
237	#define POLICY_POWER_SUPERSAVE 3 /* possibly even more power saving */
238
239	#ifdef CONFIG_PCIEASPM_PERFORMANCE
240	static int aspm_policy = POLICY_PERFORMANCE;
241	#elif defined CONFIG_PCIEASPM_POWERSAVE
242	static int aspm_policy = POLICY_POWERSAVE;
243	#elif defined CONFIG_PCIEASPM_POWER_SUPERSAVE
244	static int aspm_policy = POLICY_POWER_SUPERSAVE;
245	#else
246	static int aspm_policy;
247	#endif
248
249	static const char *policy_str[] = {
250	[POLICY_DEFAULT] = "default",
251	[POLICY_PERFORMANCE] = "performance",
252	[POLICY_POWERSAVE] = "powersave",
253	[POLICY_POWER_SUPERSAVE] = "powersupersave"
254	};
255
256	/*
257	* The L1 PM substate capability is only implemented in function 0 in a
258	* multi function device.
259	*/
260	static struct pci_dev *pci_function_0(struct pci_bus *linkbus)
261	{
262	struct pci_dev *child;
263
264	list_for_each_entry(child, &linkbus->devices, bus_list)
265	if (PCI_FUNC(child->devfn) == 0)
266	return child;
267	return NULL;
268	}
269
270	static int policy_to_aspm_state(struct pcie_link_state *link)
271	{
272	switch (aspm_policy) {
273	case POLICY_PERFORMANCE:
274	/* Disable ASPM and Clock PM */
275	return 0;
276	case POLICY_POWERSAVE:
277	/* Enable ASPM L0s/L1 */
278	return (ASPM_STATE_L0S | ASPM_STATE_L1);
279	case POLICY_POWER_SUPERSAVE:
280	/* Enable Everything */
281	return ASPM_STATE_ALL;
282	case POLICY_DEFAULT:
283	return link->aspm_default;
284	}
285	return 0;
286	}
287
288	static int policy_to_clkpm_state(struct pcie_link_state *link)
289	{
290	switch (aspm_policy) {
291	case POLICY_PERFORMANCE:
292	/* Disable ASPM and Clock PM */
293	return 0;
294	case POLICY_POWERSAVE:
295	case POLICY_POWER_SUPERSAVE:
296	/* Enable Clock PM */
297	return 1;
298	case POLICY_DEFAULT:
299	return link->clkpm_default;
300	}
301	return 0;
302	}
303
304	static void pci_update_aspm_saved_state(struct pci_dev *dev)
305	{
306	struct pci_cap_saved_state *save_state;
307	u16 *cap, lnkctl, aspm_ctl;
308
309	save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP);
310	if (!save_state)
311	return;
312
313	pcie_capability_read_word(dev, PCI_EXP_LNKCTL, val: &lnkctl);
314
315	/*
316	* Update ASPM and CLKREQ bits of LNKCTL in save_state. We only
317	* write PCI_EXP_LNKCTL_CCC during enumeration, so it shouldn't
318	* change after being captured in save_state.
319	*/
320	aspm_ctl = lnkctl & (PCI_EXP_LNKCTL_ASPMC | PCI_EXP_LNKCTL_CLKREQ_EN);
321	lnkctl &= ~(PCI_EXP_LNKCTL_ASPMC | PCI_EXP_LNKCTL_CLKREQ_EN);
322
323	/* Depends on pci_save_pcie_state(): cap[1] is LNKCTL */
324	cap = (u16 *)&save_state->cap.data[0];
325	cap[1] = lnkctl | aspm_ctl;
326	}
327
328	static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)
329	{
330	struct pci_dev *child;
331	struct pci_bus *linkbus = link->pdev->subordinate;
332	u32 val = enable ? PCI_EXP_LNKCTL_CLKREQ_EN : 0;
333
334	list_for_each_entry(child, &linkbus->devices, bus_list) {
335	pcie_capability_clear_and_set_word(dev: child, PCI_EXP_LNKCTL,
336	PCI_EXP_LNKCTL_CLKREQ_EN,
337	set: val);
338	pci_update_aspm_saved_state(dev: child);
339	}
340	link->clkpm_enabled = !!enable;
341	}
342
343	static void pcie_set_clkpm(struct pcie_link_state *link, int enable)
344	{
345	/*
346	* Don't enable Clock PM if the link is not Clock PM capable
347	* or Clock PM is disabled
348	*/
349	if (!link->clkpm_capable || link->clkpm_disable)
350	enable = 0;
351	/* Need nothing if the specified equals to current state */
352	if (link->clkpm_enabled == enable)
353	return;
354	pcie_set_clkpm_nocheck(link, enable);
355	}
356
357	static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)
358	{
359	int capable = 1, enabled = 1;
360	u32 reg32;
361	u16 reg16;
362	struct pci_dev *child;
363	struct pci_bus *linkbus = link->pdev->subordinate;
364
365	/* All functions should have the same cap and state, take the worst */
366	list_for_each_entry(child, &linkbus->devices, bus_list) {
367	pcie_capability_read_dword(dev: child, PCI_EXP_LNKCAP, val: &reg32);
368	if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
369	capable = 0;
370	enabled = 0;
371	break;
372	}
373	pcie_capability_read_word(dev: child, PCI_EXP_LNKCTL, val: &reg16);
374	if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
375	enabled = 0;
376	}
377	link->clkpm_enabled = enabled;
378	link->clkpm_default = enabled;
379	link->clkpm_capable = capable;
380	link->clkpm_disable = blacklist ? 1 : 0;
381	}
382
383	/*
384	* pcie_aspm_configure_common_clock: check if the 2 ends of a link
385	* could use common clock. If they are, configure them to use the
386	* common clock. That will reduce the ASPM state exit latency.
387	*/
388	static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
389	{
390	int same_clock = 1;
391	u16 reg16, ccc, parent_old_ccc, child_old_ccc[8];
392	struct pci_dev *child, *parent = link->pdev;
393	struct pci_bus *linkbus = parent->subordinate;
394	/*
395	* All functions of a slot should have the same Slot Clock
396	* Configuration, so just check one function
397	*/
398	child = list_entry(linkbus->devices.next, struct pci_dev, bus_list);
399	BUG_ON(!pci_is_pcie(child));
400
401	/* Check downstream component if bit Slot Clock Configuration is 1 */
402	pcie_capability_read_word(dev: child, PCI_EXP_LNKSTA, val: &reg16);
403	if (!(reg16 & PCI_EXP_LNKSTA_SLC))
404	same_clock = 0;
405
406	/* Check upstream component if bit Slot Clock Configuration is 1 */
407	pcie_capability_read_word(dev: parent, PCI_EXP_LNKSTA, val: &reg16);
408	if (!(reg16 & PCI_EXP_LNKSTA_SLC))
409	same_clock = 0;
410
411	/* Port might be already in common clock mode */
412	pcie_capability_read_word(dev: parent, PCI_EXP_LNKCTL, val: &reg16);
413	parent_old_ccc = reg16 & PCI_EXP_LNKCTL_CCC;
414	if (same_clock && (reg16 & PCI_EXP_LNKCTL_CCC)) {
415	bool consistent = true;
416
417	list_for_each_entry(child, &linkbus->devices, bus_list) {
418	pcie_capability_read_word(dev: child, PCI_EXP_LNKCTL,
419	val: &reg16);
420	if (!(reg16 & PCI_EXP_LNKCTL_CCC)) {
421	consistent = false;
422	break;
423	}
424	}
425	if (consistent)
426	return;
427	pci_info(parent, "ASPM: current common clock configuration is inconsistent, reconfiguring\n");
428	}
429
430	ccc = same_clock ? PCI_EXP_LNKCTL_CCC : 0;
431	/* Configure downstream component, all functions */
432	list_for_each_entry(child, &linkbus->devices, bus_list) {
433	pcie_capability_read_word(dev: child, PCI_EXP_LNKCTL, val: &reg16);
434	child_old_ccc[PCI_FUNC(child->devfn)] = reg16 & PCI_EXP_LNKCTL_CCC;
435	pcie_capability_clear_and_set_word(dev: child, PCI_EXP_LNKCTL,
436	PCI_EXP_LNKCTL_CCC, set: ccc);
437	}
438
439	/* Configure upstream component */
440	pcie_capability_clear_and_set_word(dev: parent, PCI_EXP_LNKCTL,
441	PCI_EXP_LNKCTL_CCC, set: ccc);
442
443	if (pcie_retrain_link(pdev: link->pdev, use_lt: true)) {
444
445	/* Training failed. Restore common clock configurations */
446	pci_err(parent, "ASPM: Could not configure common clock\n");
447	list_for_each_entry(child, &linkbus->devices, bus_list)
448	pcie_capability_clear_and_set_word(dev: child, PCI_EXP_LNKCTL,
449	PCI_EXP_LNKCTL_CCC,
450	set: child_old_ccc[PCI_FUNC(child->devfn)]);
451	pcie_capability_clear_and_set_word(dev: parent, PCI_EXP_LNKCTL,
452	PCI_EXP_LNKCTL_CCC, set: parent_old_ccc);
453	}
454	}
455
456	/* Convert L0s latency encoding to ns */
457	static u32 calc_l0s_latency(u32 lnkcap)
458	{
459	u32 encoding = FIELD_GET(PCI_EXP_LNKCAP_L0SEL, lnkcap);
460
461	if (encoding == 0x7)
462	return 5 * NSEC_PER_USEC; /* > 4us */
463	return (64 << encoding);
464	}
465
466	/* Convert L0s acceptable latency encoding to ns */
467	static u32 calc_l0s_acceptable(u32 encoding)
468	{
469	if (encoding == 0x7)
470	return U32_MAX;
471	return (64 << encoding);
472	}
473
474	/* Convert L1 latency encoding to ns */
475	static u32 calc_l1_latency(u32 lnkcap)
476	{
477	u32 encoding = FIELD_GET(PCI_EXP_LNKCAP_L1EL, lnkcap);
478
479	if (encoding == 0x7)
480	return 65 * NSEC_PER_USEC; /* > 64us */
481	return NSEC_PER_USEC << encoding;
482	}
483
484	/* Convert L1 acceptable latency encoding to ns */
485	static u32 calc_l1_acceptable(u32 encoding)
486	{
487	if (encoding == 0x7)
488	return U32_MAX;
489	return NSEC_PER_USEC << encoding;
490	}
491
492	/* Convert L1SS T_pwr encoding to usec */
493	static u32 calc_l12_pwron(struct pci_dev *pdev, u32 scale, u32 val)
494	{
495	switch (scale) {
496	case 0:
497	return val * 2;
498	case 1:
499	return val * 10;
500	case 2:
501	return val * 100;
502	}
503	pci_err(pdev, "%s: Invalid T_PwrOn scale: %u\n", __func__, scale);
504	return 0;
505	}
506
507	/*
508	* Encode an LTR_L1.2_THRESHOLD value for the L1 PM Substates Control 1
509	* register. Ports enter L1.2 when the most recent LTR value is greater
510	* than or equal to LTR_L1.2_THRESHOLD, so we round up to make sure we
511	* don't enter L1.2 too aggressively.
512	*
513	* See PCIe r6.0, sec 5.5.1, 6.18, 7.8.3.3.
514	*/
515	static void encode_l12_threshold(u32 threshold_us, u32 *scale, u32 *value)
516	{
517	u64 threshold_ns = (u64)threshold_us * NSEC_PER_USEC;
518
519	/*
520	* LTR_L1.2_THRESHOLD_Value ("value") is a 10-bit field with max
521	* value of 0x3ff.
522	*/
523	if (threshold_ns <= 1 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
524	*scale = 0; /* Value times 1ns */
525	*value = threshold_ns;
526	} else if (threshold_ns <= 32 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
527	*scale = 1; /* Value times 32ns */
528	*value = roundup(threshold_ns, 32) / 32;
529	} else if (threshold_ns <= 1024 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
530	*scale = 2; /* Value times 1024ns */
531	*value = roundup(threshold_ns, 1024) / 1024;
532	} else if (threshold_ns <= 32768 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
533	*scale = 3; /* Value times 32768ns */
534	*value = roundup(threshold_ns, 32768) / 32768;
535	} else if (threshold_ns <= 1048576 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
536	*scale = 4; /* Value times 1048576ns */
537	*value = roundup(threshold_ns, 1048576) / 1048576;
538	} else if (threshold_ns <= (u64)33554432 * FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE)) {
539	*scale = 5; /* Value times 33554432ns */
540	*value = roundup(threshold_ns, 33554432) / 33554432;
541	} else {
542	*scale = 5;
543	*value = FIELD_MAX(PCI_L1SS_CTL1_LTR_L12_TH_VALUE);
544	}
545	}
546
547	static void pcie_aspm_check_latency(struct pci_dev *endpoint)
548	{
549	u32 latency, encoding, lnkcap_up, lnkcap_dw;
550	u32 l1_switch_latency = 0, latency_up_l0s;
551	u32 latency_up_l1, latency_dw_l0s, latency_dw_l1;
552	u32 acceptable_l0s, acceptable_l1;
553	struct pcie_link_state *link;
554
555	/* Device not in D0 doesn't need latency check */
556	if ((endpoint->current_state != PCI_D0) &&
557	(endpoint->current_state != PCI_UNKNOWN))
558	return;
559
560	link = endpoint->bus->self->link_state;
561
562	/* Calculate endpoint L0s acceptable latency */
563	encoding = FIELD_GET(PCI_EXP_DEVCAP_L0S, endpoint->devcap);
564	acceptable_l0s = calc_l0s_acceptable(encoding);
565
566	/* Calculate endpoint L1 acceptable latency */
567	encoding = FIELD_GET(PCI_EXP_DEVCAP_L1, endpoint->devcap);
568	acceptable_l1 = calc_l1_acceptable(encoding);
569
570	while (link) {
571	struct pci_dev *dev = pci_function_0(linkbus: link->pdev->subordinate);
572
573	/* Read direction exit latencies */
574	pcie_capability_read_dword(dev: link->pdev, PCI_EXP_LNKCAP,
575	val: &lnkcap_up);
576	pcie_capability_read_dword(dev, PCI_EXP_LNKCAP,
577	val: &lnkcap_dw);
578	latency_up_l0s = calc_l0s_latency(lnkcap: lnkcap_up);
579	latency_up_l1 = calc_l1_latency(lnkcap: lnkcap_up);
580	latency_dw_l0s = calc_l0s_latency(lnkcap: lnkcap_dw);
581	latency_dw_l1 = calc_l1_latency(lnkcap: lnkcap_dw);
582
583	/* Check upstream direction L0s latency */
584	if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&
585	(latency_up_l0s > acceptable_l0s))
586	link->aspm_capable &= ~ASPM_STATE_L0S_UP;
587
588	/* Check downstream direction L0s latency */
589	if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&
590	(latency_dw_l0s > acceptable_l0s))
591	link->aspm_capable &= ~ASPM_STATE_L0S_DW;
592	/*
593	* Check L1 latency.
594	* Every switch on the path to root complex need 1
595	* more microsecond for L1. Spec doesn't mention L0s.
596	*
597	* The exit latencies for L1 substates are not advertised
598	* by a device. Since the spec also doesn't mention a way
599	* to determine max latencies introduced by enabling L1
600	* substates on the components, it is not clear how to do
601	* a L1 substate exit latency check. We assume that the
602	* L1 exit latencies advertised by a device include L1
603	* substate latencies (and hence do not do any check).
604	*/
605	latency = max_t(u32, latency_up_l1, latency_dw_l1);
606	if ((link->aspm_capable & ASPM_STATE_L1) &&
607	(latency + l1_switch_latency > acceptable_l1))
608	link->aspm_capable &= ~ASPM_STATE_L1;
609	l1_switch_latency += NSEC_PER_USEC;
610
611	link = link->parent;
612	}
613	}
614
615	/* Calculate L1.2 PM substate timing parameters */
616	static void aspm_calc_l12_info(struct pcie_link_state *link,
617	u32 parent_l1ss_cap, u32 child_l1ss_cap)
618	{
619	struct pci_dev *child = link->downstream, *parent = link->pdev;
620	u32 val1, val2, scale1, scale2;
621	u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
622	u32 ctl1 = 0, ctl2 = 0;
623	u32 pctl1, pctl2, cctl1, cctl2;
624	u32 pl1_2_enables, cl1_2_enables;
625
626	/* Choose the greater of the two Port Common_Mode_Restore_Times */
627	val1 = FIELD_GET(PCI_L1SS_CAP_CM_RESTORE_TIME, parent_l1ss_cap);
628	val2 = FIELD_GET(PCI_L1SS_CAP_CM_RESTORE_TIME, child_l1ss_cap);
629	t_common_mode = max(val1, val2);
630
631	/* Choose the greater of the two Port T_POWER_ON times */
632	val1 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_VALUE, parent_l1ss_cap);
633	scale1 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_SCALE, parent_l1ss_cap);
634	val2 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_VALUE, child_l1ss_cap);
635	scale2 = FIELD_GET(PCI_L1SS_CAP_P_PWR_ON_SCALE, child_l1ss_cap);
636
637	if (calc_l12_pwron(pdev: parent, scale: scale1, val: val1) >
638	calc_l12_pwron(pdev: child, scale: scale2, val: val2)) {
639	ctl2 |= FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_SCALE, scale1) |
640	FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_VALUE, val1);
641	t_power_on = calc_l12_pwron(pdev: parent, scale: scale1, val: val1);
642	} else {
643	ctl2 |= FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_SCALE, scale2) |
644	FIELD_PREP(PCI_L1SS_CTL2_T_PWR_ON_VALUE, val2);
645	t_power_on = calc_l12_pwron(pdev: child, scale: scale2, val: val2);
646	}
647
648	/*
649	* Set LTR_L1.2_THRESHOLD to the time required to transition the
650	* Link from L0 to L1.2 and back to L0 so we enter L1.2 only if
651	* downstream devices report (via LTR) that they can tolerate at
652	* least that much latency.
653	*
654	* Based on PCIe r3.1, sec 5.5.3.3.1, Figures 5-16 and 5-17, and
655	* Table 5-11. T(POWER_OFF) is at most 2us and T(L1.2) is at
656	* least 4us.
657	*/
658	l1_2_threshold = 2 + 4 + t_common_mode + t_power_on;
659	encode_l12_threshold(threshold_us: l1_2_threshold, scale: &scale, value: &value);
660	ctl1 |= FIELD_PREP(PCI_L1SS_CTL1_CM_RESTORE_TIME, t_common_mode) |
661	FIELD_PREP(PCI_L1SS_CTL1_LTR_L12_TH_VALUE, value) |
662	FIELD_PREP(PCI_L1SS_CTL1_LTR_L12_TH_SCALE, scale);
663
664	/* Some broken devices only support dword access to L1 SS */
665	pci_read_config_dword(dev: parent, where: parent->l1ss + PCI_L1SS_CTL1, val: &pctl1);
666	pci_read_config_dword(dev: parent, where: parent->l1ss + PCI_L1SS_CTL2, val: &pctl2);
667	pci_read_config_dword(dev: child, where: child->l1ss + PCI_L1SS_CTL1, val: &cctl1);
668	pci_read_config_dword(dev: child, where: child->l1ss + PCI_L1SS_CTL2, val: &cctl2);
669
670	if (ctl1 == pctl1 && ctl1 == cctl1 &&
671	ctl2 == pctl2 && ctl2 == cctl2)
672	return;
673
674	/* Disable L1.2 while updating. See PCIe r5.0, sec 5.5.4, 7.8.3.3 */
675	pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;
676	cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;
677
678	if (pl1_2_enables || cl1_2_enables) {
679	pci_clear_and_set_config_dword(dev: child,
680	pos: child->l1ss + PCI_L1SS_CTL1,
681	PCI_L1SS_CTL1_L1_2_MASK, set: 0);
682	pci_clear_and_set_config_dword(dev: parent,
683	pos: parent->l1ss + PCI_L1SS_CTL1,
684	PCI_L1SS_CTL1_L1_2_MASK, set: 0);
685	}
686
687	/* Program T_POWER_ON times in both ports */
688	pci_write_config_dword(dev: parent, where: parent->l1ss + PCI_L1SS_CTL2, val: ctl2);
689	pci_write_config_dword(dev: child, where: child->l1ss + PCI_L1SS_CTL2, val: ctl2);
690
691	/* Program Common_Mode_Restore_Time in upstream device */
692	pci_clear_and_set_config_dword(dev: parent, pos: parent->l1ss + PCI_L1SS_CTL1,
693	PCI_L1SS_CTL1_CM_RESTORE_TIME, set: ctl1);
694
695	/* Program LTR_L1.2_THRESHOLD time in both ports */
696	pci_clear_and_set_config_dword(dev: parent, pos: parent->l1ss + PCI_L1SS_CTL1,
697	PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
698	PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
699	set: ctl1);
700	pci_clear_and_set_config_dword(dev: child, pos: child->l1ss + PCI_L1SS_CTL1,
701	PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
702	PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
703	set: ctl1);
704
705	if (pl1_2_enables || cl1_2_enables) {
706	pci_clear_and_set_config_dword(dev: parent,
707	pos: parent->l1ss + PCI_L1SS_CTL1, clear: 0,
708	set: pl1_2_enables);
709	pci_clear_and_set_config_dword(dev: child,
710	pos: child->l1ss + PCI_L1SS_CTL1, clear: 0,
711	set: cl1_2_enables);
712	}
713	}
714
715	static void aspm_l1ss_init(struct pcie_link_state *link)
716	{
717	struct pci_dev *child = link->downstream, *parent = link->pdev;
718	u32 parent_l1ss_cap, child_l1ss_cap;
719	u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;
720
721	if (!parent->l1ss || !child->l1ss)
722	return;
723
724	/* Setup L1 substate */
725	pci_read_config_dword(dev: parent, where: parent->l1ss + PCI_L1SS_CAP,
726	val: &parent_l1ss_cap);
727	pci_read_config_dword(dev: child, where: child->l1ss + PCI_L1SS_CAP,
728	val: &child_l1ss_cap);
729
730	if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
731	parent_l1ss_cap = 0;
732	if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
733	child_l1ss_cap = 0;
734
735	/*
736	* If we don't have LTR for the entire path from the Root Complex
737	* to this device, we can't use ASPM L1.2 because it relies on the
738	* LTR_L1.2_THRESHOLD. See PCIe r4.0, secs 5.5.4, 6.18.
739	*/
740	if (!child->ltr_path)
741	child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
742
743	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
744	link->aspm_support |= ASPM_STATE_L1_1;
745	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
746	link->aspm_support |= ASPM_STATE_L1_2;
747	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
748	link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
749	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
750	link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
751
752	if (parent_l1ss_cap)
753	pci_read_config_dword(dev: parent, where: parent->l1ss + PCI_L1SS_CTL1,
754	val: &parent_l1ss_ctl1);
755	if (child_l1ss_cap)
756	pci_read_config_dword(dev: child, where: child->l1ss + PCI_L1SS_CTL1,
757	val: &child_l1ss_ctl1);
758
759	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
760	link->aspm_enabled |= ASPM_STATE_L1_1;
761	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
762	link->aspm_enabled |= ASPM_STATE_L1_2;
763	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
764	link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
765	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
766	link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
767
768	if (link->aspm_support & ASPM_STATE_L1_2_MASK)
769	aspm_calc_l12_info(link, parent_l1ss_cap, child_l1ss_cap);
770	}
771
772	static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
773	{
774	struct pci_dev *child = link->downstream, *parent = link->pdev;
775	u32 parent_lnkcap, child_lnkcap;
776	u16 parent_lnkctl, child_lnkctl;
777	struct pci_bus *linkbus = parent->subordinate;
778
779	if (blacklist) {
780	/* Set enabled/disable so that we will disable ASPM later */
781	link->aspm_enabled = ASPM_STATE_ALL;
782	link->aspm_disable = ASPM_STATE_ALL;
783	return;
784	}
785
786	/*
787	* If ASPM not supported, don't mess with the clocks and link,
788	* bail out now.
789	*/
790	pcie_capability_read_dword(dev: parent, PCI_EXP_LNKCAP, val: &parent_lnkcap);
791	pcie_capability_read_dword(dev: child, PCI_EXP_LNKCAP, val: &child_lnkcap);
792	if (!(parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPMS))
793	return;
794
795	/* Configure common clock before checking latencies */
796	pcie_aspm_configure_common_clock(link);
797
798	/*
799	* Re-read upstream/downstream components' register state after
800	* clock configuration. L0s & L1 exit latencies in the otherwise
801	* read-only Link Capabilities may change depending on common clock
802	* configuration (PCIe r5.0, sec 7.5.3.6).
803	*/
804	pcie_capability_read_dword(dev: parent, PCI_EXP_LNKCAP, val: &parent_lnkcap);
805	pcie_capability_read_dword(dev: child, PCI_EXP_LNKCAP, val: &child_lnkcap);
806	pcie_capability_read_word(dev: parent, PCI_EXP_LNKCTL, val: &parent_lnkctl);
807	pcie_capability_read_word(dev: child, PCI_EXP_LNKCTL, val: &child_lnkctl);
808
809	/*
810	* Setup L0s state
811	*
812	* Note that we must not enable L0s in either direction on a
813	* given link unless components on both sides of the link each
814	* support L0s.
815	*/
816	if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S)
817	link->aspm_support |= ASPM_STATE_L0S;
818
819	if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
820	link->aspm_enabled |= ASPM_STATE_L0S_UP;
821	if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
822	link->aspm_enabled |= ASPM_STATE_L0S_DW;
823
824	/* Setup L1 state */
825	if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1)
826	link->aspm_support |= ASPM_STATE_L1;
827
828	if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1)
829	link->aspm_enabled |= ASPM_STATE_L1;
830
831	aspm_l1ss_init(link);
832
833	/* Save default state */
834	link->aspm_default = link->aspm_enabled;
835
836	/* Setup initial capable state. Will be updated later */
837	link->aspm_capable = link->aspm_support;
838
839	/* Get and check endpoint acceptable latencies */
840	list_for_each_entry(child, &linkbus->devices, bus_list) {
841	if (pci_pcie_type(dev: child) != PCI_EXP_TYPE_ENDPOINT &&
842	pci_pcie_type(dev: child) != PCI_EXP_TYPE_LEG_END)
843	continue;
844
845	pcie_aspm_check_latency(endpoint: child);
846	}
847	}
848
849	/* Configure the ASPM L1 substates */
850	static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
851	{
852	u32 val, enable_req;
853	struct pci_dev *child = link->downstream, *parent = link->pdev;
854
855	enable_req = (link->aspm_enabled ^ state) & state;
856
857	/*
858	* Here are the rules specified in the PCIe spec for enabling L1SS:
859	* - When enabling L1.x, enable bit at parent first, then at child
860	* - When disabling L1.x, disable bit at child first, then at parent
861	* - When enabling ASPM L1.x, need to disable L1
862	* (at child followed by parent).
863	* - The ASPM/PCIPM L1.2 must be disabled while programming timing
864	* parameters
865	*
866	* To keep it simple, disable all L1SS bits first, and later enable
867	* what is needed.
868	*/
869
870	/* Disable all L1 substates */
871	pci_clear_and_set_config_dword(dev: child, pos: child->l1ss + PCI_L1SS_CTL1,
872	PCI_L1SS_CTL1_L1SS_MASK, set: 0);
873	pci_clear_and_set_config_dword(dev: parent, pos: parent->l1ss + PCI_L1SS_CTL1,
874	PCI_L1SS_CTL1_L1SS_MASK, set: 0);
875	/*
876	* If needed, disable L1, and it gets enabled later
877	* in pcie_config_aspm_link().
878	*/
879	if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2)) {
880	pcie_capability_clear_word(dev: child, PCI_EXP_LNKCTL,
881	PCI_EXP_LNKCTL_ASPM_L1);
882	pcie_capability_clear_word(dev: parent, PCI_EXP_LNKCTL,
883	PCI_EXP_LNKCTL_ASPM_L1);
884	}
885
886	val = 0;
887	if (state & ASPM_STATE_L1_1)
888	val |= PCI_L1SS_CTL1_ASPM_L1_1;
889	if (state & ASPM_STATE_L1_2)
890	val |= PCI_L1SS_CTL1_ASPM_L1_2;
891	if (state & ASPM_STATE_L1_1_PCIPM)
892	val |= PCI_L1SS_CTL1_PCIPM_L1_1;
893	if (state & ASPM_STATE_L1_2_PCIPM)
894	val |= PCI_L1SS_CTL1_PCIPM_L1_2;
895
896	/* Enable what we need to enable */
897	pci_clear_and_set_config_dword(dev: parent, pos: parent->l1ss + PCI_L1SS_CTL1,
898	PCI_L1SS_CTL1_L1SS_MASK, set: val);
899	pci_clear_and_set_config_dword(dev: child, pos: child->l1ss + PCI_L1SS_CTL1,
900	PCI_L1SS_CTL1_L1SS_MASK, set: val);
901	}
902
903	static void pcie_config_aspm_dev(struct pci_dev *pdev, u32 val)
904	{
905	pcie_capability_clear_and_set_word(dev: pdev, PCI_EXP_LNKCTL,
906	PCI_EXP_LNKCTL_ASPMC, set: val);
907	}
908
909	static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
910	{
911	u32 upstream = 0, dwstream = 0;
912	struct pci_dev *child = link->downstream, *parent = link->pdev;
913	struct pci_bus *linkbus = parent->subordinate;
914
915	/* Enable only the states that were not explicitly disabled */
916	state &= (link->aspm_capable & ~link->aspm_disable);
917
918	/* Can't enable any substates if L1 is not enabled */
919	if (!(state & ASPM_STATE_L1))
920	state &= ~ASPM_STATE_L1SS;
921
922	/* Spec says both ports must be in D0 before enabling PCI PM substates*/
923	if (parent->current_state != PCI_D0 || child->current_state != PCI_D0) {
924	state &= ~ASPM_STATE_L1_SS_PCIPM;
925	state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);
926	}
927
928	/* Nothing to do if the link is already in the requested state */
929	if (link->aspm_enabled == state)
930	return;
931	/* Convert ASPM state to upstream/downstream ASPM register state */
932	if (state & ASPM_STATE_L0S_UP)
933	dwstream |= PCI_EXP_LNKCTL_ASPM_L0S;
934	if (state & ASPM_STATE_L0S_DW)
935	upstream |= PCI_EXP_LNKCTL_ASPM_L0S;
936	if (state & ASPM_STATE_L1) {
937	upstream |= PCI_EXP_LNKCTL_ASPM_L1;
938	dwstream |= PCI_EXP_LNKCTL_ASPM_L1;
939	}
940
941	if (link->aspm_capable & ASPM_STATE_L1SS)
942	pcie_config_aspm_l1ss(link, state);
943
944	/*
945	* Spec 2.0 suggests all functions should be configured the
946	* same setting for ASPM. Enabling ASPM L1 should be done in
947	* upstream component first and then downstream, and vice
948	* versa for disabling ASPM L1. Spec doesn't mention L0S.
949	*/
950	if (state & ASPM_STATE_L1)
951	pcie_config_aspm_dev(pdev: parent, val: upstream);
952	list_for_each_entry(child, &linkbus->devices, bus_list)
953	pcie_config_aspm_dev(pdev: child, val: dwstream);
954	if (!(state & ASPM_STATE_L1))
955	pcie_config_aspm_dev(pdev: parent, val: upstream);
956
957	link->aspm_enabled = state;
958
959	/* Update latest ASPM configuration in saved context */
960	pci_save_aspm_l1ss_state(pdev: link->downstream);
961	pci_update_aspm_saved_state(dev: link->downstream);
962	pci_save_aspm_l1ss_state(pdev: parent);
963	pci_update_aspm_saved_state(dev: parent);
964	}
965
966	static void pcie_config_aspm_path(struct pcie_link_state *link)
967	{
968	while (link) {
969	pcie_config_aspm_link(link, state: policy_to_aspm_state(link));
970	link = link->parent;
971	}
972	}
973
974	static void free_link_state(struct pcie_link_state *link)
975	{
976	link->pdev->link_state = NULL;
977	kfree(objp: link);
978	}
979
980	static int pcie_aspm_sanity_check(struct pci_dev *pdev)
981	{
982	struct pci_dev *child;
983	u32 reg32;
984
985	/*
986	* Some functions in a slot might not all be PCIe functions,
987	* very strange. Disable ASPM for the whole slot
988	*/
989	list_for_each_entry(child, &pdev->subordinate->devices, bus_list) {
990	if (!pci_is_pcie(dev: child))
991	return -EINVAL;
992
993	/*
994	* If ASPM is disabled then we're not going to change
995	* the BIOS state. It's safe to continue even if it's a
996	* pre-1.1 device
997	*/
998
999	if (aspm_disabled)
1000	continue;
1001
1002	/*
1003	* Disable ASPM for pre-1.1 PCIe device, we follow MS to use
1004	* RBER bit to determine if a function is 1.1 version device
1005	*/
1006	pcie_capability_read_dword(dev: child, PCI_EXP_DEVCAP, val: &reg32);
1007	if (!(reg32 & PCI_EXP_DEVCAP_RBER) && !aspm_force) {
1008	pci_info(child, "disabling ASPM on pre-1.1 PCIe device. You can enable it with 'pcie_aspm=force'\n");
1009	return -EINVAL;
1010	}
1011	}
1012	return 0;
1013	}
1014
1015	static struct pcie_link_state *alloc_pcie_link_state(struct pci_dev *pdev)
1016	{
1017	struct pcie_link_state *link;
1018
1019	link = kzalloc(size: sizeof(*link), GFP_KERNEL);
1020	if (!link)
1021	return NULL;
1022
1023	INIT_LIST_HEAD(list: &link->sibling);
1024	link->pdev = pdev;
1025	link->downstream = pci_function_0(linkbus: pdev->subordinate);
1026
1027	/*
1028	* Root Ports and PCI/PCI-X to PCIe Bridges are roots of PCIe
1029	* hierarchies. Note that some PCIe host implementations omit
1030	* the root ports entirely, in which case a downstream port on
1031	* a switch may become the root of the link state chain for all
1032	* its subordinate endpoints.
1033	*/
1034	if (pci_pcie_type(dev: pdev) == PCI_EXP_TYPE_ROOT_PORT ||
1035	pci_pcie_type(dev: pdev) == PCI_EXP_TYPE_PCIE_BRIDGE ||
1036	!pdev->bus->parent->self) {
1037	link->root = link;
1038	} else {
1039	struct pcie_link_state *parent;
1040
1041	parent = pdev->bus->parent->self->link_state;
1042	if (!parent) {
1043	kfree(objp: link);
1044	return NULL;
1045	}
1046
1047	link->parent = parent;
1048	link->root = link->parent->root;
1049	}
1050
1051	list_add(new: &link->sibling, head: &link_list);
1052	pdev->link_state = link;
1053	return link;
1054	}
1055
1056	static void pcie_aspm_update_sysfs_visibility(struct pci_dev *pdev)
1057	{
1058	struct pci_dev *child;
1059
1060	list_for_each_entry(child, &pdev->subordinate->devices, bus_list)
1061	sysfs_update_group(kobj: &child->dev.kobj, grp: &aspm_ctrl_attr_group);
1062	}
1063
1064	/*
1065	* pcie_aspm_init_link_state: Initiate PCI express link state.
1066	* It is called after the pcie and its children devices are scanned.
1067	* @pdev: the root port or switch downstream port
1068	*/
1069	void pcie_aspm_init_link_state(struct pci_dev *pdev)
1070	{
1071	struct pcie_link_state *link;
1072	int blacklist = !!pcie_aspm_sanity_check(pdev);
1073
1074	if (!aspm_support_enabled)
1075	return;
1076
1077	if (pdev->link_state)
1078	return;
1079
1080	/*
1081	* We allocate pcie_link_state for the component on the upstream
1082	* end of a Link, so there's nothing to do unless this device is
1083	* downstream port.
1084	*/
1085	if (!pcie_downstream_port(dev: pdev))
1086	return;
1087
1088	/* VIA has a strange chipset, root port is under a bridge */
1089	if (pci_pcie_type(dev: pdev) == PCI_EXP_TYPE_ROOT_PORT &&
1090	pdev->bus->self)
1091	return;
1092
1093	down_read(sem: &pci_bus_sem);
1094	if (list_empty(head: &pdev->subordinate->devices))
1095	goto out;
1096
1097	mutex_lock(&aspm_lock);
1098	link = alloc_pcie_link_state(pdev);
1099	if (!link)
1100	goto unlock;
1101	/*
1102	* Setup initial ASPM state. Note that we need to configure
1103	* upstream links also because capable state of them can be
1104	* update through pcie_aspm_cap_init().
1105	*/
1106	pcie_aspm_cap_init(link, blacklist);
1107
1108	/* Setup initial Clock PM state */
1109	pcie_clkpm_cap_init(link, blacklist);
1110
1111	/*
1112	* At this stage drivers haven't had an opportunity to change the
1113	* link policy setting. Enabling ASPM on broken hardware can cripple
1114	* it even before the driver has had a chance to disable ASPM, so
1115	* default to a safe level right now. If we're enabling ASPM beyond
1116	* the BIOS's expectation, we'll do so once pci_enable_device() is
1117	* called.
1118	*/
1119	if (aspm_policy != POLICY_POWERSAVE &&
1120	aspm_policy != POLICY_POWER_SUPERSAVE) {
1121	pcie_config_aspm_path(link);
1122	pcie_set_clkpm(link, enable: policy_to_clkpm_state(link));
1123	}
1124
1125	pcie_aspm_update_sysfs_visibility(pdev);
1126
1127	unlock:
1128	mutex_unlock(lock: &aspm_lock);
1129	out:
1130	up_read(sem: &pci_bus_sem);
1131	}
1132
1133	void pci_bridge_reconfigure_ltr(struct pci_dev *pdev)
1134	{
1135	struct pci_dev *bridge;
1136	u32 ctl;
1137
1138	bridge = pci_upstream_bridge(dev: pdev);
1139	if (bridge && bridge->ltr_path) {
1140	pcie_capability_read_dword(dev: bridge, PCI_EXP_DEVCTL2, val: &ctl);
1141	if (!(ctl & PCI_EXP_DEVCTL2_LTR_EN)) {
1142	pci_dbg(bridge, "re-enabling LTR\n");
1143	pcie_capability_set_word(dev: bridge, PCI_EXP_DEVCTL2,
1144	PCI_EXP_DEVCTL2_LTR_EN);
1145	}
1146	}
1147	}
1148
1149	void pci_configure_ltr(struct pci_dev *pdev)
1150	{
1151	struct pci_host_bridge *host = pci_find_host_bridge(bus: pdev->bus);
1152	struct pci_dev *bridge;
1153	u32 cap, ctl;
1154
1155	if (!pci_is_pcie(dev: pdev))
1156	return;
1157
1158	pcie_capability_read_dword(dev: pdev, PCI_EXP_DEVCAP2, val: &cap);
1159	if (!(cap & PCI_EXP_DEVCAP2_LTR))
1160	return;
1161
1162	pcie_capability_read_dword(dev: pdev, PCI_EXP_DEVCTL2, val: &ctl);
1163	if (ctl & PCI_EXP_DEVCTL2_LTR_EN) {
1164	if (pci_pcie_type(dev: pdev) == PCI_EXP_TYPE_ROOT_PORT) {
1165	pdev->ltr_path = 1;
1166	return;
1167	}
1168
1169	bridge = pci_upstream_bridge(dev: pdev);
1170	if (bridge && bridge->ltr_path)
1171	pdev->ltr_path = 1;
1172
1173	return;
1174	}
1175
1176	if (!host->native_ltr)
1177	return;
1178
1179	/*
1180	* Software must not enable LTR in an Endpoint unless the Root
1181	* Complex and all intermediate Switches indicate support for LTR.
1182	* PCIe r4.0, sec 6.18.
1183	*/
1184	if (pci_pcie_type(dev: pdev) == PCI_EXP_TYPE_ROOT_PORT) {
1185	pcie_capability_set_word(dev: pdev, PCI_EXP_DEVCTL2,
1186	PCI_EXP_DEVCTL2_LTR_EN);
1187	pdev->ltr_path = 1;
1188	return;
1189	}
1190
1191	/*
1192	* If we're configuring a hot-added device, LTR was likely
1193	* disabled in the upstream bridge, so re-enable it before enabling
1194	* it in the new device.
1195	*/
1196	bridge = pci_upstream_bridge(dev: pdev);
1197	if (bridge && bridge->ltr_path) {
1198	pci_bridge_reconfigure_ltr(pdev);
1199	pcie_capability_set_word(dev: pdev, PCI_EXP_DEVCTL2,
1200	PCI_EXP_DEVCTL2_LTR_EN);
1201	pdev->ltr_path = 1;
1202	}
1203	}
1204
1205	/* Recheck latencies and update aspm_capable for links under the root */
1206	static void pcie_update_aspm_capable(struct pcie_link_state *root)
1207	{
1208	struct pcie_link_state *link;
1209	BUG_ON(root->parent);
1210	list_for_each_entry(link, &link_list, sibling) {
1211	if (link->root != root)
1212	continue;
1213	link->aspm_capable = link->aspm_support;
1214	}
1215	list_for_each_entry(link, &link_list, sibling) {
1216	struct pci_dev *child;
1217	struct pci_bus *linkbus = link->pdev->subordinate;
1218	if (link->root != root)
1219	continue;
1220	list_for_each_entry(child, &linkbus->devices, bus_list) {
1221	if ((pci_pcie_type(dev: child) != PCI_EXP_TYPE_ENDPOINT) &&
1222	(pci_pcie_type(dev: child) != PCI_EXP_TYPE_LEG_END))
1223	continue;
1224	pcie_aspm_check_latency(endpoint: child);
1225	}
1226	}
1227	}
1228
1229	/* @pdev: the endpoint device */
1230	void pcie_aspm_exit_link_state(struct pci_dev *pdev)
1231	{
1232	struct pci_dev *parent = pdev->bus->self;
1233	struct pcie_link_state *link, *root, *parent_link;
1234
1235	if (!parent || !parent->link_state)
1236	return;
1237
1238	down_read(sem: &pci_bus_sem);
1239	mutex_lock(&aspm_lock);
1240
1241	link = parent->link_state;
1242	root = link->root;
1243	parent_link = link->parent;
1244
1245	/*
1246	* link->downstream is a pointer to the pci_dev of function 0. If
1247	* we remove that function, the pci_dev is about to be deallocated,
1248	* so we can't use link->downstream again. Free the link state to
1249	* avoid this.
1250	*
1251	* If we're removing a non-0 function, it's possible we could
1252	* retain the link state, but PCIe r6.0, sec 7.5.3.7, recommends
1253	* programming the same ASPM Control value for all functions of
1254	* multi-function devices, so disable ASPM for all of them.
1255	*/
1256	pcie_config_aspm_link(link, state: 0);
1257	list_del(entry: &link->sibling);
1258	free_link_state(link);
1259
1260	/* Recheck latencies and configure upstream links */
1261	if (parent_link) {
1262	pcie_update_aspm_capable(root);
1263	pcie_config_aspm_path(link: parent_link);
1264	}
1265
1266	mutex_unlock(lock: &aspm_lock);
1267	up_read(sem: &pci_bus_sem);
1268	}
1269
1270	/*
1271	* @pdev: the root port or switch downstream port
1272	* @locked: whether pci_bus_sem is held
1273	*/
1274	void pcie_aspm_pm_state_change(struct pci_dev *pdev, bool locked)
1275	{
1276	struct pcie_link_state *link = pdev->link_state;
1277
1278	if (aspm_disabled || !link)
1279	return;
1280	/*
1281	* Devices changed PM state, we should recheck if latency
1282	* meets all functions' requirement
1283	*/
1284	if (!locked)
1285	down_read(sem: &pci_bus_sem);
1286	mutex_lock(&aspm_lock);
1287	pcie_update_aspm_capable(root: link->root);
1288	pcie_config_aspm_path(link);
1289	mutex_unlock(lock: &aspm_lock);
1290	if (!locked)
1291	up_read(sem: &pci_bus_sem);
1292	}
1293
1294	void pcie_aspm_powersave_config_link(struct pci_dev *pdev)
1295	{
1296	struct pcie_link_state *link = pdev->link_state;
1297
1298	if (aspm_disabled || !link)
1299	return;
1300
1301	if (aspm_policy != POLICY_POWERSAVE &&
1302	aspm_policy != POLICY_POWER_SUPERSAVE)
1303	return;
1304
1305	down_read(sem: &pci_bus_sem);
1306	mutex_lock(&aspm_lock);
1307	pcie_config_aspm_path(link);
1308	pcie_set_clkpm(link, enable: policy_to_clkpm_state(link));
1309	mutex_unlock(lock: &aspm_lock);
1310	up_read(sem: &pci_bus_sem);
1311	}
1312
1313	static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev)
1314	{
1315	struct pci_dev *bridge;
1316
1317	if (!pci_is_pcie(dev: pdev))
1318	return NULL;
1319
1320	bridge = pci_upstream_bridge(dev: pdev);
1321	if (!bridge || !pci_is_pcie(dev: bridge))
1322	return NULL;
1323
1324	return bridge->link_state;
1325	}
1326
1327	static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool locked)
1328	{
1329	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1330
1331	if (!link)
1332	return -EINVAL;
1333	/*
1334	* A driver requested that ASPM be disabled on this device, but
1335	* if we don't have permission to manage ASPM (e.g., on ACPI
1336	* systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and
1337	* the _OSC method), we can't honor that request. Windows has
1338	* a similar mechanism using "PciASPMOptOut", which is also
1339	* ignored in this situation.
1340	*/
1341	if (aspm_disabled) {
1342	pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n");
1343	return -EPERM;
1344	}
1345
1346	if (!locked)
1347	down_read(sem: &pci_bus_sem);
1348	mutex_lock(&aspm_lock);
1349	if (state & PCIE_LINK_STATE_L0S)
1350	link->aspm_disable |= ASPM_STATE_L0S;
1351	if (state & PCIE_LINK_STATE_L1)
1352	/* L1 PM substates require L1 */
1353	link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS;
1354	if (state & PCIE_LINK_STATE_L1_1)
1355	link->aspm_disable |= ASPM_STATE_L1_1;
1356	if (state & PCIE_LINK_STATE_L1_2)
1357	link->aspm_disable |= ASPM_STATE_L1_2;
1358	if (state & PCIE_LINK_STATE_L1_1_PCIPM)
1359	link->aspm_disable |= ASPM_STATE_L1_1_PCIPM;
1360	if (state & PCIE_LINK_STATE_L1_2_PCIPM)
1361	link->aspm_disable |= ASPM_STATE_L1_2_PCIPM;
1362	pcie_config_aspm_link(link, state: policy_to_aspm_state(link));
1363
1364	if (state & PCIE_LINK_STATE_CLKPM)
1365	link->clkpm_disable = 1;
1366	pcie_set_clkpm(link, enable: policy_to_clkpm_state(link));
1367	mutex_unlock(lock: &aspm_lock);
1368	if (!locked)
1369	up_read(sem: &pci_bus_sem);
1370
1371	return 0;
1372	}
1373
1374	int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
1375	{
1376	lockdep_assert_held_read(&pci_bus_sem);
1377
1378	return __pci_disable_link_state(pdev, state, locked: true);
1379	}
1380	EXPORT_SYMBOL(pci_disable_link_state_locked);
1381
1382	/**
1383	* pci_disable_link_state - Disable device's link state, so the link will
1384	* never enter specific states. Note that if the BIOS didn't grant ASPM
1385	* control to the OS, this does nothing because we can't touch the LNKCTL
1386	* register. Returns 0 or a negative errno.
1387	*
1388	* @pdev: PCI device
1389	* @state: ASPM link state to disable
1390	*/
1391	int pci_disable_link_state(struct pci_dev *pdev, int state)
1392	{
1393	return __pci_disable_link_state(pdev, state, locked: false);
1394	}
1395	EXPORT_SYMBOL(pci_disable_link_state);
1396
1397	static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked)
1398	{
1399	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1400
1401	if (!link)
1402	return -EINVAL;
1403	/*
1404	* A driver requested that ASPM be enabled on this device, but
1405	* if we don't have permission to manage ASPM (e.g., on ACPI
1406	* systems we have to observe the FADT ACPI_FADT_NO_ASPM bit and
1407	* the _OSC method), we can't honor that request.
1408	*/
1409	if (aspm_disabled) {
1410	pci_warn(pdev, "can't override BIOS ASPM; OS doesn't have ASPM control\n");
1411	return -EPERM;
1412	}
1413
1414	if (!locked)
1415	down_read(sem: &pci_bus_sem);
1416	mutex_lock(&aspm_lock);
1417	link->aspm_default = 0;
1418	if (state & PCIE_LINK_STATE_L0S)
1419	link->aspm_default |= ASPM_STATE_L0S;
1420	if (state & PCIE_LINK_STATE_L1)
1421	link->aspm_default |= ASPM_STATE_L1;
1422	/* L1 PM substates require L1 */
1423	if (state & PCIE_LINK_STATE_L1_1)
1424	link->aspm_default |= ASPM_STATE_L1_1 | ASPM_STATE_L1;
1425	if (state & PCIE_LINK_STATE_L1_2)
1426	link->aspm_default |= ASPM_STATE_L1_2 | ASPM_STATE_L1;
1427	if (state & PCIE_LINK_STATE_L1_1_PCIPM)
1428	link->aspm_default |= ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1;
1429	if (state & PCIE_LINK_STATE_L1_2_PCIPM)
1430	link->aspm_default |= ASPM_STATE_L1_2_PCIPM | ASPM_STATE_L1;
1431	pcie_config_aspm_link(link, state: policy_to_aspm_state(link));
1432
1433	link->clkpm_default = (state & PCIE_LINK_STATE_CLKPM) ? 1 : 0;
1434	pcie_set_clkpm(link, enable: policy_to_clkpm_state(link));
1435	mutex_unlock(lock: &aspm_lock);
1436	if (!locked)
1437	up_read(sem: &pci_bus_sem);
1438
1439	return 0;
1440	}
1441
1442	/**
1443	* pci_enable_link_state - Clear and set the default device link state so that
1444	* the link may be allowed to enter the specified states. Note that if the
1445	* BIOS didn't grant ASPM control to the OS, this does nothing because we can't
1446	* touch the LNKCTL register. Also note that this does not enable states
1447	* disabled by pci_disable_link_state(). Return 0 or a negative errno.
1448	*
1449	* @pdev: PCI device
1450	* @state: Mask of ASPM link states to enable
1451	*/
1452	int pci_enable_link_state(struct pci_dev *pdev, int state)
1453	{
1454	return __pci_enable_link_state(pdev, state, locked: false);
1455	}
1456	EXPORT_SYMBOL(pci_enable_link_state);
1457
1458	/**
1459	* pci_enable_link_state_locked - Clear and set the default device link state
1460	* so that the link may be allowed to enter the specified states. Note that if
1461	* the BIOS didn't grant ASPM control to the OS, this does nothing because we
1462	* can't touch the LNKCTL register. Also note that this does not enable states
1463	* disabled by pci_disable_link_state(). Return 0 or a negative errno.
1464	*
1465	* @pdev: PCI device
1466	* @state: Mask of ASPM link states to enable
1467	*
1468	* Context: Caller holds pci_bus_sem read lock.
1469	*/
1470	int pci_enable_link_state_locked(struct pci_dev *pdev, int state)
1471	{
1472	lockdep_assert_held_read(&pci_bus_sem);
1473
1474	return __pci_enable_link_state(pdev, state, locked: true);
1475	}
1476	EXPORT_SYMBOL(pci_enable_link_state_locked);
1477
1478	static int pcie_aspm_set_policy(const char *val,
1479	const struct kernel_param *kp)
1480	{
1481	int i;
1482	struct pcie_link_state *link;
1483
1484	if (aspm_disabled)
1485	return -EPERM;
1486	i = sysfs_match_string(policy_str, val);
1487	if (i < 0)
1488	return i;
1489	if (i == aspm_policy)
1490	return 0;
1491
1492	down_read(sem: &pci_bus_sem);
1493	mutex_lock(&aspm_lock);
1494	aspm_policy = i;
1495	list_for_each_entry(link, &link_list, sibling) {
1496	pcie_config_aspm_link(link, state: policy_to_aspm_state(link));
1497	pcie_set_clkpm(link, enable: policy_to_clkpm_state(link));
1498	}
1499	mutex_unlock(lock: &aspm_lock);
1500	up_read(sem: &pci_bus_sem);
1501	return 0;
1502	}
1503
1504	static int pcie_aspm_get_policy(char *buffer, const struct kernel_param *kp)
1505	{
1506	int i, cnt = 0;
1507	for (i = 0; i < ARRAY_SIZE(policy_str); i++)
1508	if (i == aspm_policy)
1509	cnt += sprintf(buf: buffer + cnt, fmt: "[%s] ", policy_str[i]);
1510	else
1511	cnt += sprintf(buf: buffer + cnt, fmt: "%s ", policy_str[i]);
1512	cnt += sprintf(buf: buffer + cnt, fmt: "\n");
1513	return cnt;
1514	}
1515
1516	module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
1517	NULL, 0644);
1518
1519	/**
1520	* pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
1521	* @pdev: Target device.
1522	*
1523	* Relies on the upstream bridge's link_state being valid. The link_state
1524	* is deallocated only when the last child of the bridge (i.e., @pdev or a
1525	* sibling) is removed, and the caller should be holding a reference to
1526	* @pdev, so this should be safe.
1527	*/
1528	bool pcie_aspm_enabled(struct pci_dev *pdev)
1529	{
1530	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1531
1532	if (!link)
1533	return false;
1534
1535	return link->aspm_enabled;
1536	}
1537	EXPORT_SYMBOL_GPL(pcie_aspm_enabled);
1538
1539	static ssize_t aspm_attr_show_common(struct device *dev,
1540	struct device_attribute *attr,
1541	char *buf, u8 state)
1542	{
1543	struct pci_dev *pdev = to_pci_dev(dev);
1544	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1545
1546	return sysfs_emit(buf, fmt: "%d\n", (link->aspm_enabled & state) ? 1 : 0);
1547	}
1548
1549	static ssize_t aspm_attr_store_common(struct device *dev,
1550	struct device_attribute *attr,
1551	const char *buf, size_t len, u8 state)
1552	{
1553	struct pci_dev *pdev = to_pci_dev(dev);
1554	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1555	bool state_enable;
1556
1557	if (kstrtobool(s: buf, res: &state_enable) < 0)
1558	return -EINVAL;
1559
1560	down_read(sem: &pci_bus_sem);
1561	mutex_lock(&aspm_lock);
1562
1563	if (state_enable) {
1564	link->aspm_disable &= ~state;
1565	/* need to enable L1 for substates */
1566	if (state & ASPM_STATE_L1SS)
1567	link->aspm_disable &= ~ASPM_STATE_L1;
1568	} else {
1569	link->aspm_disable |= state;
1570	if (state & ASPM_STATE_L1)
1571	link->aspm_disable |= ASPM_STATE_L1SS;
1572	}
1573
1574	pcie_config_aspm_link(link, state: policy_to_aspm_state(link));
1575
1576	mutex_unlock(lock: &aspm_lock);
1577	up_read(sem: &pci_bus_sem);
1578
1579	return len;
1580	}
1581
1582	#define ASPM_ATTR(_f, _s) \
1583	static ssize_t _f##_show(struct device *dev, \
1584	struct device_attribute *attr, char *buf) \
1585	{ return aspm_attr_show_common(dev, attr, buf, ASPM_STATE_##_s); } \
1586	\
1587	static ssize_t _f##_store(struct device *dev, \
1588	struct device_attribute *attr, \
1589	const char *buf, size_t len) \
1590	{ return aspm_attr_store_common(dev, attr, buf, len, ASPM_STATE_##_s); }
1591
1592	ASPM_ATTR(l0s_aspm, L0S)
1593	ASPM_ATTR(l1_aspm, L1)
1594	ASPM_ATTR(l1_1_aspm, L1_1)
1595	ASPM_ATTR(l1_2_aspm, L1_2)
1596	ASPM_ATTR(l1_1_pcipm, L1_1_PCIPM)
1597	ASPM_ATTR(l1_2_pcipm, L1_2_PCIPM)
1598
1599	static ssize_t clkpm_show(struct device *dev,
1600	struct device_attribute *attr, char *buf)
1601	{
1602	struct pci_dev *pdev = to_pci_dev(dev);
1603	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1604
1605	return sysfs_emit(buf, fmt: "%d\n", link->clkpm_enabled);
1606	}
1607
1608	static ssize_t clkpm_store(struct device *dev,
1609	struct device_attribute *attr,
1610	const char *buf, size_t len)
1611	{
1612	struct pci_dev *pdev = to_pci_dev(dev);
1613	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1614	bool state_enable;
1615
1616	if (kstrtobool(s: buf, res: &state_enable) < 0)
1617	return -EINVAL;
1618
1619	down_read(sem: &pci_bus_sem);
1620	mutex_lock(&aspm_lock);
1621
1622	link->clkpm_disable = !state_enable;
1623	pcie_set_clkpm(link, enable: policy_to_clkpm_state(link));
1624
1625	mutex_unlock(lock: &aspm_lock);
1626	up_read(sem: &pci_bus_sem);
1627
1628	return len;
1629	}
1630
1631	static DEVICE_ATTR_RW(clkpm);
1632	static DEVICE_ATTR_RW(l0s_aspm);
1633	static DEVICE_ATTR_RW(l1_aspm);
1634	static DEVICE_ATTR_RW(l1_1_aspm);
1635	static DEVICE_ATTR_RW(l1_2_aspm);
1636	static DEVICE_ATTR_RW(l1_1_pcipm);
1637	static DEVICE_ATTR_RW(l1_2_pcipm);
1638
1639	static struct attribute *aspm_ctrl_attrs[] = {
1640	&dev_attr_clkpm.attr,
1641	&dev_attr_l0s_aspm.attr,
1642	&dev_attr_l1_aspm.attr,
1643	&dev_attr_l1_1_aspm.attr,
1644	&dev_attr_l1_2_aspm.attr,
1645	&dev_attr_l1_1_pcipm.attr,
1646	&dev_attr_l1_2_pcipm.attr,
1647	NULL
1648	};
1649
1650	static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj,
1651	struct attribute *a, int n)
1652	{
1653	struct device *dev = kobj_to_dev(kobj);
1654	struct pci_dev *pdev = to_pci_dev(dev);
1655	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1656	static const u8 aspm_state_map[] = {
1657	ASPM_STATE_L0S,
1658	ASPM_STATE_L1,
1659	ASPM_STATE_L1_1,
1660	ASPM_STATE_L1_2,
1661	ASPM_STATE_L1_1_PCIPM,
1662	ASPM_STATE_L1_2_PCIPM,
1663	};
1664
1665	if (aspm_disabled || !link)
1666	return 0;
1667
1668	if (n == 0)
1669	return link->clkpm_capable ? a->mode : 0;
1670
1671	return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;
1672	}
1673
1674	const struct attribute_group aspm_ctrl_attr_group = {
1675	.name = "link",
1676	.attrs = aspm_ctrl_attrs,
1677	.is_visible = aspm_ctrl_attrs_are_visible,
1678	};
1679
1680	static int __init pcie_aspm_disable(char *str)
1681	{
1682	if (!strcmp(str, "off")) {
1683	aspm_policy = POLICY_DEFAULT;
1684	aspm_disabled = 1;
1685	aspm_support_enabled = false;
1686	pr_info("PCIe ASPM is disabled\n");
1687	} else if (!strcmp(str, "force")) {
1688	aspm_force = 1;
1689	pr_info("PCIe ASPM is forcibly enabled\n");
1690	}
1691	return 1;
1692	}
1693
1694	__setup("pcie_aspm=", pcie_aspm_disable);
1695
1696	void pcie_no_aspm(void)
1697	{
1698	/*
1699	* Disabling ASPM is intended to prevent the kernel from modifying
1700	* existing hardware state, not to clear existing state. To that end:
1701	* (a) set policy to POLICY_DEFAULT in order to avoid changing state
1702	* (b) prevent userspace from changing policy
1703	*/
1704	if (!aspm_force) {
1705	aspm_policy = POLICY_DEFAULT;
1706	aspm_disabled = 1;
1707	}
1708	}
1709
1710	bool pcie_aspm_support_enabled(void)
1711	{
1712	return aspm_support_enabled;
1713	}
1714
1715	#endif /* CONFIG_PCIEASPM */
1716