1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* intel_idle.c - native hardware idle loop for modern Intel processors
4	*
5	* Copyright (c) 2013 - 2020, Intel Corporation.
6	* Len Brown <len.brown@intel.com>
7	* Rafael J. Wysocki <rafael.j.wysocki@intel.com>
8	*/
9
10	/*
11	* intel_idle is a cpuidle driver that loads on all Intel CPUs with MWAIT
12	* in lieu of the legacy ACPI processor_idle driver. The intent is to
13	* make Linux more efficient on these processors, as intel_idle knows
14	* more than ACPI, as well as make Linux more immune to ACPI BIOS bugs.
15	*/
16
17	/*
18	* Design Assumptions
19	*
20	* All CPUs have same idle states as boot CPU
21	*
22	* Chipset BM_STS (bus master status) bit is a NOP
23	* for preventing entry into deep C-states
24	*
25	* CPU will flush caches as needed when entering a C-state via MWAIT
26	* (in contrast to entering ACPI C3, in which case the WBINVD
27	* instruction needs to be executed to flush the caches)
28	*/
29
30	/*
31	* Known limitations
32	*
33	* ACPI has a .suspend hack to turn off deep c-statees during suspend
34	* to avoid complications with the lapic timer workaround.
35	* Have not seen issues with suspend, but may need same workaround here.
36	*
37	*/
38
39	/* un-comment DEBUG to enable pr_debug() statements */
40	/* #define DEBUG */
41
42	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
43
44	#include <linux/acpi.h>
45	#include <linux/kernel.h>
46	#include <linux/cpuidle.h>
47	#include <linux/tick.h>
48	#include <trace/events/power.h>
49	#include <linux/sched.h>
50	#include <linux/sched/smt.h>
51	#include <linux/notifier.h>
52	#include <linux/cpu.h>
53	#include <linux/moduleparam.h>
54	#include <asm/cpu_device_id.h>
55	#include <asm/intel-family.h>
56	#include <asm/mwait.h>
57	#include <asm/spec-ctrl.h>
58	#include <asm/fpu/api.h>
59
60	#define INTEL_IDLE_VERSION "0.5.1"
61
62	static struct cpuidle_driver intel_idle_driver = {
63	.name = "intel_idle",
64	.owner = THIS_MODULE,
65	};
66	/* intel_idle.max_cstate=0 disables driver */
67	static int max_cstate = CPUIDLE_STATE_MAX - 1;
68	static unsigned int disabled_states_mask __read_mostly;
69	static unsigned int preferred_states_mask __read_mostly;
70	static bool force_irq_on __read_mostly;
71	static bool ibrs_off __read_mostly;
72
73	static struct cpuidle_device __percpu *intel_idle_cpuidle_devices;
74
75	static unsigned long auto_demotion_disable_flags;
76
77	static enum {
78	C1E_PROMOTION_PRESERVE,
79	C1E_PROMOTION_ENABLE,
80	C1E_PROMOTION_DISABLE
81	} c1e_promotion = C1E_PROMOTION_PRESERVE;
82
83	struct idle_cpu {
84	struct cpuidle_state *state_table;
85
86	/*
87	* Hardware C-state auto-demotion may not always be optimal.
88	* Indicate which enable bits to clear here.
89	*/
90	unsigned long auto_demotion_disable_flags;
91	bool byt_auto_demotion_disable_flag;
92	bool disable_promotion_to_c1e;
93	bool use_acpi;
94	};
95
96	static const struct idle_cpu *icpu __initdata;
97	static struct cpuidle_state *cpuidle_state_table __initdata;
98
99	static unsigned int mwait_substates __initdata;
100
101	/*
102	* Enable interrupts before entering the C-state. On some platforms and for
103	* some C-states, this may measurably decrease interrupt latency.
104	*/
105	#define CPUIDLE_FLAG_IRQ_ENABLE BIT(14)
106
107	/*
108	* Enable this state by default even if the ACPI _CST does not list it.
109	*/
110	#define CPUIDLE_FLAG_ALWAYS_ENABLE BIT(15)
111
112	/*
113	* Disable IBRS across idle (when KERNEL_IBRS), is exclusive vs IRQ_ENABLE
114	* above.
115	*/
116	#define CPUIDLE_FLAG_IBRS BIT(16)
117
118	/*
119	* Initialize large xstate for the C6-state entrance.
120	*/
121	#define CPUIDLE_FLAG_INIT_XSTATE BIT(17)
122
123	/*
124	* MWAIT takes an 8-bit "hint" in EAX "suggesting"
125	* the C-state (top nibble) and sub-state (bottom nibble)
126	* 0x00 means "MWAIT(C1)", 0x10 means "MWAIT(C2)" etc.
127	*
128	* We store the hint at the top of our "flags" for each state.
129	*/
130	#define flg2MWAIT(flags) (((flags) >> 24) & 0xFF)
131	#define MWAIT2flg(eax) ((eax & 0xFF) << 24)
132
133	static __always_inline int __intel_idle(struct cpuidle_device *dev,
134	struct cpuidle_driver *drv,
135	int index, bool irqoff)
136	{
137	struct cpuidle_state *state = &drv->states[index];
138	unsigned long eax = flg2MWAIT(state->flags);
139	unsigned long ecx = 1*irqoff; /* break on interrupt flag */
140
141	mwait_idle_with_hints(eax, ecx);
142
143	return index;
144	}
145
146	/**
147	* intel_idle - Ask the processor to enter the given idle state.
148	* @dev: cpuidle device of the target CPU.
149	* @drv: cpuidle driver (assumed to point to intel_idle_driver).
150	* @index: Target idle state index.
151	*
152	* Use the MWAIT instruction to notify the processor that the CPU represented by
153	* @dev is idle and it can try to enter the idle state corresponding to @index.
154	*
155	* If the local APIC timer is not known to be reliable in the target idle state,
156	* enable one-shot tick broadcasting for the target CPU before executing MWAIT.
157	*
158	* Must be called under local_irq_disable().
159	*/
160	static __cpuidle int intel_idle(struct cpuidle_device *dev,
161	struct cpuidle_driver *drv, int index)
162	{
163	return __intel_idle(dev, drv, index, irqoff: true);
164	}
165
166	static __cpuidle int intel_idle_irq(struct cpuidle_device *dev,
167	struct cpuidle_driver *drv, int index)
168	{
169	return __intel_idle(dev, drv, index, irqoff: false);
170	}
171
172	static __cpuidle int intel_idle_ibrs(struct cpuidle_device *dev,
173	struct cpuidle_driver *drv, int index)
174	{
175	bool smt_active = sched_smt_active();
176	u64 spec_ctrl = spec_ctrl_current();
177	int ret;
178
179	if (smt_active)
180	__update_spec_ctrl(val: 0);
181
182	ret = __intel_idle(dev, drv, index, irqoff: true);
183
184	if (smt_active)
185	__update_spec_ctrl(val: spec_ctrl);
186
187	return ret;
188	}
189
190	static __cpuidle int intel_idle_xstate(struct cpuidle_device *dev,
191	struct cpuidle_driver *drv, int index)
192	{
193	fpu_idle_fpregs();
194	return __intel_idle(dev, drv, index, irqoff: true);
195	}
196
197	/**
198	* intel_idle_s2idle - Ask the processor to enter the given idle state.
199	* @dev: cpuidle device of the target CPU.
200	* @drv: cpuidle driver (assumed to point to intel_idle_driver).
201	* @index: Target idle state index.
202	*
203	* Use the MWAIT instruction to notify the processor that the CPU represented by
204	* @dev is idle and it can try to enter the idle state corresponding to @index.
205	*
206	* Invoked as a suspend-to-idle callback routine with frozen user space, frozen
207	* scheduler tick and suspended scheduler clock on the target CPU.
208	*/
209	static __cpuidle int intel_idle_s2idle(struct cpuidle_device *dev,
210	struct cpuidle_driver *drv, int index)
211	{
212	unsigned long ecx = 1; /* break on interrupt flag */
213	struct cpuidle_state *state = &drv->states[index];
214	unsigned long eax = flg2MWAIT(state->flags);
215
216	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)
217	fpu_idle_fpregs();
218
219	mwait_idle_with_hints(eax, ecx);
220
221	return 0;
222	}
223
224	/*
225	* States are indexed by the cstate number,
226	* which is also the index into the MWAIT hint array.
227	* Thus C0 is a dummy.
228	*/
229	static struct cpuidle_state nehalem_cstates[] __initdata = {
230	{
231	.name = "C1",
232	.desc = "MWAIT 0x00",
233	.flags = MWAIT2flg(0x00),
234	.exit_latency = 3,
235	.target_residency = 6,
236	.enter = &intel_idle,
237	.enter_s2idle = intel_idle_s2idle, },
238	{
239	.name = "C1E",
240	.desc = "MWAIT 0x01",
241	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
242	.exit_latency = 10,
243	.target_residency = 20,
244	.enter = &intel_idle,
245	.enter_s2idle = intel_idle_s2idle, },
246	{
247	.name = "C3",
248	.desc = "MWAIT 0x10",
249	.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
250	.exit_latency = 20,
251	.target_residency = 80,
252	.enter = &intel_idle,
253	.enter_s2idle = intel_idle_s2idle, },
254	{
255	.name = "C6",
256	.desc = "MWAIT 0x20",
257	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
258	.exit_latency = 200,
259	.target_residency = 800,
260	.enter = &intel_idle,
261	.enter_s2idle = intel_idle_s2idle, },
262	{
263	.enter = NULL }
264	};
265
266	static struct cpuidle_state snb_cstates[] __initdata = {
267	{
268	.name = "C1",
269	.desc = "MWAIT 0x00",
270	.flags = MWAIT2flg(0x00),
271	.exit_latency = 2,
272	.target_residency = 2,
273	.enter = &intel_idle,
274	.enter_s2idle = intel_idle_s2idle, },
275	{
276	.name = "C1E",
277	.desc = "MWAIT 0x01",
278	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
279	.exit_latency = 10,
280	.target_residency = 20,
281	.enter = &intel_idle,
282	.enter_s2idle = intel_idle_s2idle, },
283	{
284	.name = "C3",
285	.desc = "MWAIT 0x10",
286	.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
287	.exit_latency = 80,
288	.target_residency = 211,
289	.enter = &intel_idle,
290	.enter_s2idle = intel_idle_s2idle, },
291	{
292	.name = "C6",
293	.desc = "MWAIT 0x20",
294	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
295	.exit_latency = 104,
296	.target_residency = 345,
297	.enter = &intel_idle,
298	.enter_s2idle = intel_idle_s2idle, },
299	{
300	.name = "C7",
301	.desc = "MWAIT 0x30",
302	.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
303	.exit_latency = 109,
304	.target_residency = 345,
305	.enter = &intel_idle,
306	.enter_s2idle = intel_idle_s2idle, },
307	{
308	.enter = NULL }
309	};
310
311	static struct cpuidle_state byt_cstates[] __initdata = {
312	{
313	.name = "C1",
314	.desc = "MWAIT 0x00",
315	.flags = MWAIT2flg(0x00),
316	.exit_latency = 1,
317	.target_residency = 1,
318	.enter = &intel_idle,
319	.enter_s2idle = intel_idle_s2idle, },
320	{
321	.name = "C6N",
322	.desc = "MWAIT 0x58",
323	.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
324	.exit_latency = 300,
325	.target_residency = 275,
326	.enter = &intel_idle,
327	.enter_s2idle = intel_idle_s2idle, },
328	{
329	.name = "C6S",
330	.desc = "MWAIT 0x52",
331	.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
332	.exit_latency = 500,
333	.target_residency = 560,
334	.enter = &intel_idle,
335	.enter_s2idle = intel_idle_s2idle, },
336	{
337	.name = "C7",
338	.desc = "MWAIT 0x60",
339	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
340	.exit_latency = 1200,
341	.target_residency = 4000,
342	.enter = &intel_idle,
343	.enter_s2idle = intel_idle_s2idle, },
344	{
345	.name = "C7S",
346	.desc = "MWAIT 0x64",
347	.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
348	.exit_latency = 10000,
349	.target_residency = 20000,
350	.enter = &intel_idle,
351	.enter_s2idle = intel_idle_s2idle, },
352	{
353	.enter = NULL }
354	};
355
356	static struct cpuidle_state cht_cstates[] __initdata = {
357	{
358	.name = "C1",
359	.desc = "MWAIT 0x00",
360	.flags = MWAIT2flg(0x00),
361	.exit_latency = 1,
362	.target_residency = 1,
363	.enter = &intel_idle,
364	.enter_s2idle = intel_idle_s2idle, },
365	{
366	.name = "C6N",
367	.desc = "MWAIT 0x58",
368	.flags = MWAIT2flg(0x58) | CPUIDLE_FLAG_TLB_FLUSHED,
369	.exit_latency = 80,
370	.target_residency = 275,
371	.enter = &intel_idle,
372	.enter_s2idle = intel_idle_s2idle, },
373	{
374	.name = "C6S",
375	.desc = "MWAIT 0x52",
376	.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
377	.exit_latency = 200,
378	.target_residency = 560,
379	.enter = &intel_idle,
380	.enter_s2idle = intel_idle_s2idle, },
381	{
382	.name = "C7",
383	.desc = "MWAIT 0x60",
384	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
385	.exit_latency = 1200,
386	.target_residency = 4000,
387	.enter = &intel_idle,
388	.enter_s2idle = intel_idle_s2idle, },
389	{
390	.name = "C7S",
391	.desc = "MWAIT 0x64",
392	.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
393	.exit_latency = 10000,
394	.target_residency = 20000,
395	.enter = &intel_idle,
396	.enter_s2idle = intel_idle_s2idle, },
397	{
398	.enter = NULL }
399	};
400
401	static struct cpuidle_state ivb_cstates[] __initdata = {
402	{
403	.name = "C1",
404	.desc = "MWAIT 0x00",
405	.flags = MWAIT2flg(0x00),
406	.exit_latency = 1,
407	.target_residency = 1,
408	.enter = &intel_idle,
409	.enter_s2idle = intel_idle_s2idle, },
410	{
411	.name = "C1E",
412	.desc = "MWAIT 0x01",
413	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
414	.exit_latency = 10,
415	.target_residency = 20,
416	.enter = &intel_idle,
417	.enter_s2idle = intel_idle_s2idle, },
418	{
419	.name = "C3",
420	.desc = "MWAIT 0x10",
421	.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
422	.exit_latency = 59,
423	.target_residency = 156,
424	.enter = &intel_idle,
425	.enter_s2idle = intel_idle_s2idle, },
426	{
427	.name = "C6",
428	.desc = "MWAIT 0x20",
429	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
430	.exit_latency = 80,
431	.target_residency = 300,
432	.enter = &intel_idle,
433	.enter_s2idle = intel_idle_s2idle, },
434	{
435	.name = "C7",
436	.desc = "MWAIT 0x30",
437	.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
438	.exit_latency = 87,
439	.target_residency = 300,
440	.enter = &intel_idle,
441	.enter_s2idle = intel_idle_s2idle, },
442	{
443	.enter = NULL }
444	};
445
446	static struct cpuidle_state ivt_cstates[] __initdata = {
447	{
448	.name = "C1",
449	.desc = "MWAIT 0x00",
450	.flags = MWAIT2flg(0x00),
451	.exit_latency = 1,
452	.target_residency = 1,
453	.enter = &intel_idle,
454	.enter_s2idle = intel_idle_s2idle, },
455	{
456	.name = "C1E",
457	.desc = "MWAIT 0x01",
458	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
459	.exit_latency = 10,
460	.target_residency = 80,
461	.enter = &intel_idle,
462	.enter_s2idle = intel_idle_s2idle, },
463	{
464	.name = "C3",
465	.desc = "MWAIT 0x10",
466	.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
467	.exit_latency = 59,
468	.target_residency = 156,
469	.enter = &intel_idle,
470	.enter_s2idle = intel_idle_s2idle, },
471	{
472	.name = "C6",
473	.desc = "MWAIT 0x20",
474	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
475	.exit_latency = 82,
476	.target_residency = 300,
477	.enter = &intel_idle,
478	.enter_s2idle = intel_idle_s2idle, },
479	{
480	.enter = NULL }
481	};
482
483	static struct cpuidle_state ivt_cstates_4s[] __initdata = {
484	{
485	.name = "C1",
486	.desc = "MWAIT 0x00",
487	.flags = MWAIT2flg(0x00),
488	.exit_latency = 1,
489	.target_residency = 1,
490	.enter = &intel_idle,
491	.enter_s2idle = intel_idle_s2idle, },
492	{
493	.name = "C1E",
494	.desc = "MWAIT 0x01",
495	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
496	.exit_latency = 10,
497	.target_residency = 250,
498	.enter = &intel_idle,
499	.enter_s2idle = intel_idle_s2idle, },
500	{
501	.name = "C3",
502	.desc = "MWAIT 0x10",
503	.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
504	.exit_latency = 59,
505	.target_residency = 300,
506	.enter = &intel_idle,
507	.enter_s2idle = intel_idle_s2idle, },
508	{
509	.name = "C6",
510	.desc = "MWAIT 0x20",
511	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
512	.exit_latency = 84,
513	.target_residency = 400,
514	.enter = &intel_idle,
515	.enter_s2idle = intel_idle_s2idle, },
516	{
517	.enter = NULL }
518	};
519
520	static struct cpuidle_state ivt_cstates_8s[] __initdata = {
521	{
522	.name = "C1",
523	.desc = "MWAIT 0x00",
524	.flags = MWAIT2flg(0x00),
525	.exit_latency = 1,
526	.target_residency = 1,
527	.enter = &intel_idle,
528	.enter_s2idle = intel_idle_s2idle, },
529	{
530	.name = "C1E",
531	.desc = "MWAIT 0x01",
532	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
533	.exit_latency = 10,
534	.target_residency = 500,
535	.enter = &intel_idle,
536	.enter_s2idle = intel_idle_s2idle, },
537	{
538	.name = "C3",
539	.desc = "MWAIT 0x10",
540	.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
541	.exit_latency = 59,
542	.target_residency = 600,
543	.enter = &intel_idle,
544	.enter_s2idle = intel_idle_s2idle, },
545	{
546	.name = "C6",
547	.desc = "MWAIT 0x20",
548	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
549	.exit_latency = 88,
550	.target_residency = 700,
551	.enter = &intel_idle,
552	.enter_s2idle = intel_idle_s2idle, },
553	{
554	.enter = NULL }
555	};
556
557	static struct cpuidle_state hsw_cstates[] __initdata = {
558	{
559	.name = "C1",
560	.desc = "MWAIT 0x00",
561	.flags = MWAIT2flg(0x00),
562	.exit_latency = 2,
563	.target_residency = 2,
564	.enter = &intel_idle,
565	.enter_s2idle = intel_idle_s2idle, },
566	{
567	.name = "C1E",
568	.desc = "MWAIT 0x01",
569	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
570	.exit_latency = 10,
571	.target_residency = 20,
572	.enter = &intel_idle,
573	.enter_s2idle = intel_idle_s2idle, },
574	{
575	.name = "C3",
576	.desc = "MWAIT 0x10",
577	.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
578	.exit_latency = 33,
579	.target_residency = 100,
580	.enter = &intel_idle,
581	.enter_s2idle = intel_idle_s2idle, },
582	{
583	.name = "C6",
584	.desc = "MWAIT 0x20",
585	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
586	.exit_latency = 133,
587	.target_residency = 400,
588	.enter = &intel_idle,
589	.enter_s2idle = intel_idle_s2idle, },
590	{
591	.name = "C7s",
592	.desc = "MWAIT 0x32",
593	.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
594	.exit_latency = 166,
595	.target_residency = 500,
596	.enter = &intel_idle,
597	.enter_s2idle = intel_idle_s2idle, },
598	{
599	.name = "C8",
600	.desc = "MWAIT 0x40",
601	.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
602	.exit_latency = 300,
603	.target_residency = 900,
604	.enter = &intel_idle,
605	.enter_s2idle = intel_idle_s2idle, },
606	{
607	.name = "C9",
608	.desc = "MWAIT 0x50",
609	.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
610	.exit_latency = 600,
611	.target_residency = 1800,
612	.enter = &intel_idle,
613	.enter_s2idle = intel_idle_s2idle, },
614	{
615	.name = "C10",
616	.desc = "MWAIT 0x60",
617	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
618	.exit_latency = 2600,
619	.target_residency = 7700,
620	.enter = &intel_idle,
621	.enter_s2idle = intel_idle_s2idle, },
622	{
623	.enter = NULL }
624	};
625	static struct cpuidle_state bdw_cstates[] __initdata = {
626	{
627	.name = "C1",
628	.desc = "MWAIT 0x00",
629	.flags = MWAIT2flg(0x00),
630	.exit_latency = 2,
631	.target_residency = 2,
632	.enter = &intel_idle,
633	.enter_s2idle = intel_idle_s2idle, },
634	{
635	.name = "C1E",
636	.desc = "MWAIT 0x01",
637	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
638	.exit_latency = 10,
639	.target_residency = 20,
640	.enter = &intel_idle,
641	.enter_s2idle = intel_idle_s2idle, },
642	{
643	.name = "C3",
644	.desc = "MWAIT 0x10",
645	.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
646	.exit_latency = 40,
647	.target_residency = 100,
648	.enter = &intel_idle,
649	.enter_s2idle = intel_idle_s2idle, },
650	{
651	.name = "C6",
652	.desc = "MWAIT 0x20",
653	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
654	.exit_latency = 133,
655	.target_residency = 400,
656	.enter = &intel_idle,
657	.enter_s2idle = intel_idle_s2idle, },
658	{
659	.name = "C7s",
660	.desc = "MWAIT 0x32",
661	.flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
662	.exit_latency = 166,
663	.target_residency = 500,
664	.enter = &intel_idle,
665	.enter_s2idle = intel_idle_s2idle, },
666	{
667	.name = "C8",
668	.desc = "MWAIT 0x40",
669	.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
670	.exit_latency = 300,
671	.target_residency = 900,
672	.enter = &intel_idle,
673	.enter_s2idle = intel_idle_s2idle, },
674	{
675	.name = "C9",
676	.desc = "MWAIT 0x50",
677	.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
678	.exit_latency = 600,
679	.target_residency = 1800,
680	.enter = &intel_idle,
681	.enter_s2idle = intel_idle_s2idle, },
682	{
683	.name = "C10",
684	.desc = "MWAIT 0x60",
685	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
686	.exit_latency = 2600,
687	.target_residency = 7700,
688	.enter = &intel_idle,
689	.enter_s2idle = intel_idle_s2idle, },
690	{
691	.enter = NULL }
692	};
693
694	static struct cpuidle_state skl_cstates[] __initdata = {
695	{
696	.name = "C1",
697	.desc = "MWAIT 0x00",
698	.flags = MWAIT2flg(0x00),
699	.exit_latency = 2,
700	.target_residency = 2,
701	.enter = &intel_idle,
702	.enter_s2idle = intel_idle_s2idle, },
703	{
704	.name = "C1E",
705	.desc = "MWAIT 0x01",
706	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
707	.exit_latency = 10,
708	.target_residency = 20,
709	.enter = &intel_idle,
710	.enter_s2idle = intel_idle_s2idle, },
711	{
712	.name = "C3",
713	.desc = "MWAIT 0x10",
714	.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
715	.exit_latency = 70,
716	.target_residency = 100,
717	.enter = &intel_idle,
718	.enter_s2idle = intel_idle_s2idle, },
719	{
720	.name = "C6",
721	.desc = "MWAIT 0x20",
722	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
723	.exit_latency = 85,
724	.target_residency = 200,
725	.enter = &intel_idle,
726	.enter_s2idle = intel_idle_s2idle, },
727	{
728	.name = "C7s",
729	.desc = "MWAIT 0x33",
730	.flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
731	.exit_latency = 124,
732	.target_residency = 800,
733	.enter = &intel_idle,
734	.enter_s2idle = intel_idle_s2idle, },
735	{
736	.name = "C8",
737	.desc = "MWAIT 0x40",
738	.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
739	.exit_latency = 200,
740	.target_residency = 800,
741	.enter = &intel_idle,
742	.enter_s2idle = intel_idle_s2idle, },
743	{
744	.name = "C9",
745	.desc = "MWAIT 0x50",
746	.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
747	.exit_latency = 480,
748	.target_residency = 5000,
749	.enter = &intel_idle,
750	.enter_s2idle = intel_idle_s2idle, },
751	{
752	.name = "C10",
753	.desc = "MWAIT 0x60",
754	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
755	.exit_latency = 890,
756	.target_residency = 5000,
757	.enter = &intel_idle,
758	.enter_s2idle = intel_idle_s2idle, },
759	{
760	.enter = NULL }
761	};
762
763	static struct cpuidle_state skx_cstates[] __initdata = {
764	{
765	.name = "C1",
766	.desc = "MWAIT 0x00",
767	.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
768	.exit_latency = 2,
769	.target_residency = 2,
770	.enter = &intel_idle,
771	.enter_s2idle = intel_idle_s2idle, },
772	{
773	.name = "C1E",
774	.desc = "MWAIT 0x01",
775	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
776	.exit_latency = 10,
777	.target_residency = 20,
778	.enter = &intel_idle,
779	.enter_s2idle = intel_idle_s2idle, },
780	{
781	.name = "C6",
782	.desc = "MWAIT 0x20",
783	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED | CPUIDLE_FLAG_IBRS,
784	.exit_latency = 133,
785	.target_residency = 600,
786	.enter = &intel_idle,
787	.enter_s2idle = intel_idle_s2idle, },
788	{
789	.enter = NULL }
790	};
791
792	static struct cpuidle_state icx_cstates[] __initdata = {
793	{
794	.name = "C1",
795	.desc = "MWAIT 0x00",
796	.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_IRQ_ENABLE,
797	.exit_latency = 1,
798	.target_residency = 1,
799	.enter = &intel_idle,
800	.enter_s2idle = intel_idle_s2idle, },
801	{
802	.name = "C1E",
803	.desc = "MWAIT 0x01",
804	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
805	.exit_latency = 4,
806	.target_residency = 4,
807	.enter = &intel_idle,
808	.enter_s2idle = intel_idle_s2idle, },
809	{
810	.name = "C6",
811	.desc = "MWAIT 0x20",
812	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
813	.exit_latency = 170,
814	.target_residency = 600,
815	.enter = &intel_idle,
816	.enter_s2idle = intel_idle_s2idle, },
817	{
818	.enter = NULL }
819	};
820
821	/*
822	* On AlderLake C1 has to be disabled if C1E is enabled, and vice versa.
823	* C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
824	* But in this case there is effectively no C1, because C1 requests are
825	* promoted to C1E. If the "C1E promotion" bit is cleared, then both C1
826	* and C1E requests end up with C1, so there is effectively no C1E.
827	*
828	* By default we enable C1E and disable C1 by marking it with
829	* 'CPUIDLE_FLAG_UNUSABLE'.
830	*/
831	static struct cpuidle_state adl_cstates[] __initdata = {
832	{
833	.name = "C1",
834	.desc = "MWAIT 0x00",
835	.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
836	.exit_latency = 1,
837	.target_residency = 1,
838	.enter = &intel_idle,
839	.enter_s2idle = intel_idle_s2idle, },
840	{
841	.name = "C1E",
842	.desc = "MWAIT 0x01",
843	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
844	.exit_latency = 2,
845	.target_residency = 4,
846	.enter = &intel_idle,
847	.enter_s2idle = intel_idle_s2idle, },
848	{
849	.name = "C6",
850	.desc = "MWAIT 0x20",
851	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
852	.exit_latency = 220,
853	.target_residency = 600,
854	.enter = &intel_idle,
855	.enter_s2idle = intel_idle_s2idle, },
856	{
857	.name = "C8",
858	.desc = "MWAIT 0x40",
859	.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
860	.exit_latency = 280,
861	.target_residency = 800,
862	.enter = &intel_idle,
863	.enter_s2idle = intel_idle_s2idle, },
864	{
865	.name = "C10",
866	.desc = "MWAIT 0x60",
867	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
868	.exit_latency = 680,
869	.target_residency = 2000,
870	.enter = &intel_idle,
871	.enter_s2idle = intel_idle_s2idle, },
872	{
873	.enter = NULL }
874	};
875
876	static struct cpuidle_state adl_l_cstates[] __initdata = {
877	{
878	.name = "C1",
879	.desc = "MWAIT 0x00",
880	.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
881	.exit_latency = 1,
882	.target_residency = 1,
883	.enter = &intel_idle,
884	.enter_s2idle = intel_idle_s2idle, },
885	{
886	.name = "C1E",
887	.desc = "MWAIT 0x01",
888	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
889	.exit_latency = 2,
890	.target_residency = 4,
891	.enter = &intel_idle,
892	.enter_s2idle = intel_idle_s2idle, },
893	{
894	.name = "C6",
895	.desc = "MWAIT 0x20",
896	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
897	.exit_latency = 170,
898	.target_residency = 500,
899	.enter = &intel_idle,
900	.enter_s2idle = intel_idle_s2idle, },
901	{
902	.name = "C8",
903	.desc = "MWAIT 0x40",
904	.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
905	.exit_latency = 200,
906	.target_residency = 600,
907	.enter = &intel_idle,
908	.enter_s2idle = intel_idle_s2idle, },
909	{
910	.name = "C10",
911	.desc = "MWAIT 0x60",
912	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
913	.exit_latency = 230,
914	.target_residency = 700,
915	.enter = &intel_idle,
916	.enter_s2idle = intel_idle_s2idle, },
917	{
918	.enter = NULL }
919	};
920
921	static struct cpuidle_state mtl_l_cstates[] __initdata = {
922	{
923	.name = "C1E",
924	.desc = "MWAIT 0x01",
925	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
926	.exit_latency = 1,
927	.target_residency = 1,
928	.enter = &intel_idle,
929	.enter_s2idle = intel_idle_s2idle, },
930	{
931	.name = "C6",
932	.desc = "MWAIT 0x20",
933	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
934	.exit_latency = 140,
935	.target_residency = 420,
936	.enter = &intel_idle,
937	.enter_s2idle = intel_idle_s2idle, },
938	{
939	.name = "C10",
940	.desc = "MWAIT 0x60",
941	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
942	.exit_latency = 310,
943	.target_residency = 930,
944	.enter = &intel_idle,
945	.enter_s2idle = intel_idle_s2idle, },
946	{
947	.enter = NULL }
948	};
949
950	static struct cpuidle_state gmt_cstates[] __initdata = {
951	{
952	.name = "C1",
953	.desc = "MWAIT 0x00",
954	.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_UNUSABLE,
955	.exit_latency = 1,
956	.target_residency = 1,
957	.enter = &intel_idle,
958	.enter_s2idle = intel_idle_s2idle, },
959	{
960	.name = "C1E",
961	.desc = "MWAIT 0x01",
962	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
963	.exit_latency = 2,
964	.target_residency = 4,
965	.enter = &intel_idle,
966	.enter_s2idle = intel_idle_s2idle, },
967	{
968	.name = "C6",
969	.desc = "MWAIT 0x20",
970	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
971	.exit_latency = 195,
972	.target_residency = 585,
973	.enter = &intel_idle,
974	.enter_s2idle = intel_idle_s2idle, },
975	{
976	.name = "C8",
977	.desc = "MWAIT 0x40",
978	.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
979	.exit_latency = 260,
980	.target_residency = 1040,
981	.enter = &intel_idle,
982	.enter_s2idle = intel_idle_s2idle, },
983	{
984	.name = "C10",
985	.desc = "MWAIT 0x60",
986	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
987	.exit_latency = 660,
988	.target_residency = 1980,
989	.enter = &intel_idle,
990	.enter_s2idle = intel_idle_s2idle, },
991	{
992	.enter = NULL }
993	};
994
995	static struct cpuidle_state spr_cstates[] __initdata = {
996	{
997	.name = "C1",
998	.desc = "MWAIT 0x00",
999	.flags = MWAIT2flg(0x00),
1000	.exit_latency = 1,
1001	.target_residency = 1,
1002	.enter = &intel_idle,
1003	.enter_s2idle = intel_idle_s2idle, },
1004	{
1005	.name = "C1E",
1006	.desc = "MWAIT 0x01",
1007	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1008	.exit_latency = 2,
1009	.target_residency = 4,
1010	.enter = &intel_idle,
1011	.enter_s2idle = intel_idle_s2idle, },
1012	{
1013	.name = "C6",
1014	.desc = "MWAIT 0x20",
1015	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED |
1016	CPUIDLE_FLAG_INIT_XSTATE,
1017	.exit_latency = 290,
1018	.target_residency = 800,
1019	.enter = &intel_idle,
1020	.enter_s2idle = intel_idle_s2idle, },
1021	{
1022	.enter = NULL }
1023	};
1024
1025	static struct cpuidle_state atom_cstates[] __initdata = {
1026	{
1027	.name = "C1E",
1028	.desc = "MWAIT 0x00",
1029	.flags = MWAIT2flg(0x00),
1030	.exit_latency = 10,
1031	.target_residency = 20,
1032	.enter = &intel_idle,
1033	.enter_s2idle = intel_idle_s2idle, },
1034	{
1035	.name = "C2",
1036	.desc = "MWAIT 0x10",
1037	.flags = MWAIT2flg(0x10),
1038	.exit_latency = 20,
1039	.target_residency = 80,
1040	.enter = &intel_idle,
1041	.enter_s2idle = intel_idle_s2idle, },
1042	{
1043	.name = "C4",
1044	.desc = "MWAIT 0x30",
1045	.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1046	.exit_latency = 100,
1047	.target_residency = 400,
1048	.enter = &intel_idle,
1049	.enter_s2idle = intel_idle_s2idle, },
1050	{
1051	.name = "C6",
1052	.desc = "MWAIT 0x52",
1053	.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1054	.exit_latency = 140,
1055	.target_residency = 560,
1056	.enter = &intel_idle,
1057	.enter_s2idle = intel_idle_s2idle, },
1058	{
1059	.enter = NULL }
1060	};
1061	static struct cpuidle_state tangier_cstates[] __initdata = {
1062	{
1063	.name = "C1",
1064	.desc = "MWAIT 0x00",
1065	.flags = MWAIT2flg(0x00),
1066	.exit_latency = 1,
1067	.target_residency = 4,
1068	.enter = &intel_idle,
1069	.enter_s2idle = intel_idle_s2idle, },
1070	{
1071	.name = "C4",
1072	.desc = "MWAIT 0x30",
1073	.flags = MWAIT2flg(0x30) | CPUIDLE_FLAG_TLB_FLUSHED,
1074	.exit_latency = 100,
1075	.target_residency = 400,
1076	.enter = &intel_idle,
1077	.enter_s2idle = intel_idle_s2idle, },
1078	{
1079	.name = "C6",
1080	.desc = "MWAIT 0x52",
1081	.flags = MWAIT2flg(0x52) | CPUIDLE_FLAG_TLB_FLUSHED,
1082	.exit_latency = 140,
1083	.target_residency = 560,
1084	.enter = &intel_idle,
1085	.enter_s2idle = intel_idle_s2idle, },
1086	{
1087	.name = "C7",
1088	.desc = "MWAIT 0x60",
1089	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1090	.exit_latency = 1200,
1091	.target_residency = 4000,
1092	.enter = &intel_idle,
1093	.enter_s2idle = intel_idle_s2idle, },
1094	{
1095	.name = "C9",
1096	.desc = "MWAIT 0x64",
1097	.flags = MWAIT2flg(0x64) | CPUIDLE_FLAG_TLB_FLUSHED,
1098	.exit_latency = 10000,
1099	.target_residency = 20000,
1100	.enter = &intel_idle,
1101	.enter_s2idle = intel_idle_s2idle, },
1102	{
1103	.enter = NULL }
1104	};
1105	static struct cpuidle_state avn_cstates[] __initdata = {
1106	{
1107	.name = "C1",
1108	.desc = "MWAIT 0x00",
1109	.flags = MWAIT2flg(0x00),
1110	.exit_latency = 2,
1111	.target_residency = 2,
1112	.enter = &intel_idle,
1113	.enter_s2idle = intel_idle_s2idle, },
1114	{
1115	.name = "C6",
1116	.desc = "MWAIT 0x51",
1117	.flags = MWAIT2flg(0x51) | CPUIDLE_FLAG_TLB_FLUSHED,
1118	.exit_latency = 15,
1119	.target_residency = 45,
1120	.enter = &intel_idle,
1121	.enter_s2idle = intel_idle_s2idle, },
1122	{
1123	.enter = NULL }
1124	};
1125	static struct cpuidle_state knl_cstates[] __initdata = {
1126	{
1127	.name = "C1",
1128	.desc = "MWAIT 0x00",
1129	.flags = MWAIT2flg(0x00),
1130	.exit_latency = 1,
1131	.target_residency = 2,
1132	.enter = &intel_idle,
1133	.enter_s2idle = intel_idle_s2idle },
1134	{
1135	.name = "C6",
1136	.desc = "MWAIT 0x10",
1137	.flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
1138	.exit_latency = 120,
1139	.target_residency = 500,
1140	.enter = &intel_idle,
1141	.enter_s2idle = intel_idle_s2idle },
1142	{
1143	.enter = NULL }
1144	};
1145
1146	static struct cpuidle_state bxt_cstates[] __initdata = {
1147	{
1148	.name = "C1",
1149	.desc = "MWAIT 0x00",
1150	.flags = MWAIT2flg(0x00),
1151	.exit_latency = 2,
1152	.target_residency = 2,
1153	.enter = &intel_idle,
1154	.enter_s2idle = intel_idle_s2idle, },
1155	{
1156	.name = "C1E",
1157	.desc = "MWAIT 0x01",
1158	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1159	.exit_latency = 10,
1160	.target_residency = 20,
1161	.enter = &intel_idle,
1162	.enter_s2idle = intel_idle_s2idle, },
1163	{
1164	.name = "C6",
1165	.desc = "MWAIT 0x20",
1166	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1167	.exit_latency = 133,
1168	.target_residency = 133,
1169	.enter = &intel_idle,
1170	.enter_s2idle = intel_idle_s2idle, },
1171	{
1172	.name = "C7s",
1173	.desc = "MWAIT 0x31",
1174	.flags = MWAIT2flg(0x31) | CPUIDLE_FLAG_TLB_FLUSHED,
1175	.exit_latency = 155,
1176	.target_residency = 155,
1177	.enter = &intel_idle,
1178	.enter_s2idle = intel_idle_s2idle, },
1179	{
1180	.name = "C8",
1181	.desc = "MWAIT 0x40",
1182	.flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
1183	.exit_latency = 1000,
1184	.target_residency = 1000,
1185	.enter = &intel_idle,
1186	.enter_s2idle = intel_idle_s2idle, },
1187	{
1188	.name = "C9",
1189	.desc = "MWAIT 0x50",
1190	.flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
1191	.exit_latency = 2000,
1192	.target_residency = 2000,
1193	.enter = &intel_idle,
1194	.enter_s2idle = intel_idle_s2idle, },
1195	{
1196	.name = "C10",
1197	.desc = "MWAIT 0x60",
1198	.flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
1199	.exit_latency = 10000,
1200	.target_residency = 10000,
1201	.enter = &intel_idle,
1202	.enter_s2idle = intel_idle_s2idle, },
1203	{
1204	.enter = NULL }
1205	};
1206
1207	static struct cpuidle_state dnv_cstates[] __initdata = {
1208	{
1209	.name = "C1",
1210	.desc = "MWAIT 0x00",
1211	.flags = MWAIT2flg(0x00),
1212	.exit_latency = 2,
1213	.target_residency = 2,
1214	.enter = &intel_idle,
1215	.enter_s2idle = intel_idle_s2idle, },
1216	{
1217	.name = "C1E",
1218	.desc = "MWAIT 0x01",
1219	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1220	.exit_latency = 10,
1221	.target_residency = 20,
1222	.enter = &intel_idle,
1223	.enter_s2idle = intel_idle_s2idle, },
1224	{
1225	.name = "C6",
1226	.desc = "MWAIT 0x20",
1227	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1228	.exit_latency = 50,
1229	.target_residency = 500,
1230	.enter = &intel_idle,
1231	.enter_s2idle = intel_idle_s2idle, },
1232	{
1233	.enter = NULL }
1234	};
1235
1236	/*
1237	* Note, depending on HW and FW revision, SnowRidge SoC may or may not support
1238	* C6, and this is indicated in the CPUID mwait leaf.
1239	*/
1240	static struct cpuidle_state snr_cstates[] __initdata = {
1241	{
1242	.name = "C1",
1243	.desc = "MWAIT 0x00",
1244	.flags = MWAIT2flg(0x00),
1245	.exit_latency = 2,
1246	.target_residency = 2,
1247	.enter = &intel_idle,
1248	.enter_s2idle = intel_idle_s2idle, },
1249	{
1250	.name = "C1E",
1251	.desc = "MWAIT 0x01",
1252	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1253	.exit_latency = 15,
1254	.target_residency = 25,
1255	.enter = &intel_idle,
1256	.enter_s2idle = intel_idle_s2idle, },
1257	{
1258	.name = "C6",
1259	.desc = "MWAIT 0x20",
1260	.flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
1261	.exit_latency = 130,
1262	.target_residency = 500,
1263	.enter = &intel_idle,
1264	.enter_s2idle = intel_idle_s2idle, },
1265	{
1266	.enter = NULL }
1267	};
1268
1269	static struct cpuidle_state grr_cstates[] __initdata = {
1270	{
1271	.name = "C1",
1272	.desc = "MWAIT 0x00",
1273	.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1274	.exit_latency = 1,
1275	.target_residency = 1,
1276	.enter = &intel_idle,
1277	.enter_s2idle = intel_idle_s2idle, },
1278	{
1279	.name = "C1E",
1280	.desc = "MWAIT 0x01",
1281	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1282	.exit_latency = 2,
1283	.target_residency = 10,
1284	.enter = &intel_idle,
1285	.enter_s2idle = intel_idle_s2idle, },
1286	{
1287	.name = "C6S",
1288	.desc = "MWAIT 0x22",
1289	.flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1290	.exit_latency = 140,
1291	.target_residency = 500,
1292	.enter = &intel_idle,
1293	.enter_s2idle = intel_idle_s2idle, },
1294	{
1295	.enter = NULL }
1296	};
1297
1298	static struct cpuidle_state srf_cstates[] __initdata = {
1299	{
1300	.name = "C1",
1301	.desc = "MWAIT 0x00",
1302	.flags = MWAIT2flg(0x00) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1303	.exit_latency = 1,
1304	.target_residency = 1,
1305	.enter = &intel_idle,
1306	.enter_s2idle = intel_idle_s2idle, },
1307	{
1308	.name = "C1E",
1309	.desc = "MWAIT 0x01",
1310	.flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
1311	.exit_latency = 2,
1312	.target_residency = 10,
1313	.enter = &intel_idle,
1314	.enter_s2idle = intel_idle_s2idle, },
1315	{
1316	.name = "C6S",
1317	.desc = "MWAIT 0x22",
1318	.flags = MWAIT2flg(0x22) | CPUIDLE_FLAG_TLB_FLUSHED,
1319	.exit_latency = 270,
1320	.target_residency = 700,
1321	.enter = &intel_idle,
1322	.enter_s2idle = intel_idle_s2idle, },
1323	{
1324	.name = "C6SP",
1325	.desc = "MWAIT 0x23",
1326	.flags = MWAIT2flg(0x23) | CPUIDLE_FLAG_TLB_FLUSHED,
1327	.exit_latency = 310,
1328	.target_residency = 900,
1329	.enter = &intel_idle,
1330	.enter_s2idle = intel_idle_s2idle, },
1331	{
1332	.enter = NULL }
1333	};
1334
1335	static const struct idle_cpu idle_cpu_nehalem __initconst = {
1336	.state_table = nehalem_cstates,
1337	.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1338	.disable_promotion_to_c1e = true,
1339	};
1340
1341	static const struct idle_cpu idle_cpu_nhx __initconst = {
1342	.state_table = nehalem_cstates,
1343	.auto_demotion_disable_flags = NHM_C1_AUTO_DEMOTE | NHM_C3_AUTO_DEMOTE,
1344	.disable_promotion_to_c1e = true,
1345	.use_acpi = true,
1346	};
1347
1348	static const struct idle_cpu idle_cpu_atom __initconst = {
1349	.state_table = atom_cstates,
1350	};
1351
1352	static const struct idle_cpu idle_cpu_tangier __initconst = {
1353	.state_table = tangier_cstates,
1354	};
1355
1356	static const struct idle_cpu idle_cpu_lincroft __initconst = {
1357	.state_table = atom_cstates,
1358	.auto_demotion_disable_flags = ATM_LNC_C6_AUTO_DEMOTE,
1359	};
1360
1361	static const struct idle_cpu idle_cpu_snb __initconst = {
1362	.state_table = snb_cstates,
1363	.disable_promotion_to_c1e = true,
1364	};
1365
1366	static const struct idle_cpu idle_cpu_snx __initconst = {
1367	.state_table = snb_cstates,
1368	.disable_promotion_to_c1e = true,
1369	.use_acpi = true,
1370	};
1371
1372	static const struct idle_cpu idle_cpu_byt __initconst = {
1373	.state_table = byt_cstates,
1374	.disable_promotion_to_c1e = true,
1375	.byt_auto_demotion_disable_flag = true,
1376	};
1377
1378	static const struct idle_cpu idle_cpu_cht __initconst = {
1379	.state_table = cht_cstates,
1380	.disable_promotion_to_c1e = true,
1381	.byt_auto_demotion_disable_flag = true,
1382	};
1383
1384	static const struct idle_cpu idle_cpu_ivb __initconst = {
1385	.state_table = ivb_cstates,
1386	.disable_promotion_to_c1e = true,
1387	};
1388
1389	static const struct idle_cpu idle_cpu_ivt __initconst = {
1390	.state_table = ivt_cstates,
1391	.disable_promotion_to_c1e = true,
1392	.use_acpi = true,
1393	};
1394
1395	static const struct idle_cpu idle_cpu_hsw __initconst = {
1396	.state_table = hsw_cstates,
1397	.disable_promotion_to_c1e = true,
1398	};
1399
1400	static const struct idle_cpu idle_cpu_hsx __initconst = {
1401	.state_table = hsw_cstates,
1402	.disable_promotion_to_c1e = true,
1403	.use_acpi = true,
1404	};
1405
1406	static const struct idle_cpu idle_cpu_bdw __initconst = {
1407	.state_table = bdw_cstates,
1408	.disable_promotion_to_c1e = true,
1409	};
1410
1411	static const struct idle_cpu idle_cpu_bdx __initconst = {
1412	.state_table = bdw_cstates,
1413	.disable_promotion_to_c1e = true,
1414	.use_acpi = true,
1415	};
1416
1417	static const struct idle_cpu idle_cpu_skl __initconst = {
1418	.state_table = skl_cstates,
1419	.disable_promotion_to_c1e = true,
1420	};
1421
1422	static const struct idle_cpu idle_cpu_skx __initconst = {
1423	.state_table = skx_cstates,
1424	.disable_promotion_to_c1e = true,
1425	.use_acpi = true,
1426	};
1427
1428	static const struct idle_cpu idle_cpu_icx __initconst = {
1429	.state_table = icx_cstates,
1430	.disable_promotion_to_c1e = true,
1431	.use_acpi = true,
1432	};
1433
1434	static const struct idle_cpu idle_cpu_adl __initconst = {
1435	.state_table = adl_cstates,
1436	};
1437
1438	static const struct idle_cpu idle_cpu_adl_l __initconst = {
1439	.state_table = adl_l_cstates,
1440	};
1441
1442	static const struct idle_cpu idle_cpu_mtl_l __initconst = {
1443	.state_table = mtl_l_cstates,
1444	};
1445
1446	static const struct idle_cpu idle_cpu_gmt __initconst = {
1447	.state_table = gmt_cstates,
1448	};
1449
1450	static const struct idle_cpu idle_cpu_spr __initconst = {
1451	.state_table = spr_cstates,
1452	.disable_promotion_to_c1e = true,
1453	.use_acpi = true,
1454	};
1455
1456	static const struct idle_cpu idle_cpu_avn __initconst = {
1457	.state_table = avn_cstates,
1458	.disable_promotion_to_c1e = true,
1459	.use_acpi = true,
1460	};
1461
1462	static const struct idle_cpu idle_cpu_knl __initconst = {
1463	.state_table = knl_cstates,
1464	.use_acpi = true,
1465	};
1466
1467	static const struct idle_cpu idle_cpu_bxt __initconst = {
1468	.state_table = bxt_cstates,
1469	.disable_promotion_to_c1e = true,
1470	};
1471
1472	static const struct idle_cpu idle_cpu_dnv __initconst = {
1473	.state_table = dnv_cstates,
1474	.disable_promotion_to_c1e = true,
1475	.use_acpi = true,
1476	};
1477
1478	static const struct idle_cpu idle_cpu_snr __initconst = {
1479	.state_table = snr_cstates,
1480	.disable_promotion_to_c1e = true,
1481	.use_acpi = true,
1482	};
1483
1484	static const struct idle_cpu idle_cpu_grr __initconst = {
1485	.state_table = grr_cstates,
1486	.disable_promotion_to_c1e = true,
1487	.use_acpi = true,
1488	};
1489
1490	static const struct idle_cpu idle_cpu_srf __initconst = {
1491	.state_table = srf_cstates,
1492	.disable_promotion_to_c1e = true,
1493	.use_acpi = true,
1494	};
1495
1496	static const struct x86_cpu_id intel_idle_ids[] __initconst = {
1497	X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EP, &idle_cpu_nhx),
1498	X86_MATCH_INTEL_FAM6_MODEL(NEHALEM, &idle_cpu_nehalem),
1499	X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_G, &idle_cpu_nehalem),
1500	X86_MATCH_INTEL_FAM6_MODEL(WESTMERE, &idle_cpu_nehalem),
1501	X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EP, &idle_cpu_nhx),
1502	X86_MATCH_INTEL_FAM6_MODEL(NEHALEM_EX, &idle_cpu_nhx),
1503	X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL, &idle_cpu_atom),
1504	X86_MATCH_INTEL_FAM6_MODEL(ATOM_BONNELL_MID, &idle_cpu_lincroft),
1505	X86_MATCH_INTEL_FAM6_MODEL(WESTMERE_EX, &idle_cpu_nhx),
1506	X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE, &idle_cpu_snb),
1507	X86_MATCH_INTEL_FAM6_MODEL(SANDYBRIDGE_X, &idle_cpu_snx),
1508	X86_MATCH_INTEL_FAM6_MODEL(ATOM_SALTWELL, &idle_cpu_atom),
1509	X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT, &idle_cpu_byt),
1510	X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_MID, &idle_cpu_tangier),
1511	X86_MATCH_INTEL_FAM6_MODEL(ATOM_AIRMONT, &idle_cpu_cht),
1512	X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE, &idle_cpu_ivb),
1513	X86_MATCH_INTEL_FAM6_MODEL(IVYBRIDGE_X, &idle_cpu_ivt),
1514	X86_MATCH_INTEL_FAM6_MODEL(HASWELL, &idle_cpu_hsw),
1515	X86_MATCH_INTEL_FAM6_MODEL(HASWELL_X, &idle_cpu_hsx),
1516	X86_MATCH_INTEL_FAM6_MODEL(HASWELL_L, &idle_cpu_hsw),
1517	X86_MATCH_INTEL_FAM6_MODEL(HASWELL_G, &idle_cpu_hsw),
1518	X86_MATCH_INTEL_FAM6_MODEL(ATOM_SILVERMONT_D, &idle_cpu_avn),
1519	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL, &idle_cpu_bdw),
1520	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_G, &idle_cpu_bdw),
1521	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_X, &idle_cpu_bdx),
1522	X86_MATCH_INTEL_FAM6_MODEL(BROADWELL_D, &idle_cpu_bdx),
1523	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_L, &idle_cpu_skl),
1524	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE, &idle_cpu_skl),
1525	X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE_L, &idle_cpu_skl),
1526	X86_MATCH_INTEL_FAM6_MODEL(KABYLAKE, &idle_cpu_skl),
1527	X86_MATCH_INTEL_FAM6_MODEL(SKYLAKE_X, &idle_cpu_skx),
1528	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, &idle_cpu_icx),
1529	X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, &idle_cpu_icx),
1530	X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE, &idle_cpu_adl),
1531	X86_MATCH_INTEL_FAM6_MODEL(ALDERLAKE_L, &idle_cpu_adl_l),
1532	X86_MATCH_INTEL_FAM6_MODEL(METEORLAKE_L, &idle_cpu_mtl_l),
1533	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GRACEMONT, &idle_cpu_gmt),
1534	X86_MATCH_INTEL_FAM6_MODEL(SAPPHIRERAPIDS_X, &idle_cpu_spr),
1535	X86_MATCH_INTEL_FAM6_MODEL(EMERALDRAPIDS_X, &idle_cpu_spr),
1536	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNL, &idle_cpu_knl),
1537	X86_MATCH_INTEL_FAM6_MODEL(XEON_PHI_KNM, &idle_cpu_knl),
1538	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &idle_cpu_bxt),
1539	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_PLUS, &idle_cpu_bxt),
1540	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &idle_cpu_dnv),
1541	X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, &idle_cpu_snr),
1542	X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT, &idle_cpu_grr),
1543	X86_MATCH_INTEL_FAM6_MODEL(ATOM_CRESTMONT_X, &idle_cpu_srf),
1544	{}
1545	};
1546
1547	static const struct x86_cpu_id intel_mwait_ids[] __initconst = {
1548	X86_MATCH_VENDOR_FAM_FEATURE(INTEL, 6, X86_FEATURE_MWAIT, NULL),
1549	{}
1550	};
1551
1552	static bool __init intel_idle_max_cstate_reached(int cstate)
1553	{
1554	if (cstate + 1 > max_cstate) {
1555	pr_info("max_cstate %d reached\n", max_cstate);
1556	return true;
1557	}
1558	return false;
1559	}
1560
1561	static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)
1562	{
1563	unsigned long eax = flg2MWAIT(state->flags);
1564
1565	if (boot_cpu_has(X86_FEATURE_ARAT))
1566	return false;
1567
1568	/*
1569	* Switch over to one-shot tick broadcast if the target C-state
1570	* is deeper than C1.
1571	*/
1572	return !!((eax >> MWAIT_SUBSTATE_SIZE) & MWAIT_CSTATE_MASK);
1573	}
1574
1575	#ifdef CONFIG_ACPI_PROCESSOR_CSTATE
1576	#include <acpi/processor.h>
1577
1578	static bool no_acpi __read_mostly;
1579	module_param(no_acpi, bool, 0444);
1580	MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1581
1582	static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1583	module_param_named(use_acpi, force_use_acpi, bool, 0444);
1584	MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1585
1586	static struct acpi_processor_power acpi_state_table __initdata;
1587
1588	/**
1589	* intel_idle_cst_usable - Check if the _CST information can be used.
1590	*
1591	* Check if all of the C-states listed by _CST in the max_cstate range are
1592	* ACPI_CSTATE_FFH, which means that they should be entered via MWAIT.
1593	*/
1594	static bool __init intel_idle_cst_usable(void)
1595	{
1596	int cstate, limit;
1597
1598	limit = min_t(int, min_t(int, CPUIDLE_STATE_MAX, max_cstate + 1),
1599	acpi_state_table.count);
1600
1601	for (cstate = 1; cstate < limit; cstate++) {
1602	struct acpi_processor_cx *cx = &acpi_state_table.states[cstate];
1603
1604	if (cx->entry_method != ACPI_CSTATE_FFH)
1605	return false;
1606	}
1607
1608	return true;
1609	}
1610
1611	static bool __init intel_idle_acpi_cst_extract(void)
1612	{
1613	unsigned int cpu;
1614
1615	if (no_acpi) {
1616	pr_debug("Not allowed to use ACPI _CST\n");
1617	return false;
1618	}
1619
1620	for_each_possible_cpu(cpu) {
1621	struct acpi_processor *pr = per_cpu(processors, cpu);
1622
1623	if (!pr)
1624	continue;
1625
1626	if (acpi_processor_evaluate_cst(handle: pr->handle, cpu, info: &acpi_state_table))
1627	continue;
1628
1629	acpi_state_table.count++;
1630
1631	if (!intel_idle_cst_usable())
1632	continue;
1633
1634	if (!acpi_processor_claim_cst_control())
1635	break;
1636
1637	return true;
1638	}
1639
1640	acpi_state_table.count = 0;
1641	pr_debug("ACPI _CST not found or not usable\n");
1642	return false;
1643	}
1644
1645	static void __init intel_idle_init_cstates_acpi(struct cpuidle_driver *drv)
1646	{
1647	int cstate, limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1648
1649	/*
1650	* If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1651	* the interesting states are ACPI_CSTATE_FFH.
1652	*/
1653	for (cstate = 1; cstate < limit; cstate++) {
1654	struct acpi_processor_cx *cx;
1655	struct cpuidle_state *state;
1656
1657	if (intel_idle_max_cstate_reached(cstate: cstate - 1))
1658	break;
1659
1660	cx = &acpi_state_table.states[cstate];
1661
1662	state = &drv->states[drv->state_count++];
1663
1664	snprintf(buf: state->name, CPUIDLE_NAME_LEN, fmt: "C%d_ACPI", cstate);
1665	strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);
1666	state->exit_latency = cx->latency;
1667	/*
1668	* For C1-type C-states use the same number for both the exit
1669	* latency and target residency, because that is the case for
1670	* C1 in the majority of the static C-states tables above.
1671	* For the other types of C-states, however, set the target
1672	* residency to 3 times the exit latency which should lead to
1673	* a reasonable balance between energy-efficiency and
1674	* performance in the majority of interesting cases.
1675	*/
1676	state->target_residency = cx->latency;
1677	if (cx->type > ACPI_STATE_C1)
1678	state->target_residency *= 3;
1679
1680	state->flags = MWAIT2flg(cx->address);
1681	if (cx->type > ACPI_STATE_C2)
1682	state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;
1683
1684	if (disabled_states_mask & BIT(cstate))
1685	state->flags |= CPUIDLE_FLAG_OFF;
1686
1687	if (intel_idle_state_needs_timer_stop(state))
1688	state->flags |= CPUIDLE_FLAG_TIMER_STOP;
1689
1690	state->enter = intel_idle;
1691	state->enter_s2idle = intel_idle_s2idle;
1692	}
1693	}
1694
1695	static bool __init intel_idle_off_by_default(u32 mwait_hint)
1696	{
1697	int cstate, limit;
1698
1699	/*
1700	* If there are no _CST C-states, do not disable any C-states by
1701	* default.
1702	*/
1703	if (!acpi_state_table.count)
1704	return false;
1705
1706	limit = min_t(int, CPUIDLE_STATE_MAX, acpi_state_table.count);
1707	/*
1708	* If limit > 0, intel_idle_cst_usable() has returned 'true', so all of
1709	* the interesting states are ACPI_CSTATE_FFH.
1710	*/
1711	for (cstate = 1; cstate < limit; cstate++) {
1712	if (acpi_state_table.states[cstate].address == mwait_hint)
1713	return false;
1714	}
1715	return true;
1716	}
1717	#else /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1718	#define force_use_acpi (false)
1719
1720	static inline bool intel_idle_acpi_cst_extract(void) { return false; }
1721	static inline void intel_idle_init_cstates_acpi(struct cpuidle_driver *drv) { }
1722	static inline bool intel_idle_off_by_default(u32 mwait_hint) { return false; }
1723	#endif /* !CONFIG_ACPI_PROCESSOR_CSTATE */
1724
1725	/**
1726	* ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1727	*
1728	* Tune IVT multi-socket targets.
1729	* Assumption: num_sockets == (max_package_num + 1).
1730	*/
1731	static void __init ivt_idle_state_table_update(void)
1732	{
1733	/* IVT uses a different table for 1-2, 3-4, and > 4 sockets */
1734	int cpu, package_num, num_sockets = 1;
1735
1736	for_each_online_cpu(cpu) {
1737	package_num = topology_physical_package_id(cpu);
1738	if (package_num + 1 > num_sockets) {
1739	num_sockets = package_num + 1;
1740
1741	if (num_sockets > 4) {
1742	cpuidle_state_table = ivt_cstates_8s;
1743	return;
1744	}
1745	}
1746	}
1747
1748	if (num_sockets > 2)
1749	cpuidle_state_table = ivt_cstates_4s;
1750
1751	/* else, 1 and 2 socket systems use default ivt_cstates */
1752	}
1753
1754	/**
1755	* irtl_2_usec - IRTL to microseconds conversion.
1756	* @irtl: IRTL MSR value.
1757	*
1758	* Translate the IRTL (Interrupt Response Time Limit) MSR value to microseconds.
1759	*/
1760	static unsigned long long __init irtl_2_usec(unsigned long long irtl)
1761	{
1762	static const unsigned int irtl_ns_units[] __initconst = {
1763	1, 32, 1024, 32768, 1048576, 33554432, 0, 0
1764	};
1765	unsigned long long ns;
1766
1767	if (!irtl)
1768	return 0;
1769
1770	ns = irtl_ns_units[(irtl >> 10) & 0x7];
1771
1772	return div_u64(dividend: (irtl & 0x3FF) * ns, NSEC_PER_USEC);
1773	}
1774
1775	/**
1776	* bxt_idle_state_table_update - Fix up the Broxton idle states table.
1777	*
1778	* On BXT, trust the IRTL (Interrupt Response Time Limit) MSR to show the
1779	* definitive maximum latency and use the same value for target_residency.
1780	*/
1781	static void __init bxt_idle_state_table_update(void)
1782	{
1783	unsigned long long msr;
1784	unsigned int usec;
1785
1786	rdmsrl(MSR_PKGC6_IRTL, msr);
1787	usec = irtl_2_usec(irtl: msr);
1788	if (usec) {
1789	bxt_cstates[2].exit_latency = usec;
1790	bxt_cstates[2].target_residency = usec;
1791	}
1792
1793	rdmsrl(MSR_PKGC7_IRTL, msr);
1794	usec = irtl_2_usec(irtl: msr);
1795	if (usec) {
1796	bxt_cstates[3].exit_latency = usec;
1797	bxt_cstates[3].target_residency = usec;
1798	}
1799
1800	rdmsrl(MSR_PKGC8_IRTL, msr);
1801	usec = irtl_2_usec(irtl: msr);
1802	if (usec) {
1803	bxt_cstates[4].exit_latency = usec;
1804	bxt_cstates[4].target_residency = usec;
1805	}
1806
1807	rdmsrl(MSR_PKGC9_IRTL, msr);
1808	usec = irtl_2_usec(irtl: msr);
1809	if (usec) {
1810	bxt_cstates[5].exit_latency = usec;
1811	bxt_cstates[5].target_residency = usec;
1812	}
1813
1814	rdmsrl(MSR_PKGC10_IRTL, msr);
1815	usec = irtl_2_usec(irtl: msr);
1816	if (usec) {
1817	bxt_cstates[6].exit_latency = usec;
1818	bxt_cstates[6].target_residency = usec;
1819	}
1820
1821	}
1822
1823	/**
1824	* sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1825	*
1826	* On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1827	*/
1828	static void __init sklh_idle_state_table_update(void)
1829	{
1830	unsigned long long msr;
1831	unsigned int eax, ebx, ecx, edx;
1832
1833
1834	/* if PC10 disabled via cmdline intel_idle.max_cstate=7 or shallower */
1835	if (max_cstate <= 7)
1836	return;
1837
1838	/* if PC10 not present in CPUID.MWAIT.EDX */
1839	if ((mwait_substates & (0xF << 28)) == 0)
1840	return;
1841
1842	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1843
1844	/* PC10 is not enabled in PKG C-state limit */
1845	if ((msr & 0xF) != 8)
1846	return;
1847
1848	ecx = 0;
1849	cpuid(op: 7, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &edx);
1850
1851	/* if SGX is present */
1852	if (ebx & (1 << 2)) {
1853
1854	rdmsrl(MSR_IA32_FEAT_CTL, msr);
1855
1856	/* if SGX is enabled */
1857	if (msr & (1 << 18))
1858	return;
1859	}
1860
1861	skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE; /* C8-SKL */
1862	skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE; /* C9-SKL */
1863	}
1864
1865	/**
1866	* skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1867	* idle states table.
1868	*/
1869	static void __init skx_idle_state_table_update(void)
1870	{
1871	unsigned long long msr;
1872
1873	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1874
1875	/*
1876	* 000b: C0/C1 (no package C-state support)
1877	* 001b: C2
1878	* 010b: C6 (non-retention)
1879	* 011b: C6 (retention)
1880	* 111b: No Package C state limits.
1881	*/
1882	if ((msr & 0x7) < 2) {
1883	/*
1884	* Uses the CC6 + PC0 latency and 3 times of
1885	* latency for target_residency if the PC6
1886	* is disabled in BIOS. This is consistent
1887	* with how intel_idle driver uses _CST
1888	* to set the target_residency.
1889	*/
1890	skx_cstates[2].exit_latency = 92;
1891	skx_cstates[2].target_residency = 276;
1892	}
1893	}
1894
1895	/**
1896	* adl_idle_state_table_update - Adjust AlderLake idle states table.
1897	*/
1898	static void __init adl_idle_state_table_update(void)
1899	{
1900	/* Check if user prefers C1 over C1E. */
1901	if (preferred_states_mask & BIT(1) && !(preferred_states_mask & BIT(2))) {
1902	cpuidle_state_table[0].flags &= ~CPUIDLE_FLAG_UNUSABLE;
1903	cpuidle_state_table[1].flags |= CPUIDLE_FLAG_UNUSABLE;
1904
1905	/* Disable C1E by clearing the "C1E promotion" bit. */
1906	c1e_promotion = C1E_PROMOTION_DISABLE;
1907	return;
1908	}
1909
1910	/* Make sure C1E is enabled by default */
1911	c1e_promotion = C1E_PROMOTION_ENABLE;
1912	}
1913
1914	/**
1915	* spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
1916	*/
1917	static void __init spr_idle_state_table_update(void)
1918	{
1919	unsigned long long msr;
1920
1921	/*
1922	* By default, the C6 state assumes the worst-case scenario of package
1923	* C6. However, if PC6 is disabled, we update the numbers to match
1924	* core C6.
1925	*/
1926	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr);
1927
1928	/* Limit value 2 and above allow for PC6. */
1929	if ((msr & 0x7) < 2) {
1930	spr_cstates[2].exit_latency = 190;
1931	spr_cstates[2].target_residency = 600;
1932	}
1933	}
1934
1935	static bool __init intel_idle_verify_cstate(unsigned int mwait_hint)
1936	{
1937	unsigned int mwait_cstate = (MWAIT_HINT2CSTATE(mwait_hint) + 1) &
1938	MWAIT_CSTATE_MASK;
1939	unsigned int num_substates = (mwait_substates >> mwait_cstate * 4) &
1940	MWAIT_SUBSTATE_MASK;
1941
1942	/* Ignore the C-state if there are NO sub-states in CPUID for it. */
1943	if (num_substates == 0)
1944	return false;
1945
1946	if (mwait_cstate > 2 && !boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
1947	mark_tsc_unstable(reason: "TSC halts in idle states deeper than C2");
1948
1949	return true;
1950	}
1951
1952	static void state_update_enter_method(struct cpuidle_state *state, int cstate)
1953	{
1954	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE) {
1955	/*
1956	* Combining with XSTATE with IBRS or IRQ_ENABLE flags
1957	* is not currently supported but this driver.
1958	*/
1959	WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IBRS);
1960	WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
1961	state->enter = intel_idle_xstate;
1962	return;
1963	}
1964
1965	if (cpu_feature_enabled(X86_FEATURE_KERNEL_IBRS) &&
1966	((state->flags & CPUIDLE_FLAG_IBRS) || ibrs_off)) {
1967	/*
1968	* IBRS mitigation requires that C-states are entered
1969	* with interrupts disabled.
1970	*/
1971	if (ibrs_off && (state->flags & CPUIDLE_FLAG_IRQ_ENABLE))
1972	state->flags &= ~CPUIDLE_FLAG_IRQ_ENABLE;
1973	WARN_ON_ONCE(state->flags & CPUIDLE_FLAG_IRQ_ENABLE);
1974	state->enter = intel_idle_ibrs;
1975	return;
1976	}
1977
1978	if (state->flags & CPUIDLE_FLAG_IRQ_ENABLE) {
1979	state->enter = intel_idle_irq;
1980	return;
1981	}
1982
1983	if (force_irq_on) {
1984	pr_info("forced intel_idle_irq for state %d\n", cstate);
1985	state->enter = intel_idle_irq;
1986	}
1987	}
1988
1989	static void __init intel_idle_init_cstates_icpu(struct cpuidle_driver *drv)
1990	{
1991	int cstate;
1992
1993	switch (boot_cpu_data.x86_model) {
1994	case INTEL_FAM6_IVYBRIDGE_X:
1995	ivt_idle_state_table_update();
1996	break;
1997	case INTEL_FAM6_ATOM_GOLDMONT:
1998	case INTEL_FAM6_ATOM_GOLDMONT_PLUS:
1999	bxt_idle_state_table_update();
2000	break;
2001	case INTEL_FAM6_SKYLAKE:
2002	sklh_idle_state_table_update();
2003	break;
2004	case INTEL_FAM6_SKYLAKE_X:
2005	skx_idle_state_table_update();
2006	break;
2007	case INTEL_FAM6_SAPPHIRERAPIDS_X:
2008	case INTEL_FAM6_EMERALDRAPIDS_X:
2009	spr_idle_state_table_update();
2010	break;
2011	case INTEL_FAM6_ALDERLAKE:
2012	case INTEL_FAM6_ALDERLAKE_L:
2013	case INTEL_FAM6_ATOM_GRACEMONT:
2014	adl_idle_state_table_update();
2015	break;
2016	}
2017
2018	for (cstate = 0; cstate < CPUIDLE_STATE_MAX; ++cstate) {
2019	struct cpuidle_state *state;
2020	unsigned int mwait_hint;
2021
2022	if (intel_idle_max_cstate_reached(cstate))
2023	break;
2024
2025	if (!cpuidle_state_table[cstate].enter &&
2026	!cpuidle_state_table[cstate].enter_s2idle)
2027	break;
2028
2029	/* If marked as unusable, skip this state. */
2030	if (cpuidle_state_table[cstate].flags & CPUIDLE_FLAG_UNUSABLE) {
2031	pr_debug("state %s is disabled\n",
2032	cpuidle_state_table[cstate].name);
2033	continue;
2034	}
2035
2036	mwait_hint = flg2MWAIT(cpuidle_state_table[cstate].flags);
2037	if (!intel_idle_verify_cstate(mwait_hint))
2038	continue;
2039
2040	/* Structure copy. */
2041	drv->states[drv->state_count] = cpuidle_state_table[cstate];
2042	state = &drv->states[drv->state_count];
2043
2044	state_update_enter_method(state, cstate);
2045
2046
2047	if ((disabled_states_mask & BIT(drv->state_count)) ||
2048	((icpu->use_acpi || force_use_acpi) &&
2049	intel_idle_off_by_default(mwait_hint) &&
2050	!(state->flags & CPUIDLE_FLAG_ALWAYS_ENABLE)))
2051	state->flags |= CPUIDLE_FLAG_OFF;
2052
2053	if (intel_idle_state_needs_timer_stop(state))
2054	state->flags |= CPUIDLE_FLAG_TIMER_STOP;
2055
2056	drv->state_count++;
2057	}
2058
2059	if (icpu->byt_auto_demotion_disable_flag) {
2060	wrmsrl(MSR_CC6_DEMOTION_POLICY_CONFIG, val: 0);
2061	wrmsrl(MSR_MC6_DEMOTION_POLICY_CONFIG, val: 0);
2062	}
2063	}
2064
2065	/**
2066	* intel_idle_cpuidle_driver_init - Create the list of available idle states.
2067	* @drv: cpuidle driver structure to initialize.
2068	*/
2069	static void __init intel_idle_cpuidle_driver_init(struct cpuidle_driver *drv)
2070	{
2071	cpuidle_poll_state_init(drv);
2072
2073	if (disabled_states_mask & BIT(0))
2074	drv->states[0].flags |= CPUIDLE_FLAG_OFF;
2075
2076	drv->state_count = 1;
2077
2078	if (icpu)
2079	intel_idle_init_cstates_icpu(drv);
2080	else
2081	intel_idle_init_cstates_acpi(drv);
2082	}
2083
2084	static void auto_demotion_disable(void)
2085	{
2086	unsigned long long msr_bits;
2087
2088	rdmsrl(MSR_PKG_CST_CONFIG_CONTROL, msr_bits);
2089	msr_bits &= ~auto_demotion_disable_flags;
2090	wrmsrl(MSR_PKG_CST_CONFIG_CONTROL, val: msr_bits);
2091	}
2092
2093	static void c1e_promotion_enable(void)
2094	{
2095	unsigned long long msr_bits;
2096
2097	rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2098	msr_bits |= 0x2;
2099	wrmsrl(MSR_IA32_POWER_CTL, val: msr_bits);
2100	}
2101
2102	static void c1e_promotion_disable(void)
2103	{
2104	unsigned long long msr_bits;
2105
2106	rdmsrl(MSR_IA32_POWER_CTL, msr_bits);
2107	msr_bits &= ~0x2;
2108	wrmsrl(MSR_IA32_POWER_CTL, val: msr_bits);
2109	}
2110
2111	/**
2112	* intel_idle_cpu_init - Register the target CPU with the cpuidle core.
2113	* @cpu: CPU to initialize.
2114	*
2115	* Register a cpuidle device object for @cpu and update its MSRs in accordance
2116	* with the processor model flags.
2117	*/
2118	static int intel_idle_cpu_init(unsigned int cpu)
2119	{
2120	struct cpuidle_device *dev;
2121
2122	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2123	dev->cpu = cpu;
2124
2125	if (cpuidle_register_device(dev)) {
2126	pr_debug("cpuidle_register_device %d failed!\n", cpu);
2127	return -EIO;
2128	}
2129
2130	if (auto_demotion_disable_flags)
2131	auto_demotion_disable();
2132
2133	if (c1e_promotion == C1E_PROMOTION_ENABLE)
2134	c1e_promotion_enable();
2135	else if (c1e_promotion == C1E_PROMOTION_DISABLE)
2136	c1e_promotion_disable();
2137
2138	return 0;
2139	}
2140
2141	static int intel_idle_cpu_online(unsigned int cpu)
2142	{
2143	struct cpuidle_device *dev;
2144
2145	if (!boot_cpu_has(X86_FEATURE_ARAT))
2146	tick_broadcast_enable();
2147
2148	/*
2149	* Some systems can hotplug a cpu at runtime after
2150	* the kernel has booted, we have to initialize the
2151	* driver in this case
2152	*/
2153	dev = per_cpu_ptr(intel_idle_cpuidle_devices, cpu);
2154	if (!dev->registered)
2155	return intel_idle_cpu_init(cpu);
2156
2157	return 0;
2158	}
2159
2160	/**
2161	* intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2162	*/
2163	static void __init intel_idle_cpuidle_devices_uninit(void)
2164	{
2165	int i;
2166
2167	for_each_online_cpu(i)
2168	cpuidle_unregister_device(per_cpu_ptr(intel_idle_cpuidle_devices, i));
2169	}
2170
2171	static int __init intel_idle_init(void)
2172	{
2173	const struct x86_cpu_id *id;
2174	unsigned int eax, ebx, ecx;
2175	int retval;
2176
2177	/* Do not load intel_idle at all for now if idle= is passed */
2178	if (boot_option_idle_override != IDLE_NO_OVERRIDE)
2179	return -ENODEV;
2180
2181	if (max_cstate == 0) {
2182	pr_debug("disabled\n");
2183	return -EPERM;
2184	}
2185
2186	id = x86_match_cpu(match: intel_idle_ids);
2187	if (id) {
2188	if (!boot_cpu_has(X86_FEATURE_MWAIT)) {
2189	pr_debug("Please enable MWAIT in BIOS SETUP\n");
2190	return -ENODEV;
2191	}
2192	} else {
2193	id = x86_match_cpu(match: intel_mwait_ids);
2194	if (!id)
2195	return -ENODEV;
2196	}
2197
2198	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
2199	return -ENODEV;
2200
2201	cpuid(CPUID_MWAIT_LEAF, eax: &eax, ebx: &ebx, ecx: &ecx, edx: &mwait_substates);
2202
2203	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
2204	!(ecx & CPUID5_ECX_INTERRUPT_BREAK) ||
2205	!mwait_substates)
2206	return -ENODEV;
2207
2208	pr_debug("MWAIT substates: 0x%x\n", mwait_substates);
2209
2210	icpu = (const struct idle_cpu *)id->driver_data;
2211	if (icpu) {
2212	cpuidle_state_table = icpu->state_table;
2213	auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;
2214	if (icpu->disable_promotion_to_c1e)
2215	c1e_promotion = C1E_PROMOTION_DISABLE;
2216	if (icpu->use_acpi || force_use_acpi)
2217	intel_idle_acpi_cst_extract();
2218	} else if (!intel_idle_acpi_cst_extract()) {
2219	return -ENODEV;
2220	}
2221
2222	pr_debug("v" INTEL_IDLE_VERSION " model 0x%X\n",
2223	boot_cpu_data.x86_model);
2224
2225	intel_idle_cpuidle_devices = alloc_percpu(struct cpuidle_device);
2226	if (!intel_idle_cpuidle_devices)
2227	return -ENOMEM;
2228
2229	intel_idle_cpuidle_driver_init(drv: &intel_idle_driver);
2230
2231	retval = cpuidle_register_driver(drv: &intel_idle_driver);
2232	if (retval) {
2233	struct cpuidle_driver *drv = cpuidle_get_driver();
2234	printk(KERN_DEBUG pr_fmt("intel_idle yielding to %s\n"),
2235	drv ? drv->name : "none");
2236	goto init_driver_fail;
2237	}
2238
2239	retval = cpuhp_setup_state(state: CPUHP_AP_ONLINE_DYN, name: "idle/intel:online",
2240	startup: intel_idle_cpu_online, NULL);
2241	if (retval < 0)
2242	goto hp_setup_fail;
2243
2244	pr_debug("Local APIC timer is reliable in %s\n",
2245	boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");
2246
2247	return 0;
2248
2249	hp_setup_fail:
2250	intel_idle_cpuidle_devices_uninit();
2251	cpuidle_unregister_driver(drv: &intel_idle_driver);
2252	init_driver_fail:
2253	free_percpu(pdata: intel_idle_cpuidle_devices);
2254	return retval;
2255
2256	}
2257	device_initcall(intel_idle_init);
2258
2259	/*
2260	* We are not really modular, but we used to support that. Meaning we also
2261	* support "intel_idle.max_cstate=..." at boot and also a read-only export of
2262	* it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2263	* is the easiest way (currently) to continue doing that.
2264	*/
2265	module_param(max_cstate, int, 0444);
2266	/*
2267	* The positions of the bits that are set in this number are the indices of the
2268	* idle states to be disabled by default (as reflected by the names of the
2269	* corresponding idle state directories in sysfs, "state0", "state1" ...
2270	* "state<i>" ..., where <i> is the index of the given state).
2271	*/
2272	module_param_named(states_off, disabled_states_mask, uint, 0444);
2273	MODULE_PARM_DESC(states_off, "Mask of disabled idle states");
2274	/*
2275	* Some platforms come with mutually exclusive C-states, so that if one is
2276	* enabled, the other C-states must not be used. Example: C1 and C1E on
2277	* Sapphire Rapids platform. This parameter allows for selecting the
2278	* preferred C-states among the groups of mutually exclusive C-states - the
2279	* selected C-states will be registered, the other C-states from the mutually
2280	* exclusive group won't be registered. If the platform has no mutually
2281	* exclusive C-states, this parameter has no effect.
2282	*/
2283	module_param_named(preferred_cstates, preferred_states_mask, uint, 0444);
2284	MODULE_PARM_DESC(preferred_cstates, "Mask of preferred idle states");
2285	/*
2286	* Debugging option that forces the driver to enter all C-states with
2287	* interrupts enabled. Does not apply to C-states with
2288	* 'CPUIDLE_FLAG_INIT_XSTATE' and 'CPUIDLE_FLAG_IBRS' flags.
2289	*/
2290	module_param(force_irq_on, bool, 0444);
2291	/*
2292	* Force the disabling of IBRS when X86_FEATURE_KERNEL_IBRS is on and
2293	* CPUIDLE_FLAG_IRQ_ENABLE isn't set.
2294	*/
2295	module_param(ibrs_off, bool, 0444);
2296	MODULE_PARM_DESC(ibrs_off, "Disable IBRS when idle");
2297