1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* kernel/sched/core.c
4	*
5	* Core kernel CPU scheduler code
6	*
7	* Copyright (C) 1991-2002 Linus Torvalds
8	* Copyright (C) 1998-2024 Ingo Molnar, Red Hat
9	*/
10	#define INSTANTIATE_EXPORTED_MIGRATE_DISABLE
11	#include <linux/sched.h>
12	#include <linux/highmem.h>
13	#include <linux/hrtimer_api.h>
14	#include <linux/ktime_api.h>
15	#include <linux/sched/signal.h>
16	#include <linux/syscalls_api.h>
17	#include <linux/debug_locks.h>
18	#include <linux/prefetch.h>
19	#include <linux/capability.h>
20	#include <linux/pgtable_api.h>
21	#include <linux/wait_bit.h>
22	#include <linux/jiffies.h>
23	#include <linux/spinlock_api.h>
24	#include <linux/cpumask_api.h>
25	#include <linux/lockdep_api.h>
26	#include <linux/hardirq.h>
27	#include <linux/softirq.h>
28	#include <linux/refcount_api.h>
29	#include <linux/topology.h>
30	#include <linux/sched/clock.h>
31	#include <linux/sched/cond_resched.h>
32	#include <linux/sched/cputime.h>
33	#include <linux/sched/debug.h>
34	#include <linux/sched/hotplug.h>
35	#include <linux/sched/init.h>
36	#include <linux/sched/isolation.h>
37	#include <linux/sched/loadavg.h>
38	#include <linux/sched/mm.h>
39	#include <linux/sched/nohz.h>
40	#include <linux/sched/rseq_api.h>
41	#include <linux/sched/rt.h>
42
43	#include <linux/blkdev.h>
44	#include <linux/context_tracking.h>
45	#include <linux/cpuset.h>
46	#include <linux/delayacct.h>
47	#include <linux/init_task.h>
48	#include <linux/interrupt.h>
49	#include <linux/ioprio.h>
50	#include <linux/kallsyms.h>
51	#include <linux/kcov.h>
52	#include <linux/kprobes.h>
53	#include <linux/llist_api.h>
54	#include <linux/mmu_context.h>
55	#include <linux/mmzone.h>
56	#include <linux/mutex_api.h>
57	#include <linux/nmi.h>
58	#include <linux/nospec.h>
59	#include <linux/perf_event_api.h>
60	#include <linux/profile.h>
61	#include <linux/psi.h>
62	#include <linux/rcuwait_api.h>
63	#include <linux/rseq.h>
64	#include <linux/sched/wake_q.h>
65	#include <linux/scs.h>
66	#include <linux/slab.h>
67	#include <linux/syscalls.h>
68	#include <linux/vtime.h>
69	#include <linux/wait_api.h>
70	#include <linux/workqueue_api.h>
71	#include <linux/livepatch_sched.h>
72
73	#ifdef CONFIG_PREEMPT_DYNAMIC
74	# ifdef CONFIG_GENERIC_IRQ_ENTRY
75	# include <linux/irq-entry-common.h>
76	# endif
77	#endif
78
79	#include <uapi/linux/sched/types.h>
80
81	#include <asm/irq_regs.h>
82	#include <asm/switch_to.h>
83	#include <asm/tlb.h>
84
85	#define CREATE_TRACE_POINTS
86	#include <linux/sched/rseq_api.h>
87	#include <trace/events/sched.h>
88	#include <trace/events/ipi.h>
89	#undef CREATE_TRACE_POINTS
90
91	#include "sched.h"
92	#include "stats.h"
93
94	#include "autogroup.h"
95	#include "pelt.h"
96	#include "smp.h"
97
98	#include "../workqueue_internal.h"
99	#include "../../io_uring/io-wq.h"
100	#include "../smpboot.h"
101	#include "../locking/mutex.h"
102
103	EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu);
104	EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask);
105
106	/*
107	* Export tracepoints that act as a bare tracehook (ie: have no trace event
108	* associated with them) to allow external modules to probe them.
109	*/
110	EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_cfs_tp);
111	EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_rt_tp);
112	EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_dl_tp);
113	EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_irq_tp);
114	EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_se_tp);
115	EXPORT_TRACEPOINT_SYMBOL_GPL(pelt_hw_tp);
116	EXPORT_TRACEPOINT_SYMBOL_GPL(sched_cpu_capacity_tp);
117	EXPORT_TRACEPOINT_SYMBOL_GPL(sched_overutilized_tp);
118	EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_cfs_tp);
119	EXPORT_TRACEPOINT_SYMBOL_GPL(sched_util_est_se_tp);
120	EXPORT_TRACEPOINT_SYMBOL_GPL(sched_update_nr_running_tp);
121	EXPORT_TRACEPOINT_SYMBOL_GPL(sched_compute_energy_tp);
122
123	DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
124	DEFINE_PER_CPU(struct rnd_state, sched_rnd_state);
125
126	#ifdef CONFIG_SCHED_PROXY_EXEC
127	DEFINE_STATIC_KEY_TRUE(__sched_proxy_exec);
128	static int __init setup_proxy_exec(char *str)
129	{
130	bool proxy_enable = true;
131
132	if (*str && kstrtobool(s: str + 1, res: &proxy_enable)) {
133	pr_warn("Unable to parse sched_proxy_exec=\n");
134	return 0;
135	}
136
137	if (proxy_enable) {
138	pr_info("sched_proxy_exec enabled via boot arg\n");
139	static_branch_enable(&__sched_proxy_exec);
140	} else {
141	pr_info("sched_proxy_exec disabled via boot arg\n");
142	static_branch_disable(&__sched_proxy_exec);
143	}
144	return 1;
145	}
146	#else
147	static int __init setup_proxy_exec(char *str)
148	{
149	pr_warn("CONFIG_SCHED_PROXY_EXEC=n, so it cannot be enabled or disabled at boot time\n");
150	return 0;
151	}
152	#endif
153	__setup("sched_proxy_exec", setup_proxy_exec);
154
155	/*
156	* Debugging: various feature bits
157	*
158	* If SCHED_DEBUG is disabled, each compilation unit has its own copy of
159	* sysctl_sched_features, defined in sched.h, to allow constants propagation
160	* at compile time and compiler optimization based on features default.
161	*/
162	#define SCHED_FEAT(name, enabled) \
163	(1UL << __SCHED_FEAT_##name) * enabled |
164	__read_mostly unsigned int sysctl_sched_features =
165	#include "features.h"
166	0;
167	#undef SCHED_FEAT
168
169	/*
170	* Print a warning if need_resched is set for the given duration (if
171	* LATENCY_WARN is enabled).
172	*
173	* If sysctl_resched_latency_warn_once is set, only one warning will be shown
174	* per boot.
175	*/
176	__read_mostly int sysctl_resched_latency_warn_ms = 100;
177	__read_mostly int sysctl_resched_latency_warn_once = 1;
178
179	/*
180	* Number of tasks to iterate in a single balance run.
181	* Limited because this is done with IRQs disabled.
182	*/
183	__read_mostly unsigned int sysctl_sched_nr_migrate = SCHED_NR_MIGRATE_BREAK;
184
185	__read_mostly int scheduler_running;
186
187	#ifdef CONFIG_SCHED_CORE
188
189	DEFINE_STATIC_KEY_FALSE(__sched_core_enabled);
190
191	/* kernel prio, less is more */
192	static inline int __task_prio(const struct task_struct *p)
193	{
194	if (p->sched_class == &stop_sched_class) /* trumps deadline */
195	return -2;
196
197	if (p->dl_server)
198	return -1; /* deadline */
199
200	if (rt_or_dl_prio(prio: p->prio))
201	return p->prio; /* [-1, 99] */
202
203	if (p->sched_class == &idle_sched_class)
204	return MAX_RT_PRIO + NICE_WIDTH; /* 140 */
205
206	if (task_on_scx(p))
207	return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */
208
209	return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */
210	}
211
212	/*
213	* l(a,b)
214	* le(a,b) := !l(b,a)
215	* g(a,b) := l(b,a)
216	* ge(a,b) := !l(a,b)
217	*/
218
219	/* real prio, less is less */
220	static inline bool prio_less(const struct task_struct *a,
221	const struct task_struct *b, bool in_fi)
222	{
223
224	int pa = __task_prio(p: a), pb = __task_prio(p: b);
225
226	if (-pa < -pb)
227	return true;
228
229	if (-pb < -pa)
230	return false;
231
232	if (pa == -1) { /* dl_prio() doesn't work because of stop_class above */
233	const struct sched_dl_entity *a_dl, *b_dl;
234
235	a_dl = &a->dl;
236	/*
237	* Since,'a' and 'b' can be CFS tasks served by DL server,
238	* __task_prio() can return -1 (for DL) even for those. In that
239	* case, get to the dl_server's DL entity.
240	*/
241	if (a->dl_server)
242	a_dl = a->dl_server;
243
244	b_dl = &b->dl;
245	if (b->dl_server)
246	b_dl = b->dl_server;
247
248	return !dl_time_before(a: a_dl->deadline, b: b_dl->deadline);
249	}
250
251	if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */
252	return cfs_prio_less(a, b, fi: in_fi);
253
254	#ifdef CONFIG_SCHED_CLASS_EXT
255	if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */
256	return scx_prio_less(a, b, in_fi);
257	#endif
258
259	return false;
260	}
261
262	static inline bool __sched_core_less(const struct task_struct *a,
263	const struct task_struct *b)
264	{
265	if (a->core_cookie < b->core_cookie)
266	return true;
267
268	if (a->core_cookie > b->core_cookie)
269	return false;
270
271	/* flip prio, so high prio is leftmost */
272	if (prio_less(a: b, b: a, in_fi: !!task_rq(a)->core->core_forceidle_count))
273	return true;
274
275	return false;
276	}
277
278	#define __node_2_sc(node) rb_entry((node), struct task_struct, core_node)
279
280	static inline bool rb_sched_core_less(struct rb_node *a, const struct rb_node *b)
281	{
282	return __sched_core_less(__node_2_sc(a), __node_2_sc(b));
283	}
284
285	static inline int rb_sched_core_cmp(const void *key, const struct rb_node *node)
286	{
287	const struct task_struct *p = __node_2_sc(node);
288	unsigned long cookie = (unsigned long)key;
289
290	if (cookie < p->core_cookie)
291	return -1;
292
293	if (cookie > p->core_cookie)
294	return 1;
295
296	return 0;
297	}
298
299	void sched_core_enqueue(struct rq *rq, struct task_struct *p)
300	{
301	if (p->se.sched_delayed)
302	return;
303
304	rq->core->core_task_seq++;
305
306	if (!p->core_cookie)
307	return;
308
309	rb_add(node: &p->core_node, tree: &rq->core_tree, less: rb_sched_core_less);
310	}
311
312	void sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags)
313	{
314	if (p->se.sched_delayed)
315	return;
316
317	rq->core->core_task_seq++;
318
319	if (sched_core_enqueued(p)) {
320	rb_erase(&p->core_node, &rq->core_tree);
321	RB_CLEAR_NODE(&p->core_node);
322	}
323
324	/*
325	* Migrating the last task off the cpu, with the cpu in forced idle
326	* state. Reschedule to create an accounting edge for forced idle,
327	* and re-examine whether the core is still in forced idle state.
328	*/
329	if (!(flags & DEQUEUE_SAVE) && rq->nr_running == 1 &&
330	rq->core->core_forceidle_count && rq->curr == rq->idle)
331	resched_curr(rq);
332	}
333
334	static int sched_task_is_throttled(struct task_struct *p, int cpu)
335	{
336	if (p->sched_class->task_is_throttled)
337	return p->sched_class->task_is_throttled(p, cpu);
338
339	return 0;
340	}
341
342	static struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie)
343	{
344	struct rb_node *node = &p->core_node;
345	int cpu = task_cpu(p);
346
347	do {
348	node = rb_next(node);
349	if (!node)
350	return NULL;
351
352	p = __node_2_sc(node);
353	if (p->core_cookie != cookie)
354	return NULL;
355
356	} while (sched_task_is_throttled(p, cpu));
357
358	return p;
359	}
360
361	/*
362	* Find left-most (aka, highest priority) and unthrottled task matching @cookie.
363	* If no suitable task is found, NULL will be returned.
364	*/
365	static struct task_struct *sched_core_find(struct rq *rq, unsigned long cookie)
366	{
367	struct task_struct *p;
368	struct rb_node *node;
369
370	node = rb_find_first(key: (void *)cookie, tree: &rq->core_tree, cmp: rb_sched_core_cmp);
371	if (!node)
372	return NULL;
373
374	p = __node_2_sc(node);
375	if (!sched_task_is_throttled(p, cpu: rq->cpu))
376	return p;
377
378	return sched_core_next(p, cookie);
379	}
380
381	/*
382	* Magic required such that:
383	*
384	* raw_spin_rq_lock(rq);
385	* ...
386	* raw_spin_rq_unlock(rq);
387	*
388	* ends up locking and unlocking the _same_ lock, and all CPUs
389	* always agree on what rq has what lock.
390	*
391	* XXX entirely possible to selectively enable cores, don't bother for now.
392	*/
393
394	static DEFINE_MUTEX(sched_core_mutex);
395	static atomic_t sched_core_count;
396	static struct cpumask sched_core_mask;
397
398	static void sched_core_lock(int cpu, unsigned long *flags)
399	{
400	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
401	int t, i = 0;
402
403	local_irq_save(*flags);
404	for_each_cpu(t, smt_mask)
405	raw_spin_lock_nested(&cpu_rq(t)->__lock, i++);
406	}
407
408	static void sched_core_unlock(int cpu, unsigned long *flags)
409	{
410	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
411	int t;
412
413	for_each_cpu(t, smt_mask)
414	raw_spin_unlock(&cpu_rq(t)->__lock);
415	local_irq_restore(*flags);
416	}
417
418	static void __sched_core_flip(bool enabled)
419	{
420	unsigned long flags;
421	int cpu, t;
422
423	cpus_read_lock();
424
425	/*
426	* Toggle the online cores, one by one.
427	*/
428	cpumask_copy(dstp: &sched_core_mask, cpu_online_mask);
429	for_each_cpu(cpu, &sched_core_mask) {
430	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
431
432	sched_core_lock(cpu, flags: &flags);
433
434	for_each_cpu(t, smt_mask)
435	cpu_rq(t)->core_enabled = enabled;
436
437	cpu_rq(cpu)->core->core_forceidle_start = 0;
438
439	sched_core_unlock(cpu, flags: &flags);
440
441	cpumask_andnot(dstp: &sched_core_mask, src1p: &sched_core_mask, src2p: smt_mask);
442	}
443
444	/*
445	* Toggle the offline CPUs.
446	*/
447	for_each_cpu_andnot(cpu, cpu_possible_mask, cpu_online_mask)
448	cpu_rq(cpu)->core_enabled = enabled;
449
450	cpus_read_unlock();
451	}
452
453	static void sched_core_assert_empty(void)
454	{
455	int cpu;
456
457	for_each_possible_cpu(cpu)
458	WARN_ON_ONCE(!RB_EMPTY_ROOT(&cpu_rq(cpu)->core_tree));
459	}
460
461	static void __sched_core_enable(void)
462	{
463	static_branch_enable(&__sched_core_enabled);
464	/*
465	* Ensure all previous instances of raw_spin_rq_*lock() have finished
466	* and future ones will observe !sched_core_disabled().
467	*/
468	synchronize_rcu();
469	__sched_core_flip(enabled: true);
470	sched_core_assert_empty();
471	}
472
473	static void __sched_core_disable(void)
474	{
475	sched_core_assert_empty();
476	__sched_core_flip(enabled: false);
477	static_branch_disable(&__sched_core_enabled);
478	}
479
480	void sched_core_get(void)
481	{
482	if (atomic_inc_not_zero(v: &sched_core_count))
483	return;
484
485	mutex_lock(&sched_core_mutex);
486	if (!atomic_read(v: &sched_core_count))
487	__sched_core_enable();
488
489	smp_mb__before_atomic();
490	atomic_inc(v: &sched_core_count);
491	mutex_unlock(lock: &sched_core_mutex);
492	}
493
494	static void __sched_core_put(struct work_struct *work)
495	{
496	if (atomic_dec_and_mutex_lock(cnt: &sched_core_count, lock: &sched_core_mutex)) {
497	__sched_core_disable();
498	mutex_unlock(lock: &sched_core_mutex);
499	}
500	}
501
502	void sched_core_put(void)
503	{
504	static DECLARE_WORK(_work, __sched_core_put);
505
506	/*
507	* "There can be only one"
508	*
509	* Either this is the last one, or we don't actually need to do any
510	* 'work'. If it is the last *again*, we rely on
511	* WORK_STRUCT_PENDING_BIT.
512	*/
513	if (!atomic_add_unless(v: &sched_core_count, a: -1, u: 1))
514	schedule_work(work: &_work);
515	}
516
517	#else /* !CONFIG_SCHED_CORE: */
518
519	static inline void sched_core_enqueue(struct rq *rq, struct task_struct *p) { }
520	static inline void
521	sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { }
522
523	#endif /* !CONFIG_SCHED_CORE */
524
525	/* need a wrapper since we may need to trace from modules */
526	EXPORT_TRACEPOINT_SYMBOL(sched_set_state_tp);
527
528	/* Call via the helper macro trace_set_current_state. */
529	void __trace_set_current_state(int state_value)
530	{
531	trace_sched_set_state_tp(current, state: state_value);
532	}
533	EXPORT_SYMBOL(__trace_set_current_state);
534
535	/*
536	* Serialization rules:
537	*
538	* Lock order:
539	*
540	* p->pi_lock
541	* rq->lock
542	* hrtimer_cpu_base->lock (hrtimer_start() for bandwidth controls)
543	*
544	* rq1->lock
545	* rq2->lock where: rq1 < rq2
546	*
547	* Regular state:
548	*
549	* Normal scheduling state is serialized by rq->lock. __schedule() takes the
550	* local CPU's rq->lock, it optionally removes the task from the runqueue and
551	* always looks at the local rq data structures to find the most eligible task
552	* to run next.
553	*
554	* Task enqueue is also under rq->lock, possibly taken from another CPU.
555	* Wakeups from another LLC domain might use an IPI to transfer the enqueue to
556	* the local CPU to avoid bouncing the runqueue state around [ see
557	* ttwu_queue_wakelist() ]
558	*
559	* Task wakeup, specifically wakeups that involve migration, are horribly
560	* complicated to avoid having to take two rq->locks.
561	*
562	* Special state:
563	*
564	* System-calls and anything external will use task_rq_lock() which acquires
565	* both p->pi_lock and rq->lock. As a consequence the state they change is
566	* stable while holding either lock:
567	*
568	* - sched_setaffinity()/
569	* set_cpus_allowed_ptr(): p->cpus_ptr, p->nr_cpus_allowed
570	* - set_user_nice(): p->se.load, p->*prio
571	* - __sched_setscheduler(): p->sched_class, p->policy, p->*prio,
572	* p->se.load, p->rt_priority,
573	* p->dl.dl_{runtime, deadline, period, flags, bw, density}
574	* - sched_setnuma(): p->numa_preferred_nid
575	* - sched_move_task(): p->sched_task_group
576	* - uclamp_update_active() p->uclamp*
577	*
578	* p->state <- TASK_*:
579	*
580	* is changed locklessly using set_current_state(), __set_current_state() or
581	* set_special_state(), see their respective comments, or by
582	* try_to_wake_up(). This latter uses p->pi_lock to serialize against
583	* concurrent self.
584	*
585	* p->on_rq <- { 0, 1 = TASK_ON_RQ_QUEUED, 2 = TASK_ON_RQ_MIGRATING }:
586	*
587	* is set by activate_task() and cleared by deactivate_task()/block_task(),
588	* under rq->lock. Non-zero indicates the task is runnable, the special
589	* ON_RQ_MIGRATING state is used for migration without holding both
590	* rq->locks. It indicates task_cpu() is not stable, see task_rq_lock().
591	*
592	* Additionally it is possible to be ->on_rq but still be considered not
593	* runnable when p->se.sched_delayed is true. These tasks are on the runqueue
594	* but will be dequeued as soon as they get picked again. See the
595	* task_is_runnable() helper.
596	*
597	* p->on_cpu <- { 0, 1 }:
598	*
599	* is set by prepare_task() and cleared by finish_task() such that it will be
600	* set before p is scheduled-in and cleared after p is scheduled-out, both
601	* under rq->lock. Non-zero indicates the task is running on its CPU.
602	*
603	* [ The astute reader will observe that it is possible for two tasks on one
604	* CPU to have ->on_cpu = 1 at the same time. ]
605	*
606	* task_cpu(p): is changed by set_task_cpu(), the rules are:
607	*
608	* - Don't call set_task_cpu() on a blocked task:
609	*
610	* We don't care what CPU we're not running on, this simplifies hotplug,
611	* the CPU assignment of blocked tasks isn't required to be valid.
612	*
613	* - for try_to_wake_up(), called under p->pi_lock:
614	*
615	* This allows try_to_wake_up() to only take one rq->lock, see its comment.
616	*
617	* - for migration called under rq->lock:
618	* [ see task_on_rq_migrating() in task_rq_lock() ]
619	*
620	* o move_queued_task()
621	* o detach_task()
622	*
623	* - for migration called under double_rq_lock():
624	*
625	* o __migrate_swap_task()
626	* o push_rt_task() / pull_rt_task()
627	* o push_dl_task() / pull_dl_task()
628	* o dl_task_offline_migration()
629	*
630	*/
631
632	void raw_spin_rq_lock_nested(struct rq *rq, int subclass)
633	{
634	raw_spinlock_t *lock;
635
636	/* Matches synchronize_rcu() in __sched_core_enable() */
637	preempt_disable();
638	if (sched_core_disabled()) {
639	raw_spin_lock_nested(&rq->__lock, subclass);
640	/* preempt_count *MUST* be > 1 */
641	preempt_enable_no_resched();
642	return;
643	}
644
645	for (;;) {
646	lock = __rq_lockp(rq);
647	raw_spin_lock_nested(lock, subclass);
648	if (likely(lock == __rq_lockp(rq))) {
649	/* preempt_count *MUST* be > 1 */
650	preempt_enable_no_resched();
651	return;
652	}
653	raw_spin_unlock(lock);
654	}
655	}
656
657	bool raw_spin_rq_trylock(struct rq *rq)
658	{
659	raw_spinlock_t *lock;
660	bool ret;
661
662	/* Matches synchronize_rcu() in __sched_core_enable() */
663	preempt_disable();
664	if (sched_core_disabled()) {
665	ret = raw_spin_trylock(&rq->__lock);
666	preempt_enable();
667	return ret;
668	}
669
670	for (;;) {
671	lock = __rq_lockp(rq);
672	ret = raw_spin_trylock(lock);
673	if (!ret || (likely(lock == __rq_lockp(rq)))) {
674	preempt_enable();
675	return ret;
676	}
677	raw_spin_unlock(lock);
678	}
679	}
680
681	void raw_spin_rq_unlock(struct rq *rq)
682	{
683	raw_spin_unlock(rq_lockp(rq));
684	}
685
686	/*
687	* double_rq_lock - safely lock two runqueues
688	*/
689	void double_rq_lock(struct rq *rq1, struct rq *rq2)
690	{
691	lockdep_assert_irqs_disabled();
692
693	if (rq_order_less(rq1: rq2, rq2: rq1))
694	swap(rq1, rq2);
695
696	raw_spin_rq_lock(rq: rq1);
697	if (__rq_lockp(rq: rq1) != __rq_lockp(rq: rq2))
698	raw_spin_rq_lock_nested(rq: rq2, SINGLE_DEPTH_NESTING);
699
700	double_rq_clock_clear_update(rq1, rq2);
701	}
702
703	/*
704	* __task_rq_lock - lock the rq @p resides on.
705	*/
706	struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf)
707	__acquires(rq->lock)
708	{
709	struct rq *rq;
710
711	lockdep_assert_held(&p->pi_lock);
712
713	for (;;) {
714	rq = task_rq(p);
715	raw_spin_rq_lock(rq);
716	if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
717	rq_pin_lock(rq, rf);
718	return rq;
719	}
720	raw_spin_rq_unlock(rq);
721
722	while (unlikely(task_on_rq_migrating(p)))
723	cpu_relax();
724	}
725	}
726
727	/*
728	* task_rq_lock - lock p->pi_lock and lock the rq @p resides on.
729	*/
730	struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
731	__acquires(p->pi_lock)
732	__acquires(rq->lock)
733	{
734	struct rq *rq;
735
736	for (;;) {
737	raw_spin_lock_irqsave(&p->pi_lock, rf->flags);
738	rq = task_rq(p);
739	raw_spin_rq_lock(rq);
740	/*
741	* move_queued_task() task_rq_lock()
742	*
743	* ACQUIRE (rq->lock)
744	* [S] ->on_rq = MIGRATING [L] rq = task_rq()
745	* WMB (__set_task_cpu()) ACQUIRE (rq->lock);
746	* [S] ->cpu = new_cpu [L] task_rq()
747	* [L] ->on_rq
748	* RELEASE (rq->lock)
749	*
750	* If we observe the old CPU in task_rq_lock(), the acquire of
751	* the old rq->lock will fully serialize against the stores.
752	*
753	* If we observe the new CPU in task_rq_lock(), the address
754	* dependency headed by '[L] rq = task_rq()' and the acquire
755	* will pair with the WMB to ensure we then also see migrating.
756	*/
757	if (likely(rq == task_rq(p) && !task_on_rq_migrating(p))) {
758	rq_pin_lock(rq, rf);
759	return rq;
760	}
761	raw_spin_rq_unlock(rq);
762	raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags);
763
764	while (unlikely(task_on_rq_migrating(p)))
765	cpu_relax();
766	}
767	}
768
769	/*
770	* RQ-clock updating methods:
771	*/
772
773	static void update_rq_clock_task(struct rq *rq, s64 delta)
774	{
775	/*
776	* In theory, the compile should just see 0 here, and optimize out the call
777	* to sched_rt_avg_update. But I don't trust it...
778	*/
779	s64 __maybe_unused steal = 0, irq_delta = 0;
780
781	#ifdef CONFIG_IRQ_TIME_ACCOUNTING
782	if (irqtime_enabled()) {
783	irq_delta = irq_time_read(cpu: cpu_of(rq)) - rq->prev_irq_time;
784
785	/*
786	* Since irq_time is only updated on {soft,}irq_exit, we might run into
787	* this case when a previous update_rq_clock() happened inside a
788	* {soft,}IRQ region.
789	*
790	* When this happens, we stop ->clock_task and only update the
791	* prev_irq_time stamp to account for the part that fit, so that a next
792	* update will consume the rest. This ensures ->clock_task is
793	* monotonic.
794	*
795	* It does however cause some slight miss-attribution of {soft,}IRQ
796	* time, a more accurate solution would be to update the irq_time using
797	* the current rq->clock timestamp, except that would require using
798	* atomic ops.
799	*/
800	if (irq_delta > delta)
801	irq_delta = delta;
802
803	rq->prev_irq_time += irq_delta;
804	delta -= irq_delta;
805	delayacct_irq(task: rq->curr, delta: irq_delta);
806	}
807	#endif
808	#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING
809	if (static_key_false(key: (&paravirt_steal_rq_enabled))) {
810	u64 prev_steal;
811
812	steal = prev_steal = paravirt_steal_clock(cpu: cpu_of(rq));
813	steal -= rq->prev_steal_time_rq;
814
815	if (unlikely(steal > delta))
816	steal = delta;
817
818	rq->prev_steal_time_rq = prev_steal;
819	delta -= steal;
820	}
821	#endif
822
823	rq->clock_task += delta;
824
825	#ifdef CONFIG_HAVE_SCHED_AVG_IRQ
826	if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY))
827	update_irq_load_avg(rq, running: irq_delta + steal);
828	#endif
829	update_rq_clock_pelt(rq, delta);
830	}
831
832	void update_rq_clock(struct rq *rq)
833	{
834	s64 delta;
835	u64 clock;
836
837	lockdep_assert_rq_held(rq);
838
839	if (rq->clock_update_flags & RQCF_ACT_SKIP)
840	return;
841
842	if (sched_feat(WARN_DOUBLE_CLOCK))
843	WARN_ON_ONCE(rq->clock_update_flags & RQCF_UPDATED);
844	rq->clock_update_flags |= RQCF_UPDATED;
845
846	clock = sched_clock_cpu(cpu: cpu_of(rq));
847	scx_rq_clock_update(rq, clock);
848
849	delta = clock - rq->clock;
850	if (delta < 0)
851	return;
852	rq->clock += delta;
853
854	update_rq_clock_task(rq, delta);
855	}
856
857	#ifdef CONFIG_SCHED_HRTICK
858	/*
859	* Use HR-timers to deliver accurate preemption points.
860	*/
861
862	static void hrtick_clear(struct rq *rq)
863	{
864	if (hrtimer_active(timer: &rq->hrtick_timer))
865	hrtimer_cancel(timer: &rq->hrtick_timer);
866	}
867
868	/*
869	* High-resolution timer tick.
870	* Runs from hardirq context with interrupts disabled.
871	*/
872	static enum hrtimer_restart hrtick(struct hrtimer *timer)
873	{
874	struct rq *rq = container_of(timer, struct rq, hrtick_timer);
875	struct rq_flags rf;
876
877	WARN_ON_ONCE(cpu_of(rq) != smp_processor_id());
878
879	rq_lock(rq, rf: &rf);
880	update_rq_clock(rq);
881	rq->donor->sched_class->task_tick(rq, rq->donor, 1);
882	rq_unlock(rq, rf: &rf);
883
884	return HRTIMER_NORESTART;
885	}
886
887	static void __hrtick_restart(struct rq *rq)
888	{
889	struct hrtimer *timer = &rq->hrtick_timer;
890	ktime_t time = rq->hrtick_time;
891
892	hrtimer_start(timer, tim: time, mode: HRTIMER_MODE_ABS_PINNED_HARD);
893	}
894
895	/*
896	* called from hardirq (IPI) context
897	*/
898	static void __hrtick_start(void *arg)
899	{
900	struct rq *rq = arg;
901	struct rq_flags rf;
902
903	rq_lock(rq, rf: &rf);
904	__hrtick_restart(rq);
905	rq_unlock(rq, rf: &rf);
906	}
907
908	/*
909	* Called to set the hrtick timer state.
910	*
911	* called with rq->lock held and IRQs disabled
912	*/
913	void hrtick_start(struct rq *rq, u64 delay)
914	{
915	struct hrtimer *timer = &rq->hrtick_timer;
916	s64 delta;
917
918	/*
919	* Don't schedule slices shorter than 10000ns, that just
920	* doesn't make sense and can cause timer DoS.
921	*/
922	delta = max_t(s64, delay, 10000LL);
923	rq->hrtick_time = ktime_add_ns(hrtimer_cb_get_time(timer), delta);
924
925	if (rq == this_rq())
926	__hrtick_restart(rq);
927	else
928	smp_call_function_single_async(cpu: cpu_of(rq), csd: &rq->hrtick_csd);
929	}
930
931	static void hrtick_rq_init(struct rq *rq)
932	{
933	INIT_CSD(&rq->hrtick_csd, __hrtick_start, rq);
934	hrtimer_setup(timer: &rq->hrtick_timer, function: hrtick, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL_HARD);
935	}
936	#else /* !CONFIG_SCHED_HRTICK: */
937	static inline void hrtick_clear(struct rq *rq)
938	{
939	}
940
941	static inline void hrtick_rq_init(struct rq *rq)
942	{
943	}
944	#endif /* !CONFIG_SCHED_HRTICK */
945
946	/*
947	* try_cmpxchg based fetch_or() macro so it works for different integer types:
948	*/
949	#define fetch_or(ptr, mask) \
950	({ \
951	typeof(ptr) _ptr = (ptr); \
952	typeof(mask) _mask = (mask); \
953	typeof(*_ptr) _val = *_ptr; \
954	\
955	do { \
956	} while (!try_cmpxchg(_ptr, &_val, _val | _mask)); \
957	_val; \
958	})
959
960	#ifdef TIF_POLLING_NRFLAG
961	/*
962	* Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG,
963	* this avoids any races wrt polling state changes and thereby avoids
964	* spurious IPIs.
965	*/
966	static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif)
967	{
968	return !(fetch_or(&ti->flags, 1 << tif) & _TIF_POLLING_NRFLAG);
969	}
970
971	/*
972	* Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set.
973	*
974	* If this returns true, then the idle task promises to call
975	* sched_ttwu_pending() and reschedule soon.
976	*/
977	static bool set_nr_if_polling(struct task_struct *p)
978	{
979	struct thread_info *ti = task_thread_info(p);
980	typeof(ti->flags) val = READ_ONCE(ti->flags);
981
982	do {
983	if (!(val & _TIF_POLLING_NRFLAG))
984	return false;
985	if (val & _TIF_NEED_RESCHED)
986	return true;
987	} while (!try_cmpxchg(&ti->flags, &val, val | _TIF_NEED_RESCHED));
988
989	return true;
990	}
991
992	#else
993	static inline bool set_nr_and_not_polling(struct thread_info *ti, int tif)
994	{
995	set_ti_thread_flag(ti, tif);
996	return true;
997	}
998
999	static inline bool set_nr_if_polling(struct task_struct *p)
1000	{
1001	return false;
1002	}
1003	#endif
1004
1005	static bool __wake_q_add(struct wake_q_head *head, struct task_struct *task)
1006	{
1007	struct wake_q_node *node = &task->wake_q;
1008
1009	/*
1010	* Atomically grab the task, if ->wake_q is !nil already it means
1011	* it's already queued (either by us or someone else) and will get the
1012	* wakeup due to that.
1013	*
1014	* In order to ensure that a pending wakeup will observe our pending
1015	* state, even in the failed case, an explicit smp_mb() must be used.
1016	*/
1017	smp_mb__before_atomic();
1018	if (unlikely(cmpxchg_relaxed(&node->next, NULL, WAKE_Q_TAIL)))
1019	return false;
1020
1021	/*
1022	* The head is context local, there can be no concurrency.
1023	*/
1024	*head->lastp = node;
1025	head->lastp = &node->next;
1026	return true;
1027	}
1028
1029	/**
1030	* wake_q_add() - queue a wakeup for 'later' waking.
1031	* @head: the wake_q_head to add @task to
1032	* @task: the task to queue for 'later' wakeup
1033	*
1034	* Queue a task for later wakeup, most likely by the wake_up_q() call in the
1035	* same context, _HOWEVER_ this is not guaranteed, the wakeup can come
1036	* instantly.
1037	*
1038	* This function must be used as-if it were wake_up_process(); IOW the task
1039	* must be ready to be woken at this location.
1040	*/
1041	void wake_q_add(struct wake_q_head *head, struct task_struct *task)
1042	{
1043	if (__wake_q_add(head, task))
1044	get_task_struct(t: task);
1045	}
1046
1047	/**
1048	* wake_q_add_safe() - safely queue a wakeup for 'later' waking.
1049	* @head: the wake_q_head to add @task to
1050	* @task: the task to queue for 'later' wakeup
1051	*
1052	* Queue a task for later wakeup, most likely by the wake_up_q() call in the
1053	* same context, _HOWEVER_ this is not guaranteed, the wakeup can come
1054	* instantly.
1055	*
1056	* This function must be used as-if it were wake_up_process(); IOW the task
1057	* must be ready to be woken at this location.
1058	*
1059	* This function is essentially a task-safe equivalent to wake_q_add(). Callers
1060	* that already hold reference to @task can call the 'safe' version and trust
1061	* wake_q to do the right thing depending whether or not the @task is already
1062	* queued for wakeup.
1063	*/
1064	void wake_q_add_safe(struct wake_q_head *head, struct task_struct *task)
1065	{
1066	if (!__wake_q_add(head, task))
1067	put_task_struct(t: task);
1068	}
1069
1070	void wake_up_q(struct wake_q_head *head)
1071	{
1072	struct wake_q_node *node = head->first;
1073
1074	while (node != WAKE_Q_TAIL) {
1075	struct task_struct *task;
1076
1077	task = container_of(node, struct task_struct, wake_q);
1078	node = node->next;
1079	/* pairs with cmpxchg_relaxed() in __wake_q_add() */
1080	WRITE_ONCE(task->wake_q.next, NULL);
1081	/* Task can safely be re-inserted now. */
1082
1083	/*
1084	* wake_up_process() executes a full barrier, which pairs with
1085	* the queueing in wake_q_add() so as not to miss wakeups.
1086	*/
1087	wake_up_process(tsk: task);
1088	put_task_struct(t: task);
1089	}
1090	}
1091
1092	/*
1093	* resched_curr - mark rq's current task 'to be rescheduled now'.
1094	*
1095	* On UP this means the setting of the need_resched flag, on SMP it
1096	* might also involve a cross-CPU call to trigger the scheduler on
1097	* the target CPU.
1098	*/
1099	static void __resched_curr(struct rq *rq, int tif)
1100	{
1101	struct task_struct *curr = rq->curr;
1102	struct thread_info *cti = task_thread_info(curr);
1103	int cpu;
1104
1105	lockdep_assert_rq_held(rq);
1106
1107	/*
1108	* Always immediately preempt the idle task; no point in delaying doing
1109	* actual work.
1110	*/
1111	if (is_idle_task(p: curr) && tif == TIF_NEED_RESCHED_LAZY)
1112	tif = TIF_NEED_RESCHED;
1113
1114	if (cti->flags & ((1 << tif) | _TIF_NEED_RESCHED))
1115	return;
1116
1117	cpu = cpu_of(rq);
1118
1119	trace_sched_set_need_resched_tp(tsk: curr, cpu, tif);
1120	if (cpu == smp_processor_id()) {
1121	set_ti_thread_flag(ti: cti, flag: tif);
1122	if (tif == TIF_NEED_RESCHED)
1123	set_preempt_need_resched();
1124	return;
1125	}
1126
1127	if (set_nr_and_not_polling(ti: cti, tif)) {
1128	if (tif == TIF_NEED_RESCHED)
1129	smp_send_reschedule(cpu);
1130	} else {
1131	trace_sched_wake_idle_without_ipi(cpu);
1132	}
1133	}
1134
1135	void __trace_set_need_resched(struct task_struct *curr, int tif)
1136	{
1137	trace_sched_set_need_resched_tp(tsk: curr, smp_processor_id(), tif);
1138	}
1139
1140	void resched_curr(struct rq *rq)
1141	{
1142	__resched_curr(rq, TIF_NEED_RESCHED);
1143	}
1144
1145	#ifdef CONFIG_PREEMPT_DYNAMIC
1146	static DEFINE_STATIC_KEY_FALSE(sk_dynamic_preempt_lazy);
1147	static __always_inline bool dynamic_preempt_lazy(void)
1148	{
1149	return static_branch_unlikely(&sk_dynamic_preempt_lazy);
1150	}
1151	#else
1152	static __always_inline bool dynamic_preempt_lazy(void)
1153	{
1154	return IS_ENABLED(CONFIG_PREEMPT_LAZY);
1155	}
1156	#endif
1157
1158	static __always_inline int get_lazy_tif_bit(void)
1159	{
1160	if (dynamic_preempt_lazy())
1161	return TIF_NEED_RESCHED_LAZY;
1162
1163	return TIF_NEED_RESCHED;
1164	}
1165
1166	void resched_curr_lazy(struct rq *rq)
1167	{
1168	__resched_curr(rq, tif: get_lazy_tif_bit());
1169	}
1170
1171	void resched_cpu(int cpu)
1172	{
1173	struct rq *rq = cpu_rq(cpu);
1174	unsigned long flags;
1175
1176	raw_spin_rq_lock_irqsave(rq, flags);
1177	if (cpu_online(cpu) || cpu == smp_processor_id())
1178	resched_curr(rq);
1179	raw_spin_rq_unlock_irqrestore(rq, flags);
1180	}
1181
1182	#ifdef CONFIG_NO_HZ_COMMON
1183	/*
1184	* In the semi idle case, use the nearest busy CPU for migrating timers
1185	* from an idle CPU. This is good for power-savings.
1186	*
1187	* We don't do similar optimization for completely idle system, as
1188	* selecting an idle CPU will add more delays to the timers than intended
1189	* (as that CPU's timer base may not be up to date wrt jiffies etc).
1190	*/
1191	int get_nohz_timer_target(void)
1192	{
1193	int i, cpu = smp_processor_id(), default_cpu = -1;
1194	struct sched_domain *sd;
1195	const struct cpumask *hk_mask;
1196
1197	if (housekeeping_cpu(cpu, type: HK_TYPE_KERNEL_NOISE)) {
1198	if (!idle_cpu(cpu))
1199	return cpu;
1200	default_cpu = cpu;
1201	}
1202
1203	hk_mask = housekeeping_cpumask(type: HK_TYPE_KERNEL_NOISE);
1204
1205	guard(rcu)();
1206
1207	for_each_domain(cpu, sd) {
1208	for_each_cpu_and(i, sched_domain_span(sd), hk_mask) {
1209	if (cpu == i)
1210	continue;
1211
1212	if (!idle_cpu(cpu: i))
1213	return i;
1214	}
1215	}
1216
1217	if (default_cpu == -1)
1218	default_cpu = housekeeping_any_cpu(type: HK_TYPE_KERNEL_NOISE);
1219
1220	return default_cpu;
1221	}
1222
1223	/*
1224	* When add_timer_on() enqueues a timer into the timer wheel of an
1225	* idle CPU then this timer might expire before the next timer event
1226	* which is scheduled to wake up that CPU. In case of a completely
1227	* idle system the next event might even be infinite time into the
1228	* future. wake_up_idle_cpu() ensures that the CPU is woken up and
1229	* leaves the inner idle loop so the newly added timer is taken into
1230	* account when the CPU goes back to idle and evaluates the timer
1231	* wheel for the next timer event.
1232	*/
1233	static void wake_up_idle_cpu(int cpu)
1234	{
1235	struct rq *rq = cpu_rq(cpu);
1236
1237	if (cpu == smp_processor_id())
1238	return;
1239
1240	/*
1241	* Set TIF_NEED_RESCHED and send an IPI if in the non-polling
1242	* part of the idle loop. This forces an exit from the idle loop
1243	* and a round trip to schedule(). Now this could be optimized
1244	* because a simple new idle loop iteration is enough to
1245	* re-evaluate the next tick. Provided some re-ordering of tick
1246	* nohz functions that would need to follow TIF_NR_POLLING
1247	* clearing:
1248	*
1249	* - On most architectures, a simple fetch_or on ti::flags with a
1250	* "0" value would be enough to know if an IPI needs to be sent.
1251	*
1252	* - x86 needs to perform a last need_resched() check between
1253	* monitor and mwait which doesn't take timers into account.
1254	* There a dedicated TIF_TIMER flag would be required to
1255	* fetch_or here and be checked along with TIF_NEED_RESCHED
1256	* before mwait().
1257	*
1258	* However, remote timer enqueue is not such a frequent event
1259	* and testing of the above solutions didn't appear to report
1260	* much benefits.
1261	*/
1262	if (set_nr_and_not_polling(task_thread_info(rq->idle), TIF_NEED_RESCHED))
1263	smp_send_reschedule(cpu);
1264	else
1265	trace_sched_wake_idle_without_ipi(cpu);
1266	}
1267
1268	static bool wake_up_full_nohz_cpu(int cpu)
1269	{
1270	/*
1271	* We just need the target to call irq_exit() and re-evaluate
1272	* the next tick. The nohz full kick at least implies that.
1273	* If needed we can still optimize that later with an
1274	* empty IRQ.
1275	*/
1276	if (cpu_is_offline(cpu))
1277	return true; /* Don't try to wake offline CPUs. */
1278	if (tick_nohz_full_cpu(cpu)) {
1279	if (cpu != smp_processor_id() ||
1280	tick_nohz_tick_stopped())
1281	tick_nohz_full_kick_cpu(cpu);
1282	return true;
1283	}
1284
1285	return false;
1286	}
1287
1288	/*
1289	* Wake up the specified CPU. If the CPU is going offline, it is the
1290	* caller's responsibility to deal with the lost wakeup, for example,
1291	* by hooking into the CPU_DEAD notifier like timers and hrtimers do.
1292	*/
1293	void wake_up_nohz_cpu(int cpu)
1294	{
1295	if (!wake_up_full_nohz_cpu(cpu))
1296	wake_up_idle_cpu(cpu);
1297	}
1298
1299	static void nohz_csd_func(void *info)
1300	{
1301	struct rq *rq = info;
1302	int cpu = cpu_of(rq);
1303	unsigned int flags;
1304
1305	/*
1306	* Release the rq::nohz_csd.
1307	*/
1308	flags = atomic_fetch_andnot(NOHZ_KICK_MASK | NOHZ_NEWILB_KICK, nohz_flags(cpu));
1309	WARN_ON(!(flags & NOHZ_KICK_MASK));
1310
1311	rq->idle_balance = idle_cpu(cpu);
1312	if (rq->idle_balance) {
1313	rq->nohz_idle_balance = flags;
1314	__raise_softirq_irqoff(nr: SCHED_SOFTIRQ);
1315	}
1316	}
1317
1318	#endif /* CONFIG_NO_HZ_COMMON */
1319
1320	#ifdef CONFIG_NO_HZ_FULL
1321	static inline bool __need_bw_check(struct rq *rq, struct task_struct *p)
1322	{
1323	if (rq->nr_running != 1)
1324	return false;
1325
1326	if (p->sched_class != &fair_sched_class)
1327	return false;
1328
1329	if (!task_on_rq_queued(p))
1330	return false;
1331
1332	return true;
1333	}
1334
1335	bool sched_can_stop_tick(struct rq *rq)
1336	{
1337	int fifo_nr_running;
1338
1339	/* Deadline tasks, even if single, need the tick */
1340	if (rq->dl.dl_nr_running)
1341	return false;
1342
1343	/*
1344	* If there are more than one RR tasks, we need the tick to affect the
1345	* actual RR behaviour.
1346	*/
1347	if (rq->rt.rr_nr_running) {
1348	if (rq->rt.rr_nr_running == 1)
1349	return true;
1350	else
1351	return false;
1352	}
1353
1354	/*
1355	* If there's no RR tasks, but FIFO tasks, we can skip the tick, no
1356	* forced preemption between FIFO tasks.
1357	*/
1358	fifo_nr_running = rq->rt.rt_nr_running - rq->rt.rr_nr_running;
1359	if (fifo_nr_running)
1360	return true;
1361
1362	/*
1363	* If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks
1364	* left. For CFS, if there's more than one we need the tick for
1365	* involuntary preemption. For SCX, ask.
1366	*/
1367	if (scx_enabled() && !scx_can_stop_tick(rq))
1368	return false;
1369
1370	if (rq->cfs.h_nr_queued > 1)
1371	return false;
1372
1373	/*
1374	* If there is one task and it has CFS runtime bandwidth constraints
1375	* and it's on the cpu now we don't want to stop the tick.
1376	* This check prevents clearing the bit if a newly enqueued task here is
1377	* dequeued by migrating while the constrained task continues to run.
1378	* E.g. going from 2->1 without going through pick_next_task().
1379	*/
1380	if (__need_bw_check(rq, rq->curr)) {
1381	if (cfs_task_bw_constrained(rq->curr))
1382	return false;
1383	}
1384
1385	return true;
1386	}
1387	#endif /* CONFIG_NO_HZ_FULL */
1388
1389	#if defined(CONFIG_RT_GROUP_SCHED) || defined(CONFIG_FAIR_GROUP_SCHED)
1390	/*
1391	* Iterate task_group tree rooted at *from, calling @down when first entering a
1392	* node and @up when leaving it for the final time.
1393	*
1394	* Caller must hold rcu_lock or sufficient equivalent.
1395	*/
1396	int walk_tg_tree_from(struct task_group *from,
1397	tg_visitor down, tg_visitor up, void *data)
1398	{
1399	struct task_group *parent, *child;
1400	int ret;
1401
1402	parent = from;
1403
1404	down:
1405	ret = (*down)(parent, data);
1406	if (ret)
1407	goto out;
1408	list_for_each_entry_rcu(child, &parent->children, siblings) {
1409	parent = child;
1410	goto down;
1411
1412	up:
1413	continue;
1414	}
1415	ret = (*up)(parent, data);
1416	if (ret || parent == from)
1417	goto out;
1418
1419	child = parent;
1420	parent = parent->parent;
1421	if (parent)
1422	goto up;
1423	out:
1424	return ret;
1425	}
1426
1427	int tg_nop(struct task_group *tg, void *data)
1428	{
1429	return 0;
1430	}
1431	#endif
1432
1433	void set_load_weight(struct task_struct *p, bool update_load)
1434	{
1435	int prio = p->static_prio - MAX_RT_PRIO;
1436	struct load_weight lw;
1437
1438	if (task_has_idle_policy(p)) {
1439	lw.weight = scale_load(WEIGHT_IDLEPRIO);
1440	lw.inv_weight = WMULT_IDLEPRIO;
1441	} else {
1442	lw.weight = scale_load(sched_prio_to_weight[prio]);
1443	lw.inv_weight = sched_prio_to_wmult[prio];
1444	}
1445
1446	/*
1447	* SCHED_OTHER tasks have to update their load when changing their
1448	* weight
1449	*/
1450	if (update_load && p->sched_class->reweight_task)
1451	p->sched_class->reweight_task(task_rq(p), p, &lw);
1452	else
1453	p->se.load = lw;
1454	}
1455
1456	#ifdef CONFIG_UCLAMP_TASK
1457	/*
1458	* Serializes updates of utilization clamp values
1459	*
1460	* The (slow-path) user-space triggers utilization clamp value updates which
1461	* can require updates on (fast-path) scheduler's data structures used to
1462	* support enqueue/dequeue operations.
1463	* While the per-CPU rq lock protects fast-path update operations, user-space
1464	* requests are serialized using a mutex to reduce the risk of conflicting
1465	* updates or API abuses.
1466	*/
1467	static __maybe_unused DEFINE_MUTEX(uclamp_mutex);
1468
1469	/* Max allowed minimum utilization */
1470	static unsigned int __maybe_unused sysctl_sched_uclamp_util_min = SCHED_CAPACITY_SCALE;
1471
1472	/* Max allowed maximum utilization */
1473	static unsigned int __maybe_unused sysctl_sched_uclamp_util_max = SCHED_CAPACITY_SCALE;
1474
1475	/*
1476	* By default RT tasks run at the maximum performance point/capacity of the
1477	* system. Uclamp enforces this by always setting UCLAMP_MIN of RT tasks to
1478	* SCHED_CAPACITY_SCALE.
1479	*
1480	* This knob allows admins to change the default behavior when uclamp is being
1481	* used. In battery powered devices, particularly, running at the maximum
1482	* capacity and frequency will increase energy consumption and shorten the
1483	* battery life.
1484	*
1485	* This knob only affects RT tasks that their uclamp_se->user_defined == false.
1486	*
1487	* This knob will not override the system default sched_util_clamp_min defined
1488	* above.
1489	*/
1490	unsigned int sysctl_sched_uclamp_util_min_rt_default = SCHED_CAPACITY_SCALE;
1491
1492	/* All clamps are required to be less or equal than these values */
1493	static struct uclamp_se uclamp_default[UCLAMP_CNT];
1494
1495	/*
1496	* This static key is used to reduce the uclamp overhead in the fast path. It
1497	* primarily disables the call to uclamp_rq_{inc, dec}() in
1498	* enqueue/dequeue_task().
1499	*
1500	* This allows users to continue to enable uclamp in their kernel config with
1501	* minimum uclamp overhead in the fast path.
1502	*
1503	* As soon as userspace modifies any of the uclamp knobs, the static key is
1504	* enabled, since we have an actual users that make use of uclamp
1505	* functionality.
1506	*
1507	* The knobs that would enable this static key are:
1508	*
1509	* * A task modifying its uclamp value with sched_setattr().
1510	* * An admin modifying the sysctl_sched_uclamp_{min, max} via procfs.
1511	* * An admin modifying the cgroup cpu.uclamp.{min, max}
1512	*/
1513	DEFINE_STATIC_KEY_FALSE(sched_uclamp_used);
1514
1515	static inline unsigned int
1516	uclamp_idle_value(struct rq *rq, enum uclamp_id clamp_id,
1517	unsigned int clamp_value)
1518	{
1519	/*
1520	* Avoid blocked utilization pushing up the frequency when we go
1521	* idle (which drops the max-clamp) by retaining the last known
1522	* max-clamp.
1523	*/
1524	if (clamp_id == UCLAMP_MAX) {
1525	rq->uclamp_flags |= UCLAMP_FLAG_IDLE;
1526	return clamp_value;
1527	}
1528
1529	return uclamp_none(clamp_id: UCLAMP_MIN);
1530	}
1531
1532	static inline void uclamp_idle_reset(struct rq *rq, enum uclamp_id clamp_id,
1533	unsigned int clamp_value)
1534	{
1535	/* Reset max-clamp retention only on idle exit */
1536	if (!(rq->uclamp_flags & UCLAMP_FLAG_IDLE))
1537	return;
1538
1539	uclamp_rq_set(rq, clamp_id, value: clamp_value);
1540	}
1541
1542	static inline
1543	unsigned int uclamp_rq_max_value(struct rq *rq, enum uclamp_id clamp_id,
1544	unsigned int clamp_value)
1545	{
1546	struct uclamp_bucket *bucket = rq->uclamp[clamp_id].bucket;
1547	int bucket_id = UCLAMP_BUCKETS - 1;
1548
1549	/*
1550	* Since both min and max clamps are max aggregated, find the
1551	* top most bucket with tasks in.
1552	*/
1553	for ( ; bucket_id >= 0; bucket_id--) {
1554	if (!bucket[bucket_id].tasks)
1555	continue;
1556	return bucket[bucket_id].value;
1557	}
1558
1559	/* No tasks -- default clamp values */
1560	return uclamp_idle_value(rq, clamp_id, clamp_value);
1561	}
1562
1563	static void __uclamp_update_util_min_rt_default(struct task_struct *p)
1564	{
1565	unsigned int default_util_min;
1566	struct uclamp_se *uc_se;
1567
1568	lockdep_assert_held(&p->pi_lock);
1569
1570	uc_se = &p->uclamp_req[UCLAMP_MIN];
1571
1572	/* Only sync if user didn't override the default */
1573	if (uc_se->user_defined)
1574	return;
1575
1576	default_util_min = sysctl_sched_uclamp_util_min_rt_default;
1577	uclamp_se_set(uc_se, value: default_util_min, user_defined: false);
1578	}
1579
1580	static void uclamp_update_util_min_rt_default(struct task_struct *p)
1581	{
1582	if (!rt_task(p))
1583	return;
1584
1585	/* Protect updates to p->uclamp_* */
1586	guard(task_rq_lock)(l: p);
1587	__uclamp_update_util_min_rt_default(p);
1588	}
1589
1590	static inline struct uclamp_se
1591	uclamp_tg_restrict(struct task_struct *p, enum uclamp_id clamp_id)
1592	{
1593	/* Copy by value as we could modify it */
1594	struct uclamp_se uc_req = p->uclamp_req[clamp_id];
1595	#ifdef CONFIG_UCLAMP_TASK_GROUP
1596	unsigned int tg_min, tg_max, value;
1597
1598	/*
1599	* Tasks in autogroups or root task group will be
1600	* restricted by system defaults.
1601	*/
1602	if (task_group_is_autogroup(tg: task_group(p)))
1603	return uc_req;
1604	if (task_group(p) == &root_task_group)
1605	return uc_req;
1606
1607	tg_min = task_group(p)->uclamp[UCLAMP_MIN].value;
1608	tg_max = task_group(p)->uclamp[UCLAMP_MAX].value;
1609	value = uc_req.value;
1610	value = clamp(value, tg_min, tg_max);
1611	uclamp_se_set(uc_se: &uc_req, value, user_defined: false);
1612	#endif
1613
1614	return uc_req;
1615	}
1616
1617	/*
1618	* The effective clamp bucket index of a task depends on, by increasing
1619	* priority:
1620	* - the task specific clamp value, when explicitly requested from userspace
1621	* - the task group effective clamp value, for tasks not either in the root
1622	* group or in an autogroup
1623	* - the system default clamp value, defined by the sysadmin
1624	*/
1625	static inline struct uclamp_se
1626	uclamp_eff_get(struct task_struct *p, enum uclamp_id clamp_id)
1627	{
1628	struct uclamp_se uc_req = uclamp_tg_restrict(p, clamp_id);
1629	struct uclamp_se uc_max = uclamp_default[clamp_id];
1630
1631	/* System default restrictions always apply */
1632	if (unlikely(uc_req.value > uc_max.value))
1633	return uc_max;
1634
1635	return uc_req;
1636	}
1637
1638	unsigned long uclamp_eff_value(struct task_struct *p, enum uclamp_id clamp_id)
1639	{
1640	struct uclamp_se uc_eff;
1641
1642	/* Task currently refcounted: use back-annotated (effective) value */
1643	if (p->uclamp[clamp_id].active)
1644	return (unsigned long)p->uclamp[clamp_id].value;
1645
1646	uc_eff = uclamp_eff_get(p, clamp_id);
1647
1648	return (unsigned long)uc_eff.value;
1649	}
1650
1651	/*
1652	* When a task is enqueued on a rq, the clamp bucket currently defined by the
1653	* task's uclamp::bucket_id is refcounted on that rq. This also immediately
1654	* updates the rq's clamp value if required.
1655	*
1656	* Tasks can have a task-specific value requested from user-space, track
1657	* within each bucket the maximum value for tasks refcounted in it.
1658	* This "local max aggregation" allows to track the exact "requested" value
1659	* for each bucket when all its RUNNABLE tasks require the same clamp.
1660	*/
1661	static inline void uclamp_rq_inc_id(struct rq *rq, struct task_struct *p,
1662	enum uclamp_id clamp_id)
1663	{
1664	struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
1665	struct uclamp_se *uc_se = &p->uclamp[clamp_id];
1666	struct uclamp_bucket *bucket;
1667
1668	lockdep_assert_rq_held(rq);
1669
1670	/* Update task effective clamp */
1671	p->uclamp[clamp_id] = uclamp_eff_get(p, clamp_id);
1672
1673	bucket = &uc_rq->bucket[uc_se->bucket_id];
1674	bucket->tasks++;
1675	uc_se->active = true;
1676
1677	uclamp_idle_reset(rq, clamp_id, clamp_value: uc_se->value);
1678
1679	/*
1680	* Local max aggregation: rq buckets always track the max
1681	* "requested" clamp value of its RUNNABLE tasks.
1682	*/
1683	if (bucket->tasks == 1 || uc_se->value > bucket->value)
1684	bucket->value = uc_se->value;
1685
1686	if (uc_se->value > uclamp_rq_get(rq, clamp_id))
1687	uclamp_rq_set(rq, clamp_id, value: uc_se->value);
1688	}
1689
1690	/*
1691	* When a task is dequeued from a rq, the clamp bucket refcounted by the task
1692	* is released. If this is the last task reference counting the rq's max
1693	* active clamp value, then the rq's clamp value is updated.
1694	*
1695	* Both refcounted tasks and rq's cached clamp values are expected to be
1696	* always valid. If it's detected they are not, as defensive programming,
1697	* enforce the expected state and warn.
1698	*/
1699	static inline void uclamp_rq_dec_id(struct rq *rq, struct task_struct *p,
1700	enum uclamp_id clamp_id)
1701	{
1702	struct uclamp_rq *uc_rq = &rq->uclamp[clamp_id];
1703	struct uclamp_se *uc_se = &p->uclamp[clamp_id];
1704	struct uclamp_bucket *bucket;
1705	unsigned int bkt_clamp;
1706	unsigned int rq_clamp;
1707
1708	lockdep_assert_rq_held(rq);
1709
1710	/*
1711	* If sched_uclamp_used was enabled after task @p was enqueued,
1712	* we could end up with unbalanced call to uclamp_rq_dec_id().
1713	*
1714	* In this case the uc_se->active flag should be false since no uclamp
1715	* accounting was performed at enqueue time and we can just return
1716	* here.
1717	*
1718	* Need to be careful of the following enqueue/dequeue ordering
1719	* problem too
1720	*
1721	* enqueue(taskA)
1722	* // sched_uclamp_used gets enabled
1723	* enqueue(taskB)
1724	* dequeue(taskA)
1725	* // Must not decrement bucket->tasks here
1726	* dequeue(taskB)
1727	*
1728	* where we could end up with stale data in uc_se and
1729	* bucket[uc_se->bucket_id].
1730	*
1731	* The following check here eliminates the possibility of such race.
1732	*/
1733	if (unlikely(!uc_se->active))
1734	return;
1735
1736	bucket = &uc_rq->bucket[uc_se->bucket_id];
1737
1738	WARN_ON_ONCE(!bucket->tasks);
1739	if (likely(bucket->tasks))
1740	bucket->tasks--;
1741
1742	uc_se->active = false;
1743
1744	/*
1745	* Keep "local max aggregation" simple and accept to (possibly)
1746	* overboost some RUNNABLE tasks in the same bucket.
1747	* The rq clamp bucket value is reset to its base value whenever
1748	* there are no more RUNNABLE tasks refcounting it.
1749	*/
1750	if (likely(bucket->tasks))
1751	return;
1752
1753	rq_clamp = uclamp_rq_get(rq, clamp_id);
1754	/*
1755	* Defensive programming: this should never happen. If it happens,
1756	* e.g. due to future modification, warn and fix up the expected value.
1757	*/
1758	WARN_ON_ONCE(bucket->value > rq_clamp);
1759	if (bucket->value >= rq_clamp) {
1760	bkt_clamp = uclamp_rq_max_value(rq, clamp_id, clamp_value: uc_se->value);
1761	uclamp_rq_set(rq, clamp_id, value: bkt_clamp);
1762	}
1763	}
1764
1765	static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p, int flags)
1766	{
1767	enum uclamp_id clamp_id;
1768
1769	/*
1770	* Avoid any overhead until uclamp is actually used by the userspace.
1771	*
1772	* The condition is constructed such that a NOP is generated when
1773	* sched_uclamp_used is disabled.
1774	*/
1775	if (!uclamp_is_used())
1776	return;
1777
1778	if (unlikely(!p->sched_class->uclamp_enabled))
1779	return;
1780
1781	/* Only inc the delayed task which being woken up. */
1782	if (p->se.sched_delayed && !(flags & ENQUEUE_DELAYED))
1783	return;
1784
1785	for_each_clamp_id(clamp_id)
1786	uclamp_rq_inc_id(rq, p, clamp_id);
1787
1788	/* Reset clamp idle holding when there is one RUNNABLE task */
1789	if (rq->uclamp_flags & UCLAMP_FLAG_IDLE)
1790	rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE;
1791	}
1792
1793	static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p)
1794	{
1795	enum uclamp_id clamp_id;
1796
1797	/*
1798	* Avoid any overhead until uclamp is actually used by the userspace.
1799	*
1800	* The condition is constructed such that a NOP is generated when
1801	* sched_uclamp_used is disabled.
1802	*/
1803	if (!uclamp_is_used())
1804	return;
1805
1806	if (unlikely(!p->sched_class->uclamp_enabled))
1807	return;
1808
1809	if (p->se.sched_delayed)
1810	return;
1811
1812	for_each_clamp_id(clamp_id)
1813	uclamp_rq_dec_id(rq, p, clamp_id);
1814	}
1815
1816	static inline void uclamp_rq_reinc_id(struct rq *rq, struct task_struct *p,
1817	enum uclamp_id clamp_id)
1818	{
1819	if (!p->uclamp[clamp_id].active)
1820	return;
1821
1822	uclamp_rq_dec_id(rq, p, clamp_id);
1823	uclamp_rq_inc_id(rq, p, clamp_id);
1824
1825	/*
1826	* Make sure to clear the idle flag if we've transiently reached 0
1827	* active tasks on rq.
1828	*/
1829	if (clamp_id == UCLAMP_MAX && (rq->uclamp_flags & UCLAMP_FLAG_IDLE))
1830	rq->uclamp_flags &= ~UCLAMP_FLAG_IDLE;
1831	}
1832
1833	static inline void
1834	uclamp_update_active(struct task_struct *p)
1835	{
1836	enum uclamp_id clamp_id;
1837	struct rq_flags rf;
1838	struct rq *rq;
1839
1840	/*
1841	* Lock the task and the rq where the task is (or was) queued.
1842	*
1843	* We might lock the (previous) rq of a !RUNNABLE task, but that's the
1844	* price to pay to safely serialize util_{min,max} updates with
1845	* enqueues, dequeues and migration operations.
1846	* This is the same locking schema used by __set_cpus_allowed_ptr().
1847	*/
1848	rq = task_rq_lock(p, rf: &rf);
1849
1850	/*
1851	* Setting the clamp bucket is serialized by task_rq_lock().
1852	* If the task is not yet RUNNABLE and its task_struct is not
1853	* affecting a valid clamp bucket, the next time it's enqueued,
1854	* it will already see the updated clamp bucket value.
1855	*/
1856	for_each_clamp_id(clamp_id)
1857	uclamp_rq_reinc_id(rq, p, clamp_id);
1858
1859	task_rq_unlock(rq, p, rf: &rf);
1860	}
1861
1862	#ifdef CONFIG_UCLAMP_TASK_GROUP
1863	static inline void
1864	uclamp_update_active_tasks(struct cgroup_subsys_state *css)
1865	{
1866	struct css_task_iter it;
1867	struct task_struct *p;
1868
1869	css_task_iter_start(css, flags: 0, it: &it);
1870	while ((p = css_task_iter_next(it: &it)))
1871	uclamp_update_active(p);
1872	css_task_iter_end(it: &it);
1873	}
1874
1875	static void cpu_util_update_eff(struct cgroup_subsys_state *css);
1876	#endif
1877
1878	#ifdef CONFIG_SYSCTL
1879	#ifdef CONFIG_UCLAMP_TASK_GROUP
1880	static void uclamp_update_root_tg(void)
1881	{
1882	struct task_group *tg = &root_task_group;
1883
1884	uclamp_se_set(uc_se: &tg->uclamp_req[UCLAMP_MIN],
1885	value: sysctl_sched_uclamp_util_min, user_defined: false);
1886	uclamp_se_set(uc_se: &tg->uclamp_req[UCLAMP_MAX],
1887	value: sysctl_sched_uclamp_util_max, user_defined: false);
1888
1889	guard(rcu)();
1890	cpu_util_update_eff(css: &root_task_group.css);
1891	}
1892	#else
1893	static void uclamp_update_root_tg(void) { }
1894	#endif
1895
1896	static void uclamp_sync_util_min_rt_default(void)
1897	{
1898	struct task_struct *g, *p;
1899
1900	/*
1901	* copy_process() sysctl_uclamp
1902	* uclamp_min_rt = X;
1903	* write_lock(&tasklist_lock) read_lock(&tasklist_lock)
1904	* // link thread smp_mb__after_spinlock()
1905	* write_unlock(&tasklist_lock) read_unlock(&tasklist_lock);
1906	* sched_post_fork() for_each_process_thread()
1907	* __uclamp_sync_rt() __uclamp_sync_rt()
1908	*
1909	* Ensures that either sched_post_fork() will observe the new
1910	* uclamp_min_rt or for_each_process_thread() will observe the new
1911	* task.
1912	*/
1913	read_lock(&tasklist_lock);
1914	smp_mb__after_spinlock();
1915	read_unlock(&tasklist_lock);
1916
1917	guard(rcu)();
1918	for_each_process_thread(g, p)
1919	uclamp_update_util_min_rt_default(p);
1920	}
1921
1922	static int sysctl_sched_uclamp_handler(const struct ctl_table *table, int write,
1923	void *buffer, size_t *lenp, loff_t *ppos)
1924	{
1925	bool update_root_tg = false;
1926	int old_min, old_max, old_min_rt;
1927	int result;
1928
1929	guard(mutex)(T: &uclamp_mutex);
1930
1931	old_min = sysctl_sched_uclamp_util_min;
1932	old_max = sysctl_sched_uclamp_util_max;
1933	old_min_rt = sysctl_sched_uclamp_util_min_rt_default;
1934
1935	result = proc_dointvec(table, write, buffer, lenp, ppos);
1936	if (result)
1937	goto undo;
1938	if (!write)
1939	return 0;
1940
1941	if (sysctl_sched_uclamp_util_min > sysctl_sched_uclamp_util_max ||
1942	sysctl_sched_uclamp_util_max > SCHED_CAPACITY_SCALE ||
1943	sysctl_sched_uclamp_util_min_rt_default > SCHED_CAPACITY_SCALE) {
1944
1945	result = -EINVAL;
1946	goto undo;
1947	}
1948
1949	if (old_min != sysctl_sched_uclamp_util_min) {
1950	uclamp_se_set(uc_se: &uclamp_default[UCLAMP_MIN],
1951	value: sysctl_sched_uclamp_util_min, user_defined: false);
1952	update_root_tg = true;
1953	}
1954	if (old_max != sysctl_sched_uclamp_util_max) {
1955	uclamp_se_set(uc_se: &uclamp_default[UCLAMP_MAX],
1956	value: sysctl_sched_uclamp_util_max, user_defined: false);
1957	update_root_tg = true;
1958	}
1959
1960	if (update_root_tg) {
1961	sched_uclamp_enable();
1962	uclamp_update_root_tg();
1963	}
1964
1965	if (old_min_rt != sysctl_sched_uclamp_util_min_rt_default) {
1966	sched_uclamp_enable();
1967	uclamp_sync_util_min_rt_default();
1968	}
1969
1970	/*
1971	* We update all RUNNABLE tasks only when task groups are in use.
1972	* Otherwise, keep it simple and do just a lazy update at each next
1973	* task enqueue time.
1974	*/
1975	return 0;
1976
1977	undo:
1978	sysctl_sched_uclamp_util_min = old_min;
1979	sysctl_sched_uclamp_util_max = old_max;
1980	sysctl_sched_uclamp_util_min_rt_default = old_min_rt;
1981	return result;
1982	}
1983	#endif /* CONFIG_SYSCTL */
1984
1985	static void uclamp_fork(struct task_struct *p)
1986	{
1987	enum uclamp_id clamp_id;
1988
1989	/*
1990	* We don't need to hold task_rq_lock() when updating p->uclamp_* here
1991	* as the task is still at its early fork stages.
1992	*/
1993	for_each_clamp_id(clamp_id)
1994	p->uclamp[clamp_id].active = false;
1995
1996	if (likely(!p->sched_reset_on_fork))
1997	return;
1998
1999	for_each_clamp_id(clamp_id) {
2000	uclamp_se_set(uc_se: &p->uclamp_req[clamp_id],
2001	value: uclamp_none(clamp_id), user_defined: false);
2002	}
2003	}
2004
2005	static void uclamp_post_fork(struct task_struct *p)
2006	{
2007	uclamp_update_util_min_rt_default(p);
2008	}
2009
2010	static void __init init_uclamp_rq(struct rq *rq)
2011	{
2012	enum uclamp_id clamp_id;
2013	struct uclamp_rq *uc_rq = rq->uclamp;
2014
2015	for_each_clamp_id(clamp_id) {
2016	uc_rq[clamp_id] = (struct uclamp_rq) {
2017	.value = uclamp_none(clamp_id)
2018	};
2019	}
2020
2021	rq->uclamp_flags = UCLAMP_FLAG_IDLE;
2022	}
2023
2024	static void __init init_uclamp(void)
2025	{
2026	struct uclamp_se uc_max = {};
2027	enum uclamp_id clamp_id;
2028	int cpu;
2029
2030	for_each_possible_cpu(cpu)
2031	init_uclamp_rq(cpu_rq(cpu));
2032
2033	for_each_clamp_id(clamp_id) {
2034	uclamp_se_set(uc_se: &init_task.uclamp_req[clamp_id],
2035	value: uclamp_none(clamp_id), user_defined: false);
2036	}
2037
2038	/* System defaults allow max clamp values for both indexes */
2039	uclamp_se_set(uc_se: &uc_max, value: uclamp_none(clamp_id: UCLAMP_MAX), user_defined: false);
2040	for_each_clamp_id(clamp_id) {
2041	uclamp_default[clamp_id] = uc_max;
2042	#ifdef CONFIG_UCLAMP_TASK_GROUP
2043	root_task_group.uclamp_req[clamp_id] = uc_max;
2044	root_task_group.uclamp[clamp_id] = uc_max;
2045	#endif
2046	}
2047	}
2048
2049	#else /* !CONFIG_UCLAMP_TASK: */
2050	static inline void uclamp_rq_inc(struct rq *rq, struct task_struct *p, int flags) { }
2051	static inline void uclamp_rq_dec(struct rq *rq, struct task_struct *p) { }
2052	static inline void uclamp_fork(struct task_struct *p) { }
2053	static inline void uclamp_post_fork(struct task_struct *p) { }
2054	static inline void init_uclamp(void) { }
2055	#endif /* !CONFIG_UCLAMP_TASK */
2056
2057	bool sched_task_on_rq(struct task_struct *p)
2058	{
2059	return task_on_rq_queued(p);
2060	}
2061
2062	unsigned long get_wchan(struct task_struct *p)
2063	{
2064	unsigned long ip = 0;
2065	unsigned int state;
2066
2067	if (!p || p == current)
2068	return 0;
2069
2070	/* Only get wchan if task is blocked and we can keep it that way. */
2071	raw_spin_lock_irq(&p->pi_lock);
2072	state = READ_ONCE(p->__state);
2073	smp_rmb(); /* see try_to_wake_up() */
2074	if (state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq)
2075	ip = __get_wchan(p);
2076	raw_spin_unlock_irq(&p->pi_lock);
2077
2078	return ip;
2079	}
2080
2081	void enqueue_task(struct rq *rq, struct task_struct *p, int flags)
2082	{
2083	if (!(flags & ENQUEUE_NOCLOCK))
2084	update_rq_clock(rq);
2085
2086	/*
2087	* Can be before ->enqueue_task() because uclamp considers the
2088	* ENQUEUE_DELAYED task before its ->sched_delayed gets cleared
2089	* in ->enqueue_task().
2090	*/
2091	uclamp_rq_inc(rq, p, flags);
2092
2093	rq->queue_mask |= p->sched_class->queue_mask;
2094	p->sched_class->enqueue_task(rq, p, flags);
2095
2096	psi_enqueue(p, flags);
2097
2098	if (!(flags & ENQUEUE_RESTORE))
2099	sched_info_enqueue(rq, t: p);
2100
2101	if (sched_core_enabled(rq))
2102	sched_core_enqueue(rq, p);
2103	}
2104
2105	/*
2106	* Must only return false when DEQUEUE_SLEEP.
2107	*/
2108	inline bool dequeue_task(struct rq *rq, struct task_struct *p, int flags)
2109	{
2110	if (sched_core_enabled(rq))
2111	sched_core_dequeue(rq, p, flags);
2112
2113	if (!(flags & DEQUEUE_NOCLOCK))
2114	update_rq_clock(rq);
2115
2116	if (!(flags & DEQUEUE_SAVE))
2117	sched_info_dequeue(rq, t: p);
2118
2119	psi_dequeue(p, flags);
2120
2121	/*
2122	* Must be before ->dequeue_task() because ->dequeue_task() can 'fail'
2123	* and mark the task ->sched_delayed.
2124	*/
2125	uclamp_rq_dec(rq, p);
2126	rq->queue_mask |= p->sched_class->queue_mask;
2127	return p->sched_class->dequeue_task(rq, p, flags);
2128	}
2129
2130	void activate_task(struct rq *rq, struct task_struct *p, int flags)
2131	{
2132	if (task_on_rq_migrating(p))
2133	flags |= ENQUEUE_MIGRATED;
2134
2135	enqueue_task(rq, p, flags);
2136
2137	WRITE_ONCE(p->on_rq, TASK_ON_RQ_QUEUED);
2138	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
2139	}
2140
2141	void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
2142	{
2143	WARN_ON_ONCE(flags & DEQUEUE_SLEEP);
2144
2145	WRITE_ONCE(p->on_rq, TASK_ON_RQ_MIGRATING);
2146	ASSERT_EXCLUSIVE_WRITER(p->on_rq);
2147
2148	/*
2149	* Code explicitly relies on TASK_ON_RQ_MIGRATING begin set *before*
2150	* dequeue_task() and cleared *after* enqueue_task().
2151	*/
2152
2153	dequeue_task(rq, p, flags);
2154	}
2155
2156	static void block_task(struct rq *rq, struct task_struct *p, int flags)
2157	{
2158	if (dequeue_task(rq, p, DEQUEUE_SLEEP | flags))
2159	__block_task(rq, p);
2160	}
2161
2162	/**
2163	* task_curr - is this task currently executing on a CPU?
2164	* @p: the task in question.
2165	*
2166	* Return: 1 if the task is currently executing. 0 otherwise.
2167	*/
2168	inline int task_curr(const struct task_struct *p)
2169	{
2170	return cpu_curr(task_cpu(p)) == p;
2171	}
2172
2173	void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags)
2174	{
2175	struct task_struct *donor = rq->donor;
2176
2177	if (p->sched_class == donor->sched_class)
2178	donor->sched_class->wakeup_preempt(rq, p, flags);
2179	else if (sched_class_above(p->sched_class, donor->sched_class))
2180	resched_curr(rq);
2181
2182	/*
2183	* A queue event has occurred, and we're going to schedule. In
2184	* this case, we can save a useless back to back clock update.
2185	*/
2186	if (task_on_rq_queued(p: donor) && test_tsk_need_resched(tsk: rq->curr))
2187	rq_clock_skip_update(rq);
2188	}
2189
2190	static __always_inline
2191	int __task_state_match(struct task_struct *p, unsigned int state)
2192	{
2193	if (READ_ONCE(p->__state) & state)
2194	return 1;
2195
2196	if (READ_ONCE(p->saved_state) & state)
2197	return -1;
2198
2199	return 0;
2200	}
2201
2202	static __always_inline
2203	int task_state_match(struct task_struct *p, unsigned int state)
2204	{
2205	/*
2206	* Serialize against current_save_and_set_rtlock_wait_state(),
2207	* current_restore_rtlock_saved_state(), and __refrigerator().
2208	*/
2209	guard(raw_spinlock_irq)(l: &p->pi_lock);
2210	return __task_state_match(p, state);
2211	}
2212
2213	/*
2214	* wait_task_inactive - wait for a thread to unschedule.
2215	*
2216	* Wait for the thread to block in any of the states set in @match_state.
2217	* If it changes, i.e. @p might have woken up, then return zero. When we
2218	* succeed in waiting for @p to be off its CPU, we return a positive number
2219	* (its total switch count). If a second call a short while later returns the
2220	* same number, the caller can be sure that @p has remained unscheduled the
2221	* whole time.
2222	*
2223	* The caller must ensure that the task *will* unschedule sometime soon,
2224	* else this function might spin for a *long* time. This function can't
2225	* be called with interrupts off, or it may introduce deadlock with
2226	* smp_call_function() if an IPI is sent by the same process we are
2227	* waiting to become inactive.
2228	*/
2229	unsigned long wait_task_inactive(struct task_struct *p, unsigned int match_state)
2230	{
2231	int running, queued, match;
2232	struct rq_flags rf;
2233	unsigned long ncsw;
2234	struct rq *rq;
2235
2236	for (;;) {
2237	/*
2238	* We do the initial early heuristics without holding
2239	* any task-queue locks at all. We'll only try to get
2240	* the runqueue lock when things look like they will
2241	* work out!
2242	*/
2243	rq = task_rq(p);
2244
2245	/*
2246	* If the task is actively running on another CPU
2247	* still, just relax and busy-wait without holding
2248	* any locks.
2249	*
2250	* NOTE! Since we don't hold any locks, it's not
2251	* even sure that "rq" stays as the right runqueue!
2252	* But we don't care, since "task_on_cpu()" will
2253	* return false if the runqueue has changed and p
2254	* is actually now running somewhere else!
2255	*/
2256	while (task_on_cpu(rq, p)) {
2257	if (!task_state_match(p, state: match_state))
2258	return 0;
2259	cpu_relax();
2260	}
2261
2262	/*
2263	* Ok, time to look more closely! We need the rq
2264	* lock now, to be *sure*. If we're wrong, we'll
2265	* just go back and repeat.
2266	*/
2267	rq = task_rq_lock(p, rf: &rf);
2268	/*
2269	* If task is sched_delayed, force dequeue it, to avoid always
2270	* hitting the tick timeout in the queued case
2271	*/
2272	if (p->se.sched_delayed)
2273	dequeue_task(rq, p, DEQUEUE_SLEEP | DEQUEUE_DELAYED);
2274	trace_sched_wait_task(p);
2275	running = task_on_cpu(rq, p);
2276	queued = task_on_rq_queued(p);
2277	ncsw = 0;
2278	if ((match = __task_state_match(p, state: match_state))) {
2279	/*
2280	* When matching on p->saved_state, consider this task
2281	* still queued so it will wait.
2282	*/
2283	if (match < 0)
2284	queued = 1;
2285	ncsw = p->nvcsw | LONG_MIN; /* sets MSB */
2286	}
2287	task_rq_unlock(rq, p, rf: &rf);
2288
2289	/*
2290	* If it changed from the expected state, bail out now.
2291	*/
2292	if (unlikely(!ncsw))
2293	break;
2294
2295	/*
2296	* Was it really running after all now that we
2297	* checked with the proper locks actually held?
2298	*
2299	* Oops. Go back and try again..
2300	*/
2301	if (unlikely(running)) {
2302	cpu_relax();
2303	continue;
2304	}
2305
2306	/*
2307	* It's not enough that it's not actively running,
2308	* it must be off the runqueue _entirely_, and not
2309	* preempted!
2310	*
2311	* So if it was still runnable (but just not actively
2312	* running right now), it's preempted, and we should
2313	* yield - it could be a while.
2314	*/
2315	if (unlikely(queued)) {
2316	ktime_t to = NSEC_PER_SEC / HZ;
2317
2318	set_current_state(TASK_UNINTERRUPTIBLE);
2319	schedule_hrtimeout(expires: &to, mode: HRTIMER_MODE_REL_HARD);
2320	continue;
2321	}
2322
2323	/*
2324	* Ahh, all good. It wasn't running, and it wasn't
2325	* runnable, which means that it will never become
2326	* running in the future either. We're all done!
2327	*/
2328	break;
2329	}
2330
2331	return ncsw;
2332	}
2333
2334	static void
2335	do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx);
2336
2337	static void migrate_disable_switch(struct rq *rq, struct task_struct *p)
2338	{
2339	struct affinity_context ac = {
2340	.new_mask = cpumask_of(rq->cpu),
2341	.flags = SCA_MIGRATE_DISABLE,
2342	};
2343
2344	if (likely(!p->migration_disabled))
2345	return;
2346
2347	if (p->cpus_ptr != &p->cpus_mask)
2348	return;
2349
2350	scoped_guard (task_rq_lock, p)
2351	do_set_cpus_allowed(p, ctx: &ac);
2352	}
2353
2354	void ___migrate_enable(void)
2355	{
2356	struct task_struct *p = current;
2357	struct affinity_context ac = {
2358	.new_mask = &p->cpus_mask,
2359	.flags = SCA_MIGRATE_ENABLE,
2360	};
2361
2362	__set_cpus_allowed_ptr(p, ctx: &ac);
2363	}
2364	EXPORT_SYMBOL_GPL(___migrate_enable);
2365
2366	void migrate_disable(void)
2367	{
2368	__migrate_disable();
2369	}
2370	EXPORT_SYMBOL_GPL(migrate_disable);
2371
2372	void migrate_enable(void)
2373	{
2374	__migrate_enable();
2375	}
2376	EXPORT_SYMBOL_GPL(migrate_enable);
2377
2378	static inline bool rq_has_pinned_tasks(struct rq *rq)
2379	{
2380	return rq->nr_pinned;
2381	}
2382
2383	/*
2384	* Per-CPU kthreads are allowed to run on !active && online CPUs, see
2385	* __set_cpus_allowed_ptr() and select_fallback_rq().
2386	*/
2387	static inline bool is_cpu_allowed(struct task_struct *p, int cpu)
2388	{
2389	/* When not in the task's cpumask, no point in looking further. */
2390	if (!task_allowed_on_cpu(p, cpu))
2391	return false;
2392
2393	/* migrate_disabled() must be allowed to finish. */
2394	if (is_migration_disabled(p))
2395	return cpu_online(cpu);
2396
2397	/* Non kernel threads are not allowed during either online or offline. */
2398	if (!(p->flags & PF_KTHREAD))
2399	return cpu_active(cpu);
2400
2401	/* KTHREAD_IS_PER_CPU is always allowed. */
2402	if (kthread_is_per_cpu(k: p))
2403	return cpu_online(cpu);
2404
2405	/* Regular kernel threads don't get to stay during offline. */
2406	if (cpu_dying(cpu))
2407	return false;
2408
2409	/* But are allowed during online. */
2410	return cpu_online(cpu);
2411	}
2412
2413	/*
2414	* This is how migration works:
2415	*
2416	* 1) we invoke migration_cpu_stop() on the target CPU using
2417	* stop_one_cpu().
2418	* 2) stopper starts to run (implicitly forcing the migrated thread
2419	* off the CPU)
2420	* 3) it checks whether the migrated task is still in the wrong runqueue.
2421	* 4) if it's in the wrong runqueue then the migration thread removes
2422	* it and puts it into the right queue.
2423	* 5) stopper completes and stop_one_cpu() returns and the migration
2424	* is done.
2425	*/
2426
2427	/*
2428	* move_queued_task - move a queued task to new rq.
2429	*
2430	* Returns (locked) new rq. Old rq's lock is released.
2431	*/
2432	static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf,
2433	struct task_struct *p, int new_cpu)
2434	{
2435	lockdep_assert_rq_held(rq);
2436
2437	deactivate_task(rq, p, DEQUEUE_NOCLOCK);
2438	set_task_cpu(p, cpu: new_cpu);
2439	rq_unlock(rq, rf);
2440
2441	rq = cpu_rq(new_cpu);
2442
2443	rq_lock(rq, rf);
2444	WARN_ON_ONCE(task_cpu(p) != new_cpu);
2445	activate_task(rq, p, flags: 0);
2446	wakeup_preempt(rq, p, flags: 0);
2447
2448	return rq;
2449	}
2450
2451	struct migration_arg {
2452	struct task_struct *task;
2453	int dest_cpu;
2454	struct set_affinity_pending *pending;
2455	};
2456
2457	/*
2458	* @refs: number of wait_for_completion()
2459	* @stop_pending: is @stop_work in use
2460	*/
2461	struct set_affinity_pending {
2462	refcount_t refs;
2463	unsigned int stop_pending;
2464	struct completion done;
2465	struct cpu_stop_work stop_work;
2466	struct migration_arg arg;
2467	};
2468
2469	/*
2470	* Move (not current) task off this CPU, onto the destination CPU. We're doing
2471	* this because either it can't run here any more (set_cpus_allowed()
2472	* away from this CPU, or CPU going down), or because we're
2473	* attempting to rebalance this task on exec (sched_exec).
2474	*
2475	* So we race with normal scheduler movements, but that's OK, as long
2476	* as the task is no longer on this CPU.
2477	*/
2478	static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf,
2479	struct task_struct *p, int dest_cpu)
2480	{
2481	/* Affinity changed (again). */
2482	if (!is_cpu_allowed(p, cpu: dest_cpu))
2483	return rq;
2484
2485	rq = move_queued_task(rq, rf, p, new_cpu: dest_cpu);
2486
2487	return rq;
2488	}
2489
2490	/*
2491	* migration_cpu_stop - this will be executed by a high-prio stopper thread
2492	* and performs thread migration by bumping thread off CPU then
2493	* 'pushing' onto another runqueue.
2494	*/
2495	static int migration_cpu_stop(void *data)
2496	{
2497	struct migration_arg *arg = data;
2498	struct set_affinity_pending *pending = arg->pending;
2499	struct task_struct *p = arg->task;
2500	struct rq *rq = this_rq();
2501	bool complete = false;
2502	struct rq_flags rf;
2503
2504	/*
2505	* The original target CPU might have gone down and we might
2506	* be on another CPU but it doesn't matter.
2507	*/
2508	local_irq_save(rf.flags);
2509	/*
2510	* We need to explicitly wake pending tasks before running
2511	* __migrate_task() such that we will not miss enforcing cpus_ptr
2512	* during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test.
2513	*/
2514	flush_smp_call_function_queue();
2515
2516	raw_spin_lock(&p->pi_lock);
2517	rq_lock(rq, rf: &rf);
2518
2519	/*
2520	* If we were passed a pending, then ->stop_pending was set, thus
2521	* p->migration_pending must have remained stable.
2522	*/
2523	WARN_ON_ONCE(pending && pending != p->migration_pending);
2524
2525	/*
2526	* If task_rq(p) != rq, it cannot be migrated here, because we're
2527	* holding rq->lock, if p->on_rq == 0 it cannot get enqueued because
2528	* we're holding p->pi_lock.
2529	*/
2530	if (task_rq(p) == rq) {
2531	if (is_migration_disabled(p))
2532	goto out;
2533
2534	if (pending) {
2535	p->migration_pending = NULL;
2536	complete = true;
2537
2538	if (cpumask_test_cpu(cpu: task_cpu(p), cpumask: &p->cpus_mask))
2539	goto out;
2540	}
2541
2542	if (task_on_rq_queued(p)) {
2543	update_rq_clock(rq);
2544	rq = __migrate_task(rq, rf: &rf, p, dest_cpu: arg->dest_cpu);
2545	} else {
2546	p->wake_cpu = arg->dest_cpu;
2547	}
2548
2549	/*
2550	* XXX __migrate_task() can fail, at which point we might end
2551	* up running on a dodgy CPU, AFAICT this can only happen
2552	* during CPU hotplug, at which point we'll get pushed out
2553	* anyway, so it's probably not a big deal.
2554	*/
2555
2556	} else if (pending) {
2557	/*
2558	* This happens when we get migrated between migrate_enable()'s
2559	* preempt_enable() and scheduling the stopper task. At that
2560	* point we're a regular task again and not current anymore.
2561	*
2562	* A !PREEMPT kernel has a giant hole here, which makes it far
2563	* more likely.
2564	*/
2565
2566	/*
2567	* The task moved before the stopper got to run. We're holding
2568	* ->pi_lock, so the allowed mask is stable - if it got
2569	* somewhere allowed, we're done.
2570	*/
2571	if (cpumask_test_cpu(cpu: task_cpu(p), cpumask: p->cpus_ptr)) {
2572	p->migration_pending = NULL;
2573	complete = true;
2574	goto out;
2575	}
2576
2577	/*
2578	* When migrate_enable() hits a rq mis-match we can't reliably
2579	* determine is_migration_disabled() and so have to chase after
2580	* it.
2581	*/
2582	WARN_ON_ONCE(!pending->stop_pending);
2583	preempt_disable();
2584	rq_unlock(rq, rf: &rf);
2585	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
2586	stop_one_cpu_nowait(cpu: task_cpu(p), fn: migration_cpu_stop,
2587	arg: &pending->arg, work_buf: &pending->stop_work);
2588	preempt_enable();
2589	return 0;
2590	}
2591	out:
2592	if (pending)
2593	pending->stop_pending = false;
2594	rq_unlock(rq, rf: &rf);
2595	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
2596
2597	if (complete)
2598	complete_all(&pending->done);
2599
2600	return 0;
2601	}
2602
2603	int push_cpu_stop(void *arg)
2604	{
2605	struct rq *lowest_rq = NULL, *rq = this_rq();
2606	struct task_struct *p = arg;
2607
2608	raw_spin_lock_irq(&p->pi_lock);
2609	raw_spin_rq_lock(rq);
2610
2611	if (task_rq(p) != rq)
2612	goto out_unlock;
2613
2614	if (is_migration_disabled(p)) {
2615	p->migration_flags |= MDF_PUSH;
2616	goto out_unlock;
2617	}
2618
2619	p->migration_flags &= ~MDF_PUSH;
2620
2621	if (p->sched_class->find_lock_rq)
2622	lowest_rq = p->sched_class->find_lock_rq(p, rq);
2623
2624	if (!lowest_rq)
2625	goto out_unlock;
2626
2627	// XXX validate p is still the highest prio task
2628	if (task_rq(p) == rq) {
2629	move_queued_task_locked(src_rq: rq, dst_rq: lowest_rq, task: p);
2630	resched_curr(rq: lowest_rq);
2631	}
2632
2633	double_unlock_balance(this_rq: rq, busiest: lowest_rq);
2634
2635	out_unlock:
2636	rq->push_busy = false;
2637	raw_spin_rq_unlock(rq);
2638	raw_spin_unlock_irq(&p->pi_lock);
2639
2640	put_task_struct(t: p);
2641	return 0;
2642	}
2643
2644	static inline void mm_update_cpus_allowed(struct mm_struct *mm, const cpumask_t *affmask);
2645
2646	/*
2647	* sched_class::set_cpus_allowed must do the below, but is not required to
2648	* actually call this function.
2649	*/
2650	void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx)
2651	{
2652	if (ctx->flags & (SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) {
2653	p->cpus_ptr = ctx->new_mask;
2654	return;
2655	}
2656
2657	cpumask_copy(dstp: &p->cpus_mask, srcp: ctx->new_mask);
2658	p->nr_cpus_allowed = cpumask_weight(srcp: ctx->new_mask);
2659	mm_update_cpus_allowed(mm: p->mm, affmask: ctx->new_mask);
2660
2661	/*
2662	* Swap in a new user_cpus_ptr if SCA_USER flag set
2663	*/
2664	if (ctx->flags & SCA_USER)
2665	swap(p->user_cpus_ptr, ctx->user_mask);
2666	}
2667
2668	static void
2669	do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
2670	{
2671	scoped_guard (sched_change, p, DEQUEUE_SAVE)
2672	p->sched_class->set_cpus_allowed(p, ctx);
2673	}
2674
2675	/*
2676	* Used for kthread_bind() and select_fallback_rq(), in both cases the user
2677	* affinity (if any) should be destroyed too.
2678	*/
2679	void set_cpus_allowed_force(struct task_struct *p, const struct cpumask *new_mask)
2680	{
2681	struct affinity_context ac = {
2682	.new_mask = new_mask,
2683	.user_mask = NULL,
2684	.flags = SCA_USER, /* clear the user requested mask */
2685	};
2686	union cpumask_rcuhead {
2687	cpumask_t cpumask;
2688	struct rcu_head rcu;
2689	};
2690
2691	scoped_guard (__task_rq_lock, p)
2692	do_set_cpus_allowed(p, ctx: &ac);
2693
2694	/*
2695	* Because this is called with p->pi_lock held, it is not possible
2696	* to use kfree() here (when PREEMPT_RT=y), therefore punt to using
2697	* kfree_rcu().
2698	*/
2699	kfree_rcu((union cpumask_rcuhead *)ac.user_mask, rcu);
2700	}
2701
2702	int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src,
2703	int node)
2704	{
2705	cpumask_t *user_mask;
2706	unsigned long flags;
2707
2708	/*
2709	* Always clear dst->user_cpus_ptr first as their user_cpus_ptr's
2710	* may differ by now due to racing.
2711	*/
2712	dst->user_cpus_ptr = NULL;
2713
2714	/*
2715	* This check is racy and losing the race is a valid situation.
2716	* It is not worth the extra overhead of taking the pi_lock on
2717	* every fork/clone.
2718	*/
2719	if (data_race(!src->user_cpus_ptr))
2720	return 0;
2721
2722	user_mask = alloc_user_cpus_ptr(node);
2723	if (!user_mask)
2724	return -ENOMEM;
2725
2726	/*
2727	* Use pi_lock to protect content of user_cpus_ptr
2728	*
2729	* Though unlikely, user_cpus_ptr can be reset to NULL by a concurrent
2730	* set_cpus_allowed_force().
2731	*/
2732	raw_spin_lock_irqsave(&src->pi_lock, flags);
2733	if (src->user_cpus_ptr) {
2734	swap(dst->user_cpus_ptr, user_mask);
2735	cpumask_copy(dstp: dst->user_cpus_ptr, srcp: src->user_cpus_ptr);
2736	}
2737	raw_spin_unlock_irqrestore(&src->pi_lock, flags);
2738
2739	if (unlikely(user_mask))
2740	kfree(objp: user_mask);
2741
2742	return 0;
2743	}
2744
2745	static inline struct cpumask *clear_user_cpus_ptr(struct task_struct *p)
2746	{
2747	struct cpumask *user_mask = NULL;
2748
2749	swap(p->user_cpus_ptr, user_mask);
2750
2751	return user_mask;
2752	}
2753
2754	void release_user_cpus_ptr(struct task_struct *p)
2755	{
2756	kfree(objp: clear_user_cpus_ptr(p));
2757	}
2758
2759	/*
2760	* This function is wildly self concurrent; here be dragons.
2761	*
2762	*
2763	* When given a valid mask, __set_cpus_allowed_ptr() must block until the
2764	* designated task is enqueued on an allowed CPU. If that task is currently
2765	* running, we have to kick it out using the CPU stopper.
2766	*
2767	* Migrate-Disable comes along and tramples all over our nice sandcastle.
2768	* Consider:
2769	*
2770	* Initial conditions: P0->cpus_mask = [0, 1]
2771	*
2772	* P0@CPU0 P1
2773	*
2774	* migrate_disable();
2775	* <preempted>
2776	* set_cpus_allowed_ptr(P0, [1]);
2777	*
2778	* P1 *cannot* return from this set_cpus_allowed_ptr() call until P0 executes
2779	* its outermost migrate_enable() (i.e. it exits its Migrate-Disable region).
2780	* This means we need the following scheme:
2781	*
2782	* P0@CPU0 P1
2783	*
2784	* migrate_disable();
2785	* <preempted>
2786	* set_cpus_allowed_ptr(P0, [1]);
2787	* <blocks>
2788	* <resumes>
2789	* migrate_enable();
2790	* __set_cpus_allowed_ptr();
2791	* <wakes local stopper>
2792	* `--> <woken on migration completion>
2793	*
2794	* Now the fun stuff: there may be several P1-like tasks, i.e. multiple
2795	* concurrent set_cpus_allowed_ptr(P0, [*]) calls. CPU affinity changes of any
2796	* task p are serialized by p->pi_lock, which we can leverage: the one that
2797	* should come into effect at the end of the Migrate-Disable region is the last
2798	* one. This means we only need to track a single cpumask (i.e. p->cpus_mask),
2799	* but we still need to properly signal those waiting tasks at the appropriate
2800	* moment.
2801	*
2802	* This is implemented using struct set_affinity_pending. The first
2803	* __set_cpus_allowed_ptr() caller within a given Migrate-Disable region will
2804	* setup an instance of that struct and install it on the targeted task_struct.
2805	* Any and all further callers will reuse that instance. Those then wait for
2806	* a completion signaled at the tail of the CPU stopper callback (1), triggered
2807	* on the end of the Migrate-Disable region (i.e. outermost migrate_enable()).
2808	*
2809	*
2810	* (1) In the cases covered above. There is one more where the completion is
2811	* signaled within affine_move_task() itself: when a subsequent affinity request
2812	* occurs after the stopper bailed out due to the targeted task still being
2813	* Migrate-Disable. Consider:
2814	*
2815	* Initial conditions: P0->cpus_mask = [0, 1]
2816	*
2817	* CPU0 P1 P2
2818	* <P0>
2819	* migrate_disable();
2820	* <preempted>
2821	* set_cpus_allowed_ptr(P0, [1]);
2822	* <blocks>
2823	* <migration/0>
2824	* migration_cpu_stop()
2825	* is_migration_disabled()
2826	* <bails>
2827	* set_cpus_allowed_ptr(P0, [0, 1]);
2828	* <signal completion>
2829	* <awakes>
2830	*
2831	* Note that the above is safe vs a concurrent migrate_enable(), as any
2832	* pending affinity completion is preceded by an uninstallation of
2833	* p->migration_pending done with p->pi_lock held.
2834	*/
2835	static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf,
2836	int dest_cpu, unsigned int flags)
2837	__releases(rq->lock)
2838	__releases(p->pi_lock)
2839	{
2840	struct set_affinity_pending my_pending = { }, *pending = NULL;
2841	bool stop_pending, complete = false;
2842
2843	/*
2844	* Can the task run on the task's current CPU? If so, we're done
2845	*
2846	* We are also done if the task is the current donor, boosting a lock-
2847	* holding proxy, (and potentially has been migrated outside its
2848	* current or previous affinity mask)
2849	*/
2850	if (cpumask_test_cpu(cpu: task_cpu(p), cpumask: &p->cpus_mask) ||
2851	(task_current_donor(rq, p) && !task_current(rq, p))) {
2852	struct task_struct *push_task = NULL;
2853
2854	if ((flags & SCA_MIGRATE_ENABLE) &&
2855	(p->migration_flags & MDF_PUSH) && !rq->push_busy) {
2856	rq->push_busy = true;
2857	push_task = get_task_struct(t: p);
2858	}
2859
2860	/*
2861	* If there are pending waiters, but no pending stop_work,
2862	* then complete now.
2863	*/
2864	pending = p->migration_pending;
2865	if (pending && !pending->stop_pending) {
2866	p->migration_pending = NULL;
2867	complete = true;
2868	}
2869
2870	preempt_disable();
2871	task_rq_unlock(rq, p, rf);
2872	if (push_task) {
2873	stop_one_cpu_nowait(cpu: rq->cpu, fn: push_cpu_stop,
2874	arg: p, work_buf: &rq->push_work);
2875	}
2876	preempt_enable();
2877
2878	if (complete)
2879	complete_all(&pending->done);
2880
2881	return 0;
2882	}
2883
2884	if (!(flags & SCA_MIGRATE_ENABLE)) {
2885	/* serialized by p->pi_lock */
2886	if (!p->migration_pending) {
2887	/* Install the request */
2888	refcount_set(r: &my_pending.refs, n: 1);
2889	init_completion(x: &my_pending.done);
2890	my_pending.arg = (struct migration_arg) {
2891	.task = p,
2892	.dest_cpu = dest_cpu,
2893	.pending = &my_pending,
2894	};
2895
2896	p->migration_pending = &my_pending;
2897	} else {
2898	pending = p->migration_pending;
2899	refcount_inc(r: &pending->refs);
2900	/*
2901	* Affinity has changed, but we've already installed a
2902	* pending. migration_cpu_stop() *must* see this, else
2903	* we risk a completion of the pending despite having a
2904	* task on a disallowed CPU.
2905	*
2906	* Serialized by p->pi_lock, so this is safe.
2907	*/
2908	pending->arg.dest_cpu = dest_cpu;
2909	}
2910	}
2911	pending = p->migration_pending;
2912	/*
2913	* - !MIGRATE_ENABLE:
2914	* we'll have installed a pending if there wasn't one already.
2915	*
2916	* - MIGRATE_ENABLE:
2917	* we're here because the current CPU isn't matching anymore,
2918	* the only way that can happen is because of a concurrent
2919	* set_cpus_allowed_ptr() call, which should then still be
2920	* pending completion.
2921	*
2922	* Either way, we really should have a @pending here.
2923	*/
2924	if (WARN_ON_ONCE(!pending)) {
2925	task_rq_unlock(rq, p, rf);
2926	return -EINVAL;
2927	}
2928
2929	if (task_on_cpu(rq, p) || READ_ONCE(p->__state) == TASK_WAKING) {
2930	/*
2931	* MIGRATE_ENABLE gets here because 'p == current', but for
2932	* anything else we cannot do is_migration_disabled(), punt
2933	* and have the stopper function handle it all race-free.
2934	*/
2935	stop_pending = pending->stop_pending;
2936	if (!stop_pending)
2937	pending->stop_pending = true;
2938
2939	if (flags & SCA_MIGRATE_ENABLE)
2940	p->migration_flags &= ~MDF_PUSH;
2941
2942	preempt_disable();
2943	task_rq_unlock(rq, p, rf);
2944	if (!stop_pending) {
2945	stop_one_cpu_nowait(cpu: cpu_of(rq), fn: migration_cpu_stop,
2946	arg: &pending->arg, work_buf: &pending->stop_work);
2947	}
2948	preempt_enable();
2949
2950	if (flags & SCA_MIGRATE_ENABLE)
2951	return 0;
2952	} else {
2953
2954	if (!is_migration_disabled(p)) {
2955	if (task_on_rq_queued(p))
2956	rq = move_queued_task(rq, rf, p, new_cpu: dest_cpu);
2957
2958	if (!pending->stop_pending) {
2959	p->migration_pending = NULL;
2960	complete = true;
2961	}
2962	}
2963	task_rq_unlock(rq, p, rf);
2964
2965	if (complete)
2966	complete_all(&pending->done);
2967	}
2968
2969	wait_for_completion(&pending->done);
2970
2971	if (refcount_dec_and_test(r: &pending->refs))
2972	wake_up_var(var: &pending->refs); /* No UaF, just an address */
2973
2974	/*
2975	* Block the original owner of &pending until all subsequent callers
2976	* have seen the completion and decremented the refcount
2977	*/
2978	wait_var_event(&my_pending.refs, !refcount_read(&my_pending.refs));
2979
2980	/* ARGH */
2981	WARN_ON_ONCE(my_pending.stop_pending);
2982
2983	return 0;
2984	}
2985
2986	/*
2987	* Called with both p->pi_lock and rq->lock held; drops both before returning.
2988	*/
2989	static int __set_cpus_allowed_ptr_locked(struct task_struct *p,
2990	struct affinity_context *ctx,
2991	struct rq *rq,
2992	struct rq_flags *rf)
2993	__releases(rq->lock)
2994	__releases(p->pi_lock)
2995	{
2996	const struct cpumask *cpu_allowed_mask = task_cpu_possible_mask(p);
2997	const struct cpumask *cpu_valid_mask = cpu_active_mask;
2998	bool kthread = p->flags & PF_KTHREAD;
2999	unsigned int dest_cpu;
3000	int ret = 0;
3001
3002	if (kthread || is_migration_disabled(p)) {
3003	/*
3004	* Kernel threads are allowed on online && !active CPUs,
3005	* however, during cpu-hot-unplug, even these might get pushed
3006	* away if not KTHREAD_IS_PER_CPU.
3007	*
3008	* Specifically, migration_disabled() tasks must not fail the
3009	* cpumask_any_and_distribute() pick below, esp. so on
3010	* SCA_MIGRATE_ENABLE, otherwise we'll not call
3011	* set_cpus_allowed_common() and actually reset p->cpus_ptr.
3012	*/
3013	cpu_valid_mask = cpu_online_mask;
3014	}
3015
3016	if (!kthread && !cpumask_subset(src1p: ctx->new_mask, src2p: cpu_allowed_mask)) {
3017	ret = -EINVAL;
3018	goto out;
3019	}
3020
3021	/*
3022	* Must re-check here, to close a race against __kthread_bind(),
3023	* sched_setaffinity() is not guaranteed to observe the flag.
3024	*/
3025	if ((ctx->flags & SCA_CHECK) && (p->flags & PF_NO_SETAFFINITY)) {
3026	ret = -EINVAL;
3027	goto out;
3028	}
3029
3030	if (!(ctx->flags & SCA_MIGRATE_ENABLE)) {
3031	if (cpumask_equal(src1p: &p->cpus_mask, src2p: ctx->new_mask)) {
3032	if (ctx->flags & SCA_USER)
3033	swap(p->user_cpus_ptr, ctx->user_mask);
3034	goto out;
3035	}
3036
3037	if (WARN_ON_ONCE(p == current &&
3038	is_migration_disabled(p) &&
3039	!cpumask_test_cpu(task_cpu(p), ctx->new_mask))) {
3040	ret = -EBUSY;
3041	goto out;
3042	}
3043	}
3044
3045	/*
3046	* Picking a ~random cpu helps in cases where we are changing affinity
3047	* for groups of tasks (ie. cpuset), so that load balancing is not
3048	* immediately required to distribute the tasks within their new mask.
3049	*/
3050	dest_cpu = cpumask_any_and_distribute(src1p: cpu_valid_mask, src2p: ctx->new_mask);
3051	if (dest_cpu >= nr_cpu_ids) {
3052	ret = -EINVAL;
3053	goto out;
3054	}
3055
3056	do_set_cpus_allowed(p, ctx);
3057
3058	return affine_move_task(rq, p, rf, dest_cpu, flags: ctx->flags);
3059
3060	out:
3061	task_rq_unlock(rq, p, rf);
3062
3063	return ret;
3064	}
3065
3066	/*
3067	* Change a given task's CPU affinity. Migrate the thread to a
3068	* proper CPU and schedule it away if the CPU it's executing on
3069	* is removed from the allowed bitmask.
3070	*
3071	* NOTE: the caller must have a valid reference to the task, the
3072	* task must not exit() & deallocate itself prematurely. The
3073	* call is not atomic; no spinlocks may be held.
3074	*/
3075	int __set_cpus_allowed_ptr(struct task_struct *p, struct affinity_context *ctx)
3076	{
3077	struct rq_flags rf;
3078	struct rq *rq;
3079
3080	rq = task_rq_lock(p, rf: &rf);
3081	/*
3082	* Masking should be skipped if SCA_USER or any of the SCA_MIGRATE_*
3083	* flags are set.
3084	*/
3085	if (p->user_cpus_ptr &&
3086	!(ctx->flags & (SCA_USER | SCA_MIGRATE_ENABLE | SCA_MIGRATE_DISABLE)) &&
3087	cpumask_and(dstp: rq->scratch_mask, src1p: ctx->new_mask, src2p: p->user_cpus_ptr))
3088	ctx->new_mask = rq->scratch_mask;
3089
3090	return __set_cpus_allowed_ptr_locked(p, ctx, rq, rf: &rf);
3091	}
3092
3093	int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask)
3094	{
3095	struct affinity_context ac = {
3096	.new_mask = new_mask,
3097	.flags = 0,
3098	};
3099
3100	return __set_cpus_allowed_ptr(p, ctx: &ac);
3101	}
3102	EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr);
3103
3104	/*
3105	* Change a given task's CPU affinity to the intersection of its current
3106	* affinity mask and @subset_mask, writing the resulting mask to @new_mask.
3107	* If user_cpus_ptr is defined, use it as the basis for restricting CPU
3108	* affinity or use cpu_online_mask instead.
3109	*
3110	* If the resulting mask is empty, leave the affinity unchanged and return
3111	* -EINVAL.
3112	*/
3113	static int restrict_cpus_allowed_ptr(struct task_struct *p,
3114	struct cpumask *new_mask,
3115	const struct cpumask *subset_mask)
3116	{
3117	struct affinity_context ac = {
3118	.new_mask = new_mask,
3119	.flags = 0,
3120	};
3121	struct rq_flags rf;
3122	struct rq *rq;
3123	int err;
3124
3125	rq = task_rq_lock(p, rf: &rf);
3126
3127	/*
3128	* Forcefully restricting the affinity of a deadline task is
3129	* likely to cause problems, so fail and noisily override the
3130	* mask entirely.
3131	*/
3132	if (task_has_dl_policy(p) && dl_bandwidth_enabled()) {
3133	err = -EPERM;
3134	goto err_unlock;
3135	}
3136
3137	if (!cpumask_and(dstp: new_mask, src1p: task_user_cpus(p), src2p: subset_mask)) {
3138	err = -EINVAL;
3139	goto err_unlock;
3140	}
3141
3142	return __set_cpus_allowed_ptr_locked(p, ctx: &ac, rq, rf: &rf);
3143
3144	err_unlock:
3145	task_rq_unlock(rq, p, rf: &rf);
3146	return err;
3147	}
3148
3149	/*
3150	* Restrict the CPU affinity of task @p so that it is a subset of
3151	* task_cpu_possible_mask() and point @p->user_cpus_ptr to a copy of the
3152	* old affinity mask. If the resulting mask is empty, we warn and walk
3153	* up the cpuset hierarchy until we find a suitable mask.
3154	*/
3155	void force_compatible_cpus_allowed_ptr(struct task_struct *p)
3156	{
3157	cpumask_var_t new_mask;
3158	const struct cpumask *override_mask = task_cpu_possible_mask(p);
3159
3160	alloc_cpumask_var(mask: &new_mask, GFP_KERNEL);
3161
3162	/*
3163	* __migrate_task() can fail silently in the face of concurrent
3164	* offlining of the chosen destination CPU, so take the hotplug
3165	* lock to ensure that the migration succeeds.
3166	*/
3167	cpus_read_lock();
3168	if (!cpumask_available(mask: new_mask))
3169	goto out_set_mask;
3170
3171	if (!restrict_cpus_allowed_ptr(p, new_mask, subset_mask: override_mask))
3172	goto out_free_mask;
3173
3174	/*
3175	* We failed to find a valid subset of the affinity mask for the
3176	* task, so override it based on its cpuset hierarchy.
3177	*/
3178	cpuset_cpus_allowed(p, mask: new_mask);
3179	override_mask = new_mask;
3180
3181	out_set_mask:
3182	if (printk_ratelimit()) {
3183	printk_deferred("Overriding affinity for process %d (%s) to CPUs %*pbl\n",
3184	task_pid_nr(p), p->comm,
3185	cpumask_pr_args(override_mask));
3186	}
3187
3188	WARN_ON(set_cpus_allowed_ptr(p, override_mask));
3189	out_free_mask:
3190	cpus_read_unlock();
3191	free_cpumask_var(mask: new_mask);
3192	}
3193
3194	/*
3195	* Restore the affinity of a task @p which was previously restricted by a
3196	* call to force_compatible_cpus_allowed_ptr().
3197	*
3198	* It is the caller's responsibility to serialise this with any calls to
3199	* force_compatible_cpus_allowed_ptr(@p).
3200	*/
3201	void relax_compatible_cpus_allowed_ptr(struct task_struct *p)
3202	{
3203	struct affinity_context ac = {
3204	.new_mask = task_user_cpus(p),
3205	.flags = 0,
3206	};
3207	int ret;
3208
3209	/*
3210	* Try to restore the old affinity mask with __sched_setaffinity().
3211	* Cpuset masking will be done there too.
3212	*/
3213	ret = __sched_setaffinity(p, ctx: &ac);
3214	WARN_ON_ONCE(ret);
3215	}
3216
3217	#ifdef CONFIG_SMP
3218
3219	void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
3220	{
3221	unsigned int state = READ_ONCE(p->__state);
3222
3223	/*
3224	* We should never call set_task_cpu() on a blocked task,
3225	* ttwu() will sort out the placement.
3226	*/
3227	WARN_ON_ONCE(state != TASK_RUNNING && state != TASK_WAKING && !p->on_rq);
3228
3229	/*
3230	* Migrating fair class task must have p->on_rq = TASK_ON_RQ_MIGRATING,
3231	* because schedstat_wait_{start,end} rebase migrating task's wait_start
3232	* time relying on p->on_rq.
3233	*/
3234	WARN_ON_ONCE(state == TASK_RUNNING &&
3235	p->sched_class == &fair_sched_class &&
3236	(p->on_rq && !task_on_rq_migrating(p)));
3237
3238	#ifdef CONFIG_LOCKDEP
3239	/*
3240	* The caller should hold either p->pi_lock or rq->lock, when changing
3241	* a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks.
3242	*
3243	* sched_move_task() holds both and thus holding either pins the cgroup,
3244	* see task_group().
3245	*
3246	* Furthermore, all task_rq users should acquire both locks, see
3247	* task_rq_lock().
3248	*/
3249	WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) ||
3250	lockdep_is_held(__rq_lockp(task_rq(p)))));
3251	#endif
3252	/*
3253	* Clearly, migrating tasks to offline CPUs is a fairly daft thing.
3254	*/
3255	WARN_ON_ONCE(!cpu_online(new_cpu));
3256
3257	WARN_ON_ONCE(is_migration_disabled(p));
3258
3259	trace_sched_migrate_task(p, dest_cpu: new_cpu);
3260
3261	if (task_cpu(p) != new_cpu) {
3262	if (p->sched_class->migrate_task_rq)
3263	p->sched_class->migrate_task_rq(p, new_cpu);
3264	p->se.nr_migrations++;
3265	perf_event_task_migrate(task: p);
3266	}
3267
3268	__set_task_cpu(p, cpu: new_cpu);
3269	}
3270	#endif /* CONFIG_SMP */
3271
3272	#ifdef CONFIG_NUMA_BALANCING
3273	static void __migrate_swap_task(struct task_struct *p, int cpu)
3274	{
3275	if (task_on_rq_queued(p)) {
3276	struct rq *src_rq, *dst_rq;
3277	struct rq_flags srf, drf;
3278
3279	src_rq = task_rq(p);
3280	dst_rq = cpu_rq(cpu);
3281
3282	rq_pin_lock(rq: src_rq, rf: &srf);
3283	rq_pin_lock(rq: dst_rq, rf: &drf);
3284
3285	move_queued_task_locked(src_rq, dst_rq, task: p);
3286	wakeup_preempt(rq: dst_rq, p, flags: 0);
3287
3288	rq_unpin_lock(rq: dst_rq, rf: &drf);
3289	rq_unpin_lock(rq: src_rq, rf: &srf);
3290
3291	} else {
3292	/*
3293	* Task isn't running anymore; make it appear like we migrated
3294	* it before it went to sleep. This means on wakeup we make the
3295	* previous CPU our target instead of where it really is.
3296	*/
3297	p->wake_cpu = cpu;
3298	}
3299	}
3300
3301	struct migration_swap_arg {
3302	struct task_struct *src_task, *dst_task;
3303	int src_cpu, dst_cpu;
3304	};
3305
3306	static int migrate_swap_stop(void *data)
3307	{
3308	struct migration_swap_arg *arg = data;
3309	struct rq *src_rq, *dst_rq;
3310
3311	if (!cpu_active(cpu: arg->src_cpu) || !cpu_active(cpu: arg->dst_cpu))
3312	return -EAGAIN;
3313
3314	src_rq = cpu_rq(arg->src_cpu);
3315	dst_rq = cpu_rq(arg->dst_cpu);
3316
3317	guard(double_raw_spinlock)(lock: &arg->src_task->pi_lock, lock2: &arg->dst_task->pi_lock);
3318	guard(double_rq_lock)(lock: src_rq, lock2: dst_rq);
3319
3320	if (task_cpu(p: arg->dst_task) != arg->dst_cpu)
3321	return -EAGAIN;
3322
3323	if (task_cpu(p: arg->src_task) != arg->src_cpu)
3324	return -EAGAIN;
3325
3326	if (!cpumask_test_cpu(cpu: arg->dst_cpu, cpumask: arg->src_task->cpus_ptr))
3327	return -EAGAIN;
3328
3329	if (!cpumask_test_cpu(cpu: arg->src_cpu, cpumask: arg->dst_task->cpus_ptr))
3330	return -EAGAIN;
3331
3332	__migrate_swap_task(p: arg->src_task, cpu: arg->dst_cpu);
3333	__migrate_swap_task(p: arg->dst_task, cpu: arg->src_cpu);
3334
3335	return 0;
3336	}
3337
3338	/*
3339	* Cross migrate two tasks
3340	*/
3341	int migrate_swap(struct task_struct *cur, struct task_struct *p,
3342	int target_cpu, int curr_cpu)
3343	{
3344	struct migration_swap_arg arg;
3345	int ret = -EINVAL;
3346
3347	arg = (struct migration_swap_arg){
3348	.src_task = cur,
3349	.src_cpu = curr_cpu,
3350	.dst_task = p,
3351	.dst_cpu = target_cpu,
3352	};
3353
3354	if (arg.src_cpu == arg.dst_cpu)
3355	goto out;
3356
3357	/*
3358	* These three tests are all lockless; this is OK since all of them
3359	* will be re-checked with proper locks held further down the line.
3360	*/
3361	if (!cpu_active(cpu: arg.src_cpu) || !cpu_active(cpu: arg.dst_cpu))
3362	goto out;
3363
3364	if (!cpumask_test_cpu(cpu: arg.dst_cpu, cpumask: arg.src_task->cpus_ptr))
3365	goto out;
3366
3367	if (!cpumask_test_cpu(cpu: arg.src_cpu, cpumask: arg.dst_task->cpus_ptr))
3368	goto out;
3369
3370	trace_sched_swap_numa(src_tsk: cur, src_cpu: arg.src_cpu, dst_tsk: p, dst_cpu: arg.dst_cpu);
3371	ret = stop_two_cpus(cpu1: arg.dst_cpu, cpu2: arg.src_cpu, fn: migrate_swap_stop, arg: &arg);
3372
3373	out:
3374	return ret;
3375	}
3376	#endif /* CONFIG_NUMA_BALANCING */
3377
3378	/***
3379	* kick_process - kick a running thread to enter/exit the kernel
3380	* @p: the to-be-kicked thread
3381	*
3382	* Cause a process which is running on another CPU to enter
3383	* kernel-mode, without any delay. (to get signals handled.)
3384	*
3385	* NOTE: this function doesn't have to take the runqueue lock,
3386	* because all it wants to ensure is that the remote task enters
3387	* the kernel. If the IPI races and the task has been migrated
3388	* to another CPU then no harm is done and the purpose has been
3389	* achieved as well.
3390	*/
3391	void kick_process(struct task_struct *p)
3392	{
3393	guard(preempt)();
3394	int cpu = task_cpu(p);
3395
3396	if ((cpu != smp_processor_id()) && task_curr(p))
3397	smp_send_reschedule(cpu);
3398	}
3399	EXPORT_SYMBOL_GPL(kick_process);
3400
3401	/*
3402	* ->cpus_ptr is protected by both rq->lock and p->pi_lock
3403	*
3404	* A few notes on cpu_active vs cpu_online:
3405	*
3406	* - cpu_active must be a subset of cpu_online
3407	*
3408	* - on CPU-up we allow per-CPU kthreads on the online && !active CPU,
3409	* see __set_cpus_allowed_ptr(). At this point the newly online
3410	* CPU isn't yet part of the sched domains, and balancing will not
3411	* see it.
3412	*
3413	* - on CPU-down we clear cpu_active() to mask the sched domains and
3414	* avoid the load balancer to place new tasks on the to be removed
3415	* CPU. Existing tasks will remain running there and will be taken
3416	* off.
3417	*
3418	* This means that fallback selection must not select !active CPUs.
3419	* And can assume that any active CPU must be online. Conversely
3420	* select_task_rq() below may allow selection of !active CPUs in order
3421	* to satisfy the above rules.
3422	*/
3423	static int select_fallback_rq(int cpu, struct task_struct *p)
3424	{
3425	int nid = cpu_to_node(cpu);
3426	const struct cpumask *nodemask = NULL;
3427	enum { cpuset, possible, fail } state = cpuset;
3428	int dest_cpu;
3429
3430	/*
3431	* If the node that the CPU is on has been offlined, cpu_to_node()
3432	* will return -1. There is no CPU on the node, and we should
3433	* select the CPU on the other node.
3434	*/
3435	if (nid != -1) {
3436	nodemask = cpumask_of_node(node: nid);
3437
3438	/* Look for allowed, online CPU in same node. */
3439	for_each_cpu(dest_cpu, nodemask) {
3440	if (is_cpu_allowed(p, cpu: dest_cpu))
3441	return dest_cpu;
3442	}
3443	}
3444
3445	for (;;) {
3446	/* Any allowed, online CPU? */
3447	for_each_cpu(dest_cpu, p->cpus_ptr) {
3448	if (!is_cpu_allowed(p, cpu: dest_cpu))
3449	continue;
3450
3451	goto out;
3452	}
3453
3454	/* No more Mr. Nice Guy. */
3455	switch (state) {
3456	case cpuset:
3457	if (cpuset_cpus_allowed_fallback(p)) {
3458	state = possible;
3459	break;
3460	}
3461	fallthrough;
3462	case possible:
3463	set_cpus_allowed_force(p, task_cpu_fallback_mask(p));
3464	state = fail;
3465	break;
3466	case fail:
3467	BUG();
3468	break;
3469	}
3470	}
3471
3472	out:
3473	if (state != cpuset) {
3474	/*
3475	* Don't tell them about moving exiting tasks or
3476	* kernel threads (both mm NULL), since they never
3477	* leave kernel.
3478	*/
3479	if (p->mm && printk_ratelimit()) {
3480	printk_deferred("process %d (%s) no longer affine to cpu%d\n",
3481	task_pid_nr(p), p->comm, cpu);
3482	}
3483	}
3484
3485	return dest_cpu;
3486	}
3487
3488	/*
3489	* The caller (fork, wakeup) owns p->pi_lock, ->cpus_ptr is stable.
3490	*/
3491	static inline
3492	int select_task_rq(struct task_struct *p, int cpu, int *wake_flags)
3493	{
3494	lockdep_assert_held(&p->pi_lock);
3495
3496	if (p->nr_cpus_allowed > 1 && !is_migration_disabled(p)) {
3497	cpu = p->sched_class->select_task_rq(p, cpu, *wake_flags);
3498	*wake_flags |= WF_RQ_SELECTED;
3499	} else {
3500	cpu = cpumask_any(p->cpus_ptr);
3501	}
3502
3503	/*
3504	* In order not to call set_task_cpu() on a blocking task we need
3505	* to rely on ttwu() to place the task on a valid ->cpus_ptr
3506	* CPU.
3507	*
3508	* Since this is common to all placement strategies, this lives here.
3509	*
3510	* [ this allows ->select_task() to simply return task_cpu(p) and
3511	* not worry about this generic constraint ]
3512	*/
3513	if (unlikely(!is_cpu_allowed(p, cpu)))
3514	cpu = select_fallback_rq(cpu: task_cpu(p), p);
3515
3516	return cpu;
3517	}
3518
3519	void sched_set_stop_task(int cpu, struct task_struct *stop)
3520	{
3521	static struct lock_class_key stop_pi_lock;
3522	struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 };
3523	struct task_struct *old_stop = cpu_rq(cpu)->stop;
3524
3525	if (stop) {
3526	/*
3527	* Make it appear like a SCHED_FIFO task, its something
3528	* userspace knows about and won't get confused about.
3529	*
3530	* Also, it will make PI more or less work without too
3531	* much confusion -- but then, stop work should not
3532	* rely on PI working anyway.
3533	*/
3534	sched_setscheduler_nocheck(stop, SCHED_FIFO, &param);
3535
3536	stop->sched_class = &stop_sched_class;
3537
3538	/*
3539	* The PI code calls rt_mutex_setprio() with ->pi_lock held to
3540	* adjust the effective priority of a task. As a result,
3541	* rt_mutex_setprio() can trigger (RT) balancing operations,
3542	* which can then trigger wakeups of the stop thread to push
3543	* around the current task.
3544	*
3545	* The stop task itself will never be part of the PI-chain, it
3546	* never blocks, therefore that ->pi_lock recursion is safe.
3547	* Tell lockdep about this by placing the stop->pi_lock in its
3548	* own class.
3549	*/
3550	lockdep_set_class(&stop->pi_lock, &stop_pi_lock);
3551	}
3552
3553	cpu_rq(cpu)->stop = stop;
3554
3555	if (old_stop) {
3556	/*
3557	* Reset it back to a normal scheduling class so that
3558	* it can die in pieces.
3559	*/
3560	old_stop->sched_class = &rt_sched_class;
3561	}
3562	}
3563
3564	static void
3565	ttwu_stat(struct task_struct *p, int cpu, int wake_flags)
3566	{
3567	struct rq *rq;
3568
3569	if (!schedstat_enabled())
3570	return;
3571
3572	rq = this_rq();
3573
3574	if (cpu == rq->cpu) {
3575	__schedstat_inc(rq->ttwu_local);
3576	__schedstat_inc(p->stats.nr_wakeups_local);
3577	} else {
3578	struct sched_domain *sd;
3579
3580	__schedstat_inc(p->stats.nr_wakeups_remote);
3581
3582	guard(rcu)();
3583	for_each_domain(rq->cpu, sd) {
3584	if (cpumask_test_cpu(cpu, cpumask: sched_domain_span(sd))) {
3585	__schedstat_inc(sd->ttwu_wake_remote);
3586	break;
3587	}
3588	}
3589	}
3590
3591	if (wake_flags & WF_MIGRATED)
3592	__schedstat_inc(p->stats.nr_wakeups_migrate);
3593
3594	__schedstat_inc(rq->ttwu_count);
3595	__schedstat_inc(p->stats.nr_wakeups);
3596
3597	if (wake_flags & WF_SYNC)
3598	__schedstat_inc(p->stats.nr_wakeups_sync);
3599	}
3600
3601	/*
3602	* Mark the task runnable.
3603	*/
3604	static inline void ttwu_do_wakeup(struct task_struct *p)
3605	{
3606	WRITE_ONCE(p->__state, TASK_RUNNING);
3607	trace_sched_wakeup(p);
3608	}
3609
3610	static void
3611	ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags,
3612	struct rq_flags *rf)
3613	{
3614	int en_flags = ENQUEUE_WAKEUP | ENQUEUE_NOCLOCK;
3615
3616	lockdep_assert_rq_held(rq);
3617
3618	if (p->sched_contributes_to_load)
3619	rq->nr_uninterruptible--;
3620
3621	if (wake_flags & WF_RQ_SELECTED)
3622	en_flags |= ENQUEUE_RQ_SELECTED;
3623	if (wake_flags & WF_MIGRATED)
3624	en_flags |= ENQUEUE_MIGRATED;
3625	else
3626	if (p->in_iowait) {
3627	delayacct_blkio_end(p);
3628	atomic_dec(v: &task_rq(p)->nr_iowait);
3629	}
3630
3631	activate_task(rq, p, flags: en_flags);
3632	wakeup_preempt(rq, p, flags: wake_flags);
3633
3634	ttwu_do_wakeup(p);
3635
3636	if (p->sched_class->task_woken) {
3637	/*
3638	* Our task @p is fully woken up and running; so it's safe to
3639	* drop the rq->lock, hereafter rq is only used for statistics.
3640	*/
3641	rq_unpin_lock(rq, rf);
3642	p->sched_class->task_woken(rq, p);
3643	rq_repin_lock(rq, rf);
3644	}
3645
3646	if (rq->idle_stamp) {
3647	u64 delta = rq_clock(rq) - rq->idle_stamp;
3648	u64 max = 2*rq->max_idle_balance_cost;
3649
3650	update_avg(avg: &rq->avg_idle, sample: delta);
3651
3652	if (rq->avg_idle > max)
3653	rq->avg_idle = max;
3654
3655	rq->idle_stamp = 0;
3656	}
3657	}
3658
3659	/*
3660	* Consider @p being inside a wait loop:
3661	*
3662	* for (;;) {
3663	* set_current_state(TASK_UNINTERRUPTIBLE);
3664	*
3665	* if (CONDITION)
3666	* break;
3667	*
3668	* schedule();
3669	* }
3670	* __set_current_state(TASK_RUNNING);
3671	*
3672	* between set_current_state() and schedule(). In this case @p is still
3673	* runnable, so all that needs doing is change p->state back to TASK_RUNNING in
3674	* an atomic manner.
3675	*
3676	* By taking task_rq(p)->lock we serialize against schedule(), if @p->on_rq
3677	* then schedule() must still happen and p->state can be changed to
3678	* TASK_RUNNING. Otherwise we lost the race, schedule() has happened, and we
3679	* need to do a full wakeup with enqueue.
3680	*
3681	* Returns: %true when the wakeup is done,
3682	* %false otherwise.
3683	*/
3684	static int ttwu_runnable(struct task_struct *p, int wake_flags)
3685	{
3686	struct rq_flags rf;
3687	struct rq *rq;
3688	int ret = 0;
3689
3690	rq = __task_rq_lock(p, rf: &rf);
3691	if (task_on_rq_queued(p)) {
3692	update_rq_clock(rq);
3693	if (p->se.sched_delayed)
3694	enqueue_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_DELAYED);
3695	if (!task_on_cpu(rq, p)) {
3696	/*
3697	* When on_rq && !on_cpu the task is preempted, see if
3698	* it should preempt the task that is current now.
3699	*/
3700	wakeup_preempt(rq, p, flags: wake_flags);
3701	}
3702	ttwu_do_wakeup(p);
3703	ret = 1;
3704	}
3705	__task_rq_unlock(rq, p, rf: &rf);
3706
3707	return ret;
3708	}
3709
3710	void sched_ttwu_pending(void *arg)
3711	{
3712	struct llist_node *llist = arg;
3713	struct rq *rq = this_rq();
3714	struct task_struct *p, *t;
3715	struct rq_flags rf;
3716
3717	if (!llist)
3718	return;
3719
3720	rq_lock_irqsave(rq, rf: &rf);
3721	update_rq_clock(rq);
3722
3723	llist_for_each_entry_safe(p, t, llist, wake_entry.llist) {
3724	if (WARN_ON_ONCE(p->on_cpu))
3725	smp_cond_load_acquire(&p->on_cpu, !VAL);
3726
3727	if (WARN_ON_ONCE(task_cpu(p) != cpu_of(rq)))
3728	set_task_cpu(p, new_cpu: cpu_of(rq));
3729
3730	ttwu_do_activate(rq, p, wake_flags: p->sched_remote_wakeup ? WF_MIGRATED : 0, rf: &rf);
3731	}
3732
3733	/*
3734	* Must be after enqueueing at least once task such that
3735	* idle_cpu() does not observe a false-negative -- if it does,
3736	* it is possible for select_idle_siblings() to stack a number
3737	* of tasks on this CPU during that window.
3738	*
3739	* It is OK to clear ttwu_pending when another task pending.
3740	* We will receive IPI after local IRQ enabled and then enqueue it.
3741	* Since now nr_running > 0, idle_cpu() will always get correct result.
3742	*/
3743	WRITE_ONCE(rq->ttwu_pending, 0);
3744	rq_unlock_irqrestore(rq, rf: &rf);
3745	}
3746
3747	/*
3748	* Prepare the scene for sending an IPI for a remote smp_call
3749	*
3750	* Returns true if the caller can proceed with sending the IPI.
3751	* Returns false otherwise.
3752	*/
3753	bool call_function_single_prep_ipi(int cpu)
3754	{
3755	if (set_nr_if_polling(cpu_rq(cpu)->idle)) {
3756	trace_sched_wake_idle_without_ipi(cpu);
3757	return false;
3758	}
3759
3760	return true;
3761	}
3762
3763	/*
3764	* Queue a task on the target CPUs wake_list and wake the CPU via IPI if
3765	* necessary. The wakee CPU on receipt of the IPI will queue the task
3766	* via sched_ttwu_wakeup() for activation so the wakee incurs the cost
3767	* of the wakeup instead of the waker.
3768	*/
3769	static void __ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
3770	{
3771	struct rq *rq = cpu_rq(cpu);
3772
3773	p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
3774
3775	WRITE_ONCE(rq->ttwu_pending, 1);
3776	#ifdef CONFIG_SMP
3777	__smp_call_single_queue(cpu, node: &p->wake_entry.llist);
3778	#endif
3779	}
3780
3781	void wake_up_if_idle(int cpu)
3782	{
3783	struct rq *rq = cpu_rq(cpu);
3784
3785	guard(rcu)();
3786	if (is_idle_task(rcu_dereference(rq->curr))) {
3787	guard(rq_lock_irqsave)(l: rq);
3788	if (is_idle_task(p: rq->curr))
3789	resched_curr(rq);
3790	}
3791	}
3792
3793	bool cpus_equal_capacity(int this_cpu, int that_cpu)
3794	{
3795	if (!sched_asym_cpucap_active())
3796	return true;
3797
3798	if (this_cpu == that_cpu)
3799	return true;
3800
3801	return arch_scale_cpu_capacity(cpu: this_cpu) == arch_scale_cpu_capacity(cpu: that_cpu);
3802	}
3803
3804	bool cpus_share_cache(int this_cpu, int that_cpu)
3805	{
3806	if (this_cpu == that_cpu)
3807	return true;
3808
3809	return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu);
3810	}
3811
3812	/*
3813	* Whether CPUs are share cache resources, which means LLC on non-cluster
3814	* machines and LLC tag or L2 on machines with clusters.
3815	*/
3816	bool cpus_share_resources(int this_cpu, int that_cpu)
3817	{
3818	if (this_cpu == that_cpu)
3819	return true;
3820
3821	return per_cpu(sd_share_id, this_cpu) == per_cpu(sd_share_id, that_cpu);
3822	}
3823
3824	static inline bool ttwu_queue_cond(struct task_struct *p, int cpu)
3825	{
3826	/* See SCX_OPS_ALLOW_QUEUED_WAKEUP. */
3827	if (!scx_allow_ttwu_queue(p))
3828	return false;
3829
3830	#ifdef CONFIG_SMP
3831	if (p->sched_class == &stop_sched_class)
3832	return false;
3833	#endif
3834
3835	/*
3836	* Do not complicate things with the async wake_list while the CPU is
3837	* in hotplug state.
3838	*/
3839	if (!cpu_active(cpu))
3840	return false;
3841
3842	/* Ensure the task will still be allowed to run on the CPU. */
3843	if (!cpumask_test_cpu(cpu, cpumask: p->cpus_ptr))
3844	return false;
3845
3846	/*
3847	* If the CPU does not share cache, then queue the task on the
3848	* remote rqs wakelist to avoid accessing remote data.
3849	*/
3850	if (!cpus_share_cache(smp_processor_id(), that_cpu: cpu))
3851	return true;
3852
3853	if (cpu == smp_processor_id())
3854	return false;
3855
3856	/*
3857	* If the wakee cpu is idle, or the task is descheduling and the
3858	* only running task on the CPU, then use the wakelist to offload
3859	* the task activation to the idle (or soon-to-be-idle) CPU as
3860	* the current CPU is likely busy. nr_running is checked to
3861	* avoid unnecessary task stacking.
3862	*
3863	* Note that we can only get here with (wakee) p->on_rq=0,
3864	* p->on_cpu can be whatever, we've done the dequeue, so
3865	* the wakee has been accounted out of ->nr_running.
3866	*/
3867	if (!cpu_rq(cpu)->nr_running)
3868	return true;
3869
3870	return false;
3871	}
3872
3873	static bool ttwu_queue_wakelist(struct task_struct *p, int cpu, int wake_flags)
3874	{
3875	if (sched_feat(TTWU_QUEUE) && ttwu_queue_cond(p, cpu)) {
3876	sched_clock_cpu(cpu); /* Sync clocks across CPUs */
3877	__ttwu_queue_wakelist(p, cpu, wake_flags);
3878	return true;
3879	}
3880
3881	return false;
3882	}
3883
3884	static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
3885	{
3886	struct rq *rq = cpu_rq(cpu);
3887	struct rq_flags rf;
3888
3889	if (ttwu_queue_wakelist(p, cpu, wake_flags))
3890	return;
3891
3892	rq_lock(rq, rf: &rf);
3893	update_rq_clock(rq);
3894	ttwu_do_activate(rq, p, wake_flags, rf: &rf);
3895	rq_unlock(rq, rf: &rf);
3896	}
3897
3898	/*
3899	* Invoked from try_to_wake_up() to check whether the task can be woken up.
3900	*
3901	* The caller holds p::pi_lock if p != current or has preemption
3902	* disabled when p == current.
3903	*
3904	* The rules of saved_state:
3905	*
3906	* The related locking code always holds p::pi_lock when updating
3907	* p::saved_state, which means the code is fully serialized in both cases.
3908	*
3909	* For PREEMPT_RT, the lock wait and lock wakeups happen via TASK_RTLOCK_WAIT.
3910	* No other bits set. This allows to distinguish all wakeup scenarios.
3911	*
3912	* For FREEZER, the wakeup happens via TASK_FROZEN. No other bits set. This
3913	* allows us to prevent early wakeup of tasks before they can be run on
3914	* asymmetric ISA architectures (eg ARMv9).
3915	*/
3916	static __always_inline
3917	bool ttwu_state_match(struct task_struct *p, unsigned int state, int *success)
3918	{
3919	int match;
3920
3921	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) {
3922	WARN_ON_ONCE((state & TASK_RTLOCK_WAIT) &&
3923	state != TASK_RTLOCK_WAIT);
3924	}
3925
3926	*success = !!(match = __task_state_match(p, state));
3927
3928	/*
3929	* Saved state preserves the task state across blocking on
3930	* an RT lock or TASK_FREEZABLE tasks. If the state matches,
3931	* set p::saved_state to TASK_RUNNING, but do not wake the task
3932	* because it waits for a lock wakeup or __thaw_task(). Also
3933	* indicate success because from the regular waker's point of
3934	* view this has succeeded.
3935	*
3936	* After acquiring the lock the task will restore p::__state
3937	* from p::saved_state which ensures that the regular
3938	* wakeup is not lost. The restore will also set
3939	* p::saved_state to TASK_RUNNING so any further tests will
3940	* not result in false positives vs. @success
3941	*/
3942	if (match < 0)
3943	p->saved_state = TASK_RUNNING;
3944
3945	return match > 0;
3946	}
3947
3948	/*
3949	* Notes on Program-Order guarantees on SMP systems.
3950	*
3951	* MIGRATION
3952	*
3953	* The basic program-order guarantee on SMP systems is that when a task [t]
3954	* migrates, all its activity on its old CPU [c0] happens-before any subsequent
3955	* execution on its new CPU [c1].
3956	*
3957	* For migration (of runnable tasks) this is provided by the following means:
3958	*
3959	* A) UNLOCK of the rq(c0)->lock scheduling out task t
3960	* B) migration for t is required to synchronize *both* rq(c0)->lock and
3961	* rq(c1)->lock (if not at the same time, then in that order).
3962	* C) LOCK of the rq(c1)->lock scheduling in task
3963	*
3964	* Release/acquire chaining guarantees that B happens after A and C after B.
3965	* Note: the CPU doing B need not be c0 or c1
3966	*
3967	* Example:
3968	*
3969	* CPU0 CPU1 CPU2
3970	*
3971	* LOCK rq(0)->lock
3972	* sched-out X
3973	* sched-in Y
3974	* UNLOCK rq(0)->lock
3975	*
3976	* LOCK rq(0)->lock // orders against CPU0
3977	* dequeue X
3978	* UNLOCK rq(0)->lock
3979	*
3980	* LOCK rq(1)->lock
3981	* enqueue X
3982	* UNLOCK rq(1)->lock
3983	*
3984	* LOCK rq(1)->lock // orders against CPU2
3985	* sched-out Z
3986	* sched-in X
3987	* UNLOCK rq(1)->lock
3988	*
3989	*
3990	* BLOCKING -- aka. SLEEP + WAKEUP
3991	*
3992	* For blocking we (obviously) need to provide the same guarantee as for
3993	* migration. However the means are completely different as there is no lock
3994	* chain to provide order. Instead we do:
3995	*
3996	* 1) smp_store_release(X->on_cpu, 0) -- finish_task()
3997	* 2) smp_cond_load_acquire(!X->on_cpu) -- try_to_wake_up()
3998	*
3999	* Example:
4000	*
4001	* CPU0 (schedule) CPU1 (try_to_wake_up) CPU2 (schedule)
4002	*
4003	* LOCK rq(0)->lock LOCK X->pi_lock
4004	* dequeue X
4005	* sched-out X
4006	* smp_store_release(X->on_cpu, 0);
4007	*
4008	* smp_cond_load_acquire(&X->on_cpu, !VAL);
4009	* X->state = WAKING
4010	* set_task_cpu(X,2)
4011	*
4012	* LOCK rq(2)->lock
4013	* enqueue X
4014	* X->state = RUNNING
4015	* UNLOCK rq(2)->lock
4016	*
4017	* LOCK rq(2)->lock // orders against CPU1
4018	* sched-out Z
4019	* sched-in X
4020	* UNLOCK rq(2)->lock
4021	*
4022	* UNLOCK X->pi_lock
4023	* UNLOCK rq(0)->lock
4024	*
4025	*
4026	* However, for wakeups there is a second guarantee we must provide, namely we
4027	* must ensure that CONDITION=1 done by the caller can not be reordered with
4028	* accesses to the task state; see try_to_wake_up() and set_current_state().
4029	*/
4030
4031	/**
4032	* try_to_wake_up - wake up a thread
4033	* @p: the thread to be awakened
4034	* @state: the mask of task states that can be woken
4035	* @wake_flags: wake modifier flags (WF_*)
4036	*
4037	* Conceptually does:
4038	*
4039	* If (@state & @p->state) @p->state = TASK_RUNNING.
4040	*
4041	* If the task was not queued/runnable, also place it back on a runqueue.
4042	*
4043	* This function is atomic against schedule() which would dequeue the task.
4044	*
4045	* It issues a full memory barrier before accessing @p->state, see the comment
4046	* with set_current_state().
4047	*
4048	* Uses p->pi_lock to serialize against concurrent wake-ups.
4049	*
4050	* Relies on p->pi_lock stabilizing:
4051	* - p->sched_class
4052	* - p->cpus_ptr
4053	* - p->sched_task_group
4054	* in order to do migration, see its use of select_task_rq()/set_task_cpu().
4055	*
4056	* Tries really hard to only take one task_rq(p)->lock for performance.
4057	* Takes rq->lock in:
4058	* - ttwu_runnable() -- old rq, unavoidable, see comment there;
4059	* - ttwu_queue() -- new rq, for enqueue of the task;
4060	* - psi_ttwu_dequeue() -- much sadness :-( accounting will kill us.
4061	*
4062	* As a consequence we race really badly with just about everything. See the
4063	* many memory barriers and their comments for details.
4064	*
4065	* Return: %true if @p->state changes (an actual wakeup was done),
4066	* %false otherwise.
4067	*/
4068	int try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags)
4069	{
4070	guard(preempt)();
4071	int cpu, success = 0;
4072
4073	wake_flags |= WF_TTWU;
4074
4075	if (p == current) {
4076	/*
4077	* We're waking current, this means 'p->on_rq' and 'task_cpu(p)
4078	* == smp_processor_id()'. Together this means we can special
4079	* case the whole 'p->on_rq && ttwu_runnable()' case below
4080	* without taking any locks.
4081	*
4082	* Specifically, given current runs ttwu() we must be before
4083	* schedule()'s block_task(), as such this must not observe
4084	* sched_delayed.
4085	*
4086	* In particular:
4087	* - we rely on Program-Order guarantees for all the ordering,
4088	* - we're serialized against set_special_state() by virtue of
4089	* it disabling IRQs (this allows not taking ->pi_lock).
4090	*/
4091	WARN_ON_ONCE(p->se.sched_delayed);
4092	if (!ttwu_state_match(p, state, success: &success))
4093	goto out;
4094
4095	trace_sched_waking(p);
4096	ttwu_do_wakeup(p);
4097	goto out;
4098	}
4099
4100	/*
4101	* If we are going to wake up a thread waiting for CONDITION we
4102	* need to ensure that CONDITION=1 done by the caller can not be
4103	* reordered with p->state check below. This pairs with smp_store_mb()
4104	* in set_current_state() that the waiting thread does.
4105	*/
4106	scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {
4107	smp_mb__after_spinlock();
4108	if (!ttwu_state_match(p, state, success: &success))
4109	break;
4110
4111	trace_sched_waking(p);
4112
4113	/*
4114	* Ensure we load p->on_rq _after_ p->state, otherwise it would
4115	* be possible to, falsely, observe p->on_rq == 0 and get stuck
4116	* in smp_cond_load_acquire() below.
4117	*
4118	* sched_ttwu_pending() try_to_wake_up()
4119	* STORE p->on_rq = 1 LOAD p->state
4120	* UNLOCK rq->lock
4121	*
4122	* __schedule() (switch to task 'p')
4123	* LOCK rq->lock smp_rmb();
4124	* smp_mb__after_spinlock();
4125	* UNLOCK rq->lock
4126	*
4127	* [task p]
4128	* STORE p->state = UNINTERRUPTIBLE LOAD p->on_rq
4129	*
4130	* Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
4131	* __schedule(). See the comment for smp_mb__after_spinlock().
4132	*
4133	* A similar smp_rmb() lives in __task_needs_rq_lock().
4134	*/
4135	smp_rmb();
4136	if (READ_ONCE(p->on_rq) && ttwu_runnable(p, wake_flags))
4137	break;
4138
4139	/*
4140	* Ensure we load p->on_cpu _after_ p->on_rq, otherwise it would be
4141	* possible to, falsely, observe p->on_cpu == 0.
4142	*
4143	* One must be running (->on_cpu == 1) in order to remove oneself
4144	* from the runqueue.
4145	*
4146	* __schedule() (switch to task 'p') try_to_wake_up()
4147	* STORE p->on_cpu = 1 LOAD p->on_rq
4148	* UNLOCK rq->lock
4149	*
4150	* __schedule() (put 'p' to sleep)
4151	* LOCK rq->lock smp_rmb();
4152	* smp_mb__after_spinlock();
4153	* STORE p->on_rq = 0 LOAD p->on_cpu
4154	*
4155	* Pairs with the LOCK+smp_mb__after_spinlock() on rq->lock in
4156	* __schedule(). See the comment for smp_mb__after_spinlock().
4157	*
4158	* Form a control-dep-acquire with p->on_rq == 0 above, to ensure
4159	* schedule()'s block_task() has 'happened' and p will no longer
4160	* care about it's own p->state. See the comment in __schedule().
4161	*/
4162	smp_acquire__after_ctrl_dep();
4163
4164	/*
4165	* We're doing the wakeup (@success == 1), they did a dequeue (p->on_rq
4166	* == 0), which means we need to do an enqueue, change p->state to
4167	* TASK_WAKING such that we can unlock p->pi_lock before doing the
4168	* enqueue, such as ttwu_queue_wakelist().
4169	*/
4170	WRITE_ONCE(p->__state, TASK_WAKING);
4171
4172	/*
4173	* If the owning (remote) CPU is still in the middle of schedule() with
4174	* this task as prev, considering queueing p on the remote CPUs wake_list
4175	* which potentially sends an IPI instead of spinning on p->on_cpu to
4176	* let the waker make forward progress. This is safe because IRQs are
4177	* disabled and the IPI will deliver after on_cpu is cleared.
4178	*
4179	* Ensure we load task_cpu(p) after p->on_cpu:
4180	*
4181	* set_task_cpu(p, cpu);
4182	* STORE p->cpu = @cpu
4183	* __schedule() (switch to task 'p')
4184	* LOCK rq->lock
4185	* smp_mb__after_spin_lock() smp_cond_load_acquire(&p->on_cpu)
4186	* STORE p->on_cpu = 1 LOAD p->cpu
4187	*
4188	* to ensure we observe the correct CPU on which the task is currently
4189	* scheduling.
4190	*/
4191	if (smp_load_acquire(&p->on_cpu) &&
4192	ttwu_queue_wakelist(p, cpu: task_cpu(p), wake_flags))
4193	break;
4194
4195	/*
4196	* If the owning (remote) CPU is still in the middle of schedule() with
4197	* this task as prev, wait until it's done referencing the task.
4198	*
4199	* Pairs with the smp_store_release() in finish_task().
4200	*
4201	* This ensures that tasks getting woken will be fully ordered against
4202	* their previous state and preserve Program Order.
4203	*/
4204	smp_cond_load_acquire(&p->on_cpu, !VAL);
4205
4206	cpu = select_task_rq(p, cpu: p->wake_cpu, wake_flags: &wake_flags);
4207	if (task_cpu(p) != cpu) {
4208	if (p->in_iowait) {
4209	delayacct_blkio_end(p);
4210	atomic_dec(v: &task_rq(p)->nr_iowait);
4211	}
4212
4213	wake_flags |= WF_MIGRATED;
4214	psi_ttwu_dequeue(p);
4215	set_task_cpu(p, new_cpu: cpu);
4216	}
4217
4218	ttwu_queue(p, cpu, wake_flags);
4219	}
4220	out:
4221	if (success)
4222	ttwu_stat(p, cpu: task_cpu(p), wake_flags);
4223
4224	return success;
4225	}
4226
4227	static bool __task_needs_rq_lock(struct task_struct *p)
4228	{
4229	unsigned int state = READ_ONCE(p->__state);
4230
4231	/*
4232	* Since pi->lock blocks try_to_wake_up(), we don't need rq->lock when
4233	* the task is blocked. Make sure to check @state since ttwu() can drop
4234	* locks at the end, see ttwu_queue_wakelist().
4235	*/
4236	if (state == TASK_RUNNING || state == TASK_WAKING)
4237	return true;
4238
4239	/*
4240	* Ensure we load p->on_rq after p->__state, otherwise it would be
4241	* possible to, falsely, observe p->on_rq == 0.
4242	*
4243	* See try_to_wake_up() for a longer comment.
4244	*/
4245	smp_rmb();
4246	if (p->on_rq)
4247	return true;
4248
4249	/*
4250	* Ensure the task has finished __schedule() and will not be referenced
4251	* anymore. Again, see try_to_wake_up() for a longer comment.
4252	*/
4253	smp_rmb();
4254	smp_cond_load_acquire(&p->on_cpu, !VAL);
4255
4256	return false;
4257	}
4258
4259	/**
4260	* task_call_func - Invoke a function on task in fixed state
4261	* @p: Process for which the function is to be invoked, can be @current.
4262	* @func: Function to invoke.
4263	* @arg: Argument to function.
4264	*
4265	* Fix the task in it's current state by avoiding wakeups and or rq operations
4266	* and call @func(@arg) on it. This function can use task_is_runnable() and
4267	* task_curr() to work out what the state is, if required. Given that @func
4268	* can be invoked with a runqueue lock held, it had better be quite
4269	* lightweight.
4270	*
4271	* Returns:
4272	* Whatever @func returns
4273	*/
4274	int task_call_func(struct task_struct *p, task_call_f func, void *arg)
4275	{
4276	struct rq *rq = NULL;
4277	struct rq_flags rf;
4278	int ret;
4279
4280	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
4281
4282	if (__task_needs_rq_lock(p))
4283	rq = __task_rq_lock(p, rf: &rf);
4284
4285	/*
4286	* At this point the task is pinned; either:
4287	* - blocked and we're holding off wakeups (pi->lock)
4288	* - woken, and we're holding off enqueue (rq->lock)
4289	* - queued, and we're holding off schedule (rq->lock)
4290	* - running, and we're holding off de-schedule (rq->lock)
4291	*
4292	* The called function (@func) can use: task_curr(), p->on_rq and
4293	* p->__state to differentiate between these states.
4294	*/
4295	ret = func(p, arg);
4296
4297	if (rq)
4298	__task_rq_unlock(rq, p, rf: &rf);
4299
4300	raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags);
4301	return ret;
4302	}
4303
4304	/**
4305	* cpu_curr_snapshot - Return a snapshot of the currently running task
4306	* @cpu: The CPU on which to snapshot the task.
4307	*
4308	* Returns the task_struct pointer of the task "currently" running on
4309	* the specified CPU.
4310	*
4311	* If the specified CPU was offline, the return value is whatever it
4312	* is, perhaps a pointer to the task_struct structure of that CPU's idle
4313	* task, but there is no guarantee. Callers wishing a useful return
4314	* value must take some action to ensure that the specified CPU remains
4315	* online throughout.
4316	*
4317	* This function executes full memory barriers before and after fetching
4318	* the pointer, which permits the caller to confine this function's fetch
4319	* with respect to the caller's accesses to other shared variables.
4320	*/
4321	struct task_struct *cpu_curr_snapshot(int cpu)
4322	{
4323	struct rq *rq = cpu_rq(cpu);
4324	struct task_struct *t;
4325	struct rq_flags rf;
4326
4327	rq_lock_irqsave(rq, rf: &rf);
4328	smp_mb__after_spinlock(); /* Pairing determined by caller's synchronization design. */
4329	t = rcu_dereference(cpu_curr(cpu));
4330	rq_unlock_irqrestore(rq, rf: &rf);
4331	smp_mb(); /* Pairing determined by caller's synchronization design. */
4332
4333	return t;
4334	}
4335
4336	/**
4337	* wake_up_process - Wake up a specific process
4338	* @p: The process to be woken up.
4339	*
4340	* Attempt to wake up the nominated process and move it to the set of runnable
4341	* processes.
4342	*
4343	* Return: 1 if the process was woken up, 0 if it was already running.
4344	*
4345	* This function executes a full memory barrier before accessing the task state.
4346	*/
4347	int wake_up_process(struct task_struct *p)
4348	{
4349	return try_to_wake_up(p, TASK_NORMAL, wake_flags: 0);
4350	}
4351	EXPORT_SYMBOL(wake_up_process);
4352
4353	int wake_up_state(struct task_struct *p, unsigned int state)
4354	{
4355	return try_to_wake_up(p, state, wake_flags: 0);
4356	}
4357
4358	/*
4359	* Perform scheduler related setup for a newly forked process p.
4360	* p is forked by current.
4361	*
4362	* __sched_fork() is basic setup which is also used by sched_init() to
4363	* initialize the boot CPU's idle task.
4364	*/
4365	static void __sched_fork(u64 clone_flags, struct task_struct *p)
4366	{
4367	p->on_rq = 0;
4368
4369	p->se.on_rq = 0;
4370	p->se.exec_start = 0;
4371	p->se.sum_exec_runtime = 0;
4372	p->se.prev_sum_exec_runtime = 0;
4373	p->se.nr_migrations = 0;
4374	p->se.vruntime = 0;
4375	p->se.vlag = 0;
4376	INIT_LIST_HEAD(list: &p->se.group_node);
4377
4378	/* A delayed task cannot be in clone(). */
4379	WARN_ON_ONCE(p->se.sched_delayed);
4380
4381	#ifdef CONFIG_FAIR_GROUP_SCHED
4382	p->se.cfs_rq = NULL;
4383	#ifdef CONFIG_CFS_BANDWIDTH
4384	init_cfs_throttle_work(p);
4385	#endif
4386	#endif
4387
4388	#ifdef CONFIG_SCHEDSTATS
4389	/* Even if schedstat is disabled, there should not be garbage */
4390	memset(&p->stats, 0, sizeof(p->stats));
4391	#endif
4392
4393	init_dl_entity(dl_se: &p->dl);
4394
4395	INIT_LIST_HEAD(list: &p->rt.run_list);
4396	p->rt.timeout = 0;
4397	p->rt.time_slice = sched_rr_timeslice;
4398	p->rt.on_rq = 0;
4399	p->rt.on_list = 0;
4400
4401	#ifdef CONFIG_SCHED_CLASS_EXT
4402	init_scx_entity(&p->scx);
4403	#endif
4404
4405	#ifdef CONFIG_PREEMPT_NOTIFIERS
4406	INIT_HLIST_HEAD(&p->preempt_notifiers);
4407	#endif
4408
4409	#ifdef CONFIG_COMPACTION
4410	p->capture_control = NULL;
4411	#endif
4412	init_numa_balancing(clone_flags, p);
4413	p->wake_entry.u_flags = CSD_TYPE_TTWU;
4414	p->migration_pending = NULL;
4415	}
4416
4417	DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
4418
4419	#ifdef CONFIG_NUMA_BALANCING
4420
4421	int sysctl_numa_balancing_mode;
4422
4423	static void __set_numabalancing_state(bool enabled)
4424	{
4425	if (enabled)
4426	static_branch_enable(&sched_numa_balancing);
4427	else
4428	static_branch_disable(&sched_numa_balancing);
4429	}
4430
4431	void set_numabalancing_state(bool enabled)
4432	{
4433	if (enabled)
4434	sysctl_numa_balancing_mode = NUMA_BALANCING_NORMAL;
4435	else
4436	sysctl_numa_balancing_mode = NUMA_BALANCING_DISABLED;
4437	__set_numabalancing_state(enabled);
4438	}
4439
4440	#ifdef CONFIG_PROC_SYSCTL
4441	static void reset_memory_tiering(void)
4442	{
4443	struct pglist_data *pgdat;
4444
4445	for_each_online_pgdat(pgdat) {
4446	pgdat->nbp_threshold = 0;
4447	pgdat->nbp_th_nr_cand = node_page_state(pgdat, item: PGPROMOTE_CANDIDATE);
4448	pgdat->nbp_th_start = jiffies_to_msecs(j: jiffies);
4449	}
4450	}
4451
4452	static int sysctl_numa_balancing(const struct ctl_table *table, int write,
4453	void *buffer, size_t *lenp, loff_t *ppos)
4454	{
4455	struct ctl_table t;
4456	int err;
4457	int state = sysctl_numa_balancing_mode;
4458
4459	if (write && !capable(CAP_SYS_ADMIN))
4460	return -EPERM;
4461
4462	t = *table;
4463	t.data = &state;
4464	err = proc_dointvec_minmax(table: &t, dir: write, buffer, lenp, ppos);
4465	if (err < 0)
4466	return err;
4467	if (write) {
4468	if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
4469	(state & NUMA_BALANCING_MEMORY_TIERING))
4470	reset_memory_tiering();
4471	sysctl_numa_balancing_mode = state;
4472	__set_numabalancing_state(enabled: state);
4473	}
4474	return err;
4475	}
4476	#endif /* CONFIG_PROC_SYSCTL */
4477	#endif /* CONFIG_NUMA_BALANCING */
4478
4479	#ifdef CONFIG_SCHEDSTATS
4480
4481	DEFINE_STATIC_KEY_FALSE(sched_schedstats);
4482
4483	static void set_schedstats(bool enabled)
4484	{
4485	if (enabled)
4486	static_branch_enable(&sched_schedstats);
4487	else
4488	static_branch_disable(&sched_schedstats);
4489	}
4490
4491	void force_schedstat_enabled(void)
4492	{
4493	if (!schedstat_enabled()) {
4494	pr_info("kernel profiling enabled schedstats, disable via kernel.sched_schedstats.\n");
4495	static_branch_enable(&sched_schedstats);
4496	}
4497	}
4498
4499	static int __init setup_schedstats(char *str)
4500	{
4501	int ret = 0;
4502	if (!str)
4503	goto out;
4504
4505	if (!strcmp(str, "enable")) {
4506	set_schedstats(true);
4507	ret = 1;
4508	} else if (!strcmp(str, "disable")) {
4509	set_schedstats(false);
4510	ret = 1;
4511	}
4512	out:
4513	if (!ret)
4514	pr_warn("Unable to parse schedstats=\n");
4515
4516	return ret;
4517	}
4518	__setup("schedstats=", setup_schedstats);
4519
4520	#ifdef CONFIG_PROC_SYSCTL
4521	static int sysctl_schedstats(const struct ctl_table *table, int write, void *buffer,
4522	size_t *lenp, loff_t *ppos)
4523	{
4524	struct ctl_table t;
4525	int err;
4526	int state = static_branch_likely(&sched_schedstats);
4527
4528	if (write && !capable(CAP_SYS_ADMIN))
4529	return -EPERM;
4530
4531	t = *table;
4532	t.data = &state;
4533	err = proc_dointvec_minmax(table: &t, dir: write, buffer, lenp, ppos);
4534	if (err < 0)
4535	return err;
4536	if (write)
4537	set_schedstats(state);
4538	return err;
4539	}
4540	#endif /* CONFIG_PROC_SYSCTL */
4541	#endif /* CONFIG_SCHEDSTATS */
4542
4543	#ifdef CONFIG_SYSCTL
4544	static const struct ctl_table sched_core_sysctls[] = {
4545	#ifdef CONFIG_SCHEDSTATS
4546	{
4547	.procname = "sched_schedstats",
4548	.data = NULL,
4549	.maxlen = sizeof(unsigned int),
4550	.mode = 0644,
4551	.proc_handler = sysctl_schedstats,
4552	.extra1 = SYSCTL_ZERO,
4553	.extra2 = SYSCTL_ONE,
4554	},
4555	#endif /* CONFIG_SCHEDSTATS */
4556	#ifdef CONFIG_UCLAMP_TASK
4557	{
4558	.procname = "sched_util_clamp_min",
4559	.data = &sysctl_sched_uclamp_util_min,
4560	.maxlen = sizeof(unsigned int),
4561	.mode = 0644,
4562	.proc_handler = sysctl_sched_uclamp_handler,
4563	},
4564	{
4565	.procname = "sched_util_clamp_max",
4566	.data = &sysctl_sched_uclamp_util_max,
4567	.maxlen = sizeof(unsigned int),
4568	.mode = 0644,
4569	.proc_handler = sysctl_sched_uclamp_handler,
4570	},
4571	{
4572	.procname = "sched_util_clamp_min_rt_default",
4573	.data = &sysctl_sched_uclamp_util_min_rt_default,
4574	.maxlen = sizeof(unsigned int),
4575	.mode = 0644,
4576	.proc_handler = sysctl_sched_uclamp_handler,
4577	},
4578	#endif /* CONFIG_UCLAMP_TASK */
4579	#ifdef CONFIG_NUMA_BALANCING
4580	{
4581	.procname = "numa_balancing",
4582	.data = NULL, /* filled in by handler */
4583	.maxlen = sizeof(unsigned int),
4584	.mode = 0644,
4585	.proc_handler = sysctl_numa_balancing,
4586	.extra1 = SYSCTL_ZERO,
4587	.extra2 = SYSCTL_FOUR,
4588	},
4589	#endif /* CONFIG_NUMA_BALANCING */
4590	};
4591	static int __init sched_core_sysctl_init(void)
4592	{
4593	register_sysctl_init("kernel", sched_core_sysctls);
4594	return 0;
4595	}
4596	late_initcall(sched_core_sysctl_init);
4597	#endif /* CONFIG_SYSCTL */
4598
4599	/*
4600	* fork()/clone()-time setup:
4601	*/
4602	int sched_fork(u64 clone_flags, struct task_struct *p)
4603	{
4604	__sched_fork(clone_flags, p);
4605	/*
4606	* We mark the process as NEW here. This guarantees that
4607	* nobody will actually run it, and a signal or other external
4608	* event cannot wake it up and insert it on the runqueue either.
4609	*/
4610	p->__state = TASK_NEW;
4611
4612	/*
4613	* Make sure we do not leak PI boosting priority to the child.
4614	*/
4615	p->prio = current->normal_prio;
4616
4617	uclamp_fork(p);
4618
4619	/*
4620	* Revert to default priority/policy on fork if requested.
4621	*/
4622	if (unlikely(p->sched_reset_on_fork)) {
4623	if (task_has_dl_policy(p) || task_has_rt_policy(p)) {
4624	p->policy = SCHED_NORMAL;
4625	p->static_prio = NICE_TO_PRIO(0);
4626	p->rt_priority = 0;
4627	} else if (PRIO_TO_NICE(p->static_prio) < 0)
4628	p->static_prio = NICE_TO_PRIO(0);
4629
4630	p->prio = p->normal_prio = p->static_prio;
4631	set_load_weight(p, update_load: false);
4632	p->se.custom_slice = 0;
4633	p->se.slice = sysctl_sched_base_slice;
4634
4635	/*
4636	* We don't need the reset flag anymore after the fork. It has
4637	* fulfilled its duty:
4638	*/
4639	p->sched_reset_on_fork = 0;
4640	}
4641
4642	if (dl_prio(prio: p->prio))
4643	return -EAGAIN;
4644
4645	scx_pre_fork(p);
4646
4647	if (rt_prio(prio: p->prio)) {
4648	p->sched_class = &rt_sched_class;
4649	#ifdef CONFIG_SCHED_CLASS_EXT
4650	} else if (task_should_scx(p->policy)) {
4651	p->sched_class = &ext_sched_class;
4652	#endif
4653	} else {
4654	p->sched_class = &fair_sched_class;
4655	}
4656
4657	init_entity_runnable_average(se: &p->se);
4658
4659
4660	#ifdef CONFIG_SCHED_INFO
4661	if (likely(sched_info_on()))
4662	memset(&p->sched_info, 0, sizeof(p->sched_info));
4663	#endif
4664	p->on_cpu = 0;
4665	init_task_preempt_count(p);
4666	plist_node_init(node: &p->pushable_tasks, MAX_PRIO);
4667	RB_CLEAR_NODE(&p->pushable_dl_tasks);
4668
4669	return 0;
4670	}
4671
4672	int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
4673	{
4674	unsigned long flags;
4675
4676	/*
4677	* Because we're not yet on the pid-hash, p->pi_lock isn't strictly
4678	* required yet, but lockdep gets upset if rules are violated.
4679	*/
4680	raw_spin_lock_irqsave(&p->pi_lock, flags);
4681	#ifdef CONFIG_CGROUP_SCHED
4682	if (1) {
4683	struct task_group *tg;
4684	tg = container_of(kargs->cset->subsys[cpu_cgrp_id],
4685	struct task_group, css);
4686	tg = autogroup_task_group(p, tg);
4687	p->sched_task_group = tg;
4688	}
4689	#endif
4690	/*
4691	* We're setting the CPU for the first time, we don't migrate,
4692	* so use __set_task_cpu().
4693	*/
4694	__set_task_cpu(p, smp_processor_id());
4695	if (p->sched_class->task_fork)
4696	p->sched_class->task_fork(p);
4697	raw_spin_unlock_irqrestore(&p->pi_lock, flags);
4698
4699	return scx_fork(p);
4700	}
4701
4702	void sched_cancel_fork(struct task_struct *p)
4703	{
4704	scx_cancel_fork(p);
4705	}
4706
4707	void sched_post_fork(struct task_struct *p)
4708	{
4709	uclamp_post_fork(p);
4710	scx_post_fork(p);
4711	}
4712
4713	unsigned long to_ratio(u64 period, u64 runtime)
4714	{
4715	if (runtime == RUNTIME_INF)
4716	return BW_UNIT;
4717
4718	/*
4719	* Doing this here saves a lot of checks in all
4720	* the calling paths, and returning zero seems
4721	* safe for them anyway.
4722	*/
4723	if (period == 0)
4724	return 0;
4725
4726	return div64_u64(dividend: runtime << BW_SHIFT, divisor: period);
4727	}
4728
4729	/*
4730	* wake_up_new_task - wake up a newly created task for the first time.
4731	*
4732	* This function will do some initial scheduler statistics housekeeping
4733	* that must be done for every newly created context, then puts the task
4734	* on the runqueue and wakes it.
4735	*/
4736	void wake_up_new_task(struct task_struct *p)
4737	{
4738	struct rq_flags rf;
4739	struct rq *rq;
4740	int wake_flags = WF_FORK;
4741
4742	raw_spin_lock_irqsave(&p->pi_lock, rf.flags);
4743	WRITE_ONCE(p->__state, TASK_RUNNING);
4744	/*
4745	* Fork balancing, do it here and not earlier because:
4746	* - cpus_ptr can change in the fork path
4747	* - any previously selected CPU might disappear through hotplug
4748	*
4749	* Use __set_task_cpu() to avoid calling sched_class::migrate_task_rq,
4750	* as we're not fully set-up yet.
4751	*/
4752	p->recent_used_cpu = task_cpu(p);
4753	__set_task_cpu(p, cpu: select_task_rq(p, cpu: task_cpu(p), wake_flags: &wake_flags));
4754	rq = __task_rq_lock(p, rf: &rf);
4755	update_rq_clock(rq);
4756	post_init_entity_util_avg(p);
4757
4758	activate_task(rq, p, ENQUEUE_NOCLOCK | ENQUEUE_INITIAL);
4759	trace_sched_wakeup_new(p);
4760	wakeup_preempt(rq, p, flags: wake_flags);
4761	if (p->sched_class->task_woken) {
4762	/*
4763	* Nothing relies on rq->lock after this, so it's fine to
4764	* drop it.
4765	*/
4766	rq_unpin_lock(rq, rf: &rf);
4767	p->sched_class->task_woken(rq, p);
4768	rq_repin_lock(rq, rf: &rf);
4769	}
4770	task_rq_unlock(rq, p, rf: &rf);
4771	}
4772
4773	#ifdef CONFIG_PREEMPT_NOTIFIERS
4774
4775	static DEFINE_STATIC_KEY_FALSE(preempt_notifier_key);
4776
4777	void preempt_notifier_inc(void)
4778	{
4779	static_branch_inc(&preempt_notifier_key);
4780	}
4781	EXPORT_SYMBOL_GPL(preempt_notifier_inc);
4782
4783	void preempt_notifier_dec(void)
4784	{
4785	static_branch_dec(&preempt_notifier_key);
4786	}
4787	EXPORT_SYMBOL_GPL(preempt_notifier_dec);
4788
4789	/**
4790	* preempt_notifier_register - tell me when current is being preempted & rescheduled
4791	* @notifier: notifier struct to register
4792	*/
4793	void preempt_notifier_register(struct preempt_notifier *notifier)
4794	{
4795	if (!static_branch_unlikely(&preempt_notifier_key))
4796	WARN(1, "registering preempt_notifier while notifiers disabled\n");
4797
4798	hlist_add_head(n: &notifier->link, h: &current->preempt_notifiers);
4799	}
4800	EXPORT_SYMBOL_GPL(preempt_notifier_register);
4801
4802	/**
4803	* preempt_notifier_unregister - no longer interested in preemption notifications
4804	* @notifier: notifier struct to unregister
4805	*
4806	* This is *not* safe to call from within a preemption notifier.
4807	*/
4808	void preempt_notifier_unregister(struct preempt_notifier *notifier)
4809	{
4810	hlist_del(n: &notifier->link);
4811	}
4812	EXPORT_SYMBOL_GPL(preempt_notifier_unregister);
4813
4814	static void __fire_sched_in_preempt_notifiers(struct task_struct *curr)
4815	{
4816	struct preempt_notifier *notifier;
4817
4818	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
4819	notifier->ops->sched_in(notifier, raw_smp_processor_id());
4820	}
4821
4822	static __always_inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
4823	{
4824	if (static_branch_unlikely(&preempt_notifier_key))
4825	__fire_sched_in_preempt_notifiers(curr);
4826	}
4827
4828	static void
4829	__fire_sched_out_preempt_notifiers(struct task_struct *curr,
4830	struct task_struct *next)
4831	{
4832	struct preempt_notifier *notifier;
4833
4834	hlist_for_each_entry(notifier, &curr->preempt_notifiers, link)
4835	notifier->ops->sched_out(notifier, next);
4836	}
4837
4838	static __always_inline void
4839	fire_sched_out_preempt_notifiers(struct task_struct *curr,
4840	struct task_struct *next)
4841	{
4842	if (static_branch_unlikely(&preempt_notifier_key))
4843	__fire_sched_out_preempt_notifiers(curr, next);
4844	}
4845
4846	#else /* !CONFIG_PREEMPT_NOTIFIERS: */
4847
4848	static inline void fire_sched_in_preempt_notifiers(struct task_struct *curr)
4849	{
4850	}
4851
4852	static inline void
4853	fire_sched_out_preempt_notifiers(struct task_struct *curr,
4854	struct task_struct *next)
4855	{
4856	}
4857
4858	#endif /* !CONFIG_PREEMPT_NOTIFIERS */
4859
4860	static inline void prepare_task(struct task_struct *next)
4861	{
4862	/*
4863	* Claim the task as running, we do this before switching to it
4864	* such that any running task will have this set.
4865	*
4866	* See the smp_load_acquire(&p->on_cpu) case in ttwu() and
4867	* its ordering comment.
4868	*/
4869	WRITE_ONCE(next->on_cpu, 1);
4870	}
4871
4872	static inline void finish_task(struct task_struct *prev)
4873	{
4874	/*
4875	* This must be the very last reference to @prev from this CPU. After
4876	* p->on_cpu is cleared, the task can be moved to a different CPU. We
4877	* must ensure this doesn't happen until the switch is completely
4878	* finished.
4879	*
4880	* In particular, the load of prev->state in finish_task_switch() must
4881	* happen before this.
4882	*
4883	* Pairs with the smp_cond_load_acquire() in try_to_wake_up().
4884	*/
4885	smp_store_release(&prev->on_cpu, 0);
4886	}
4887
4888	static void do_balance_callbacks(struct rq *rq, struct balance_callback *head)
4889	{
4890	void (*func)(struct rq *rq);
4891	struct balance_callback *next;
4892
4893	lockdep_assert_rq_held(rq);
4894
4895	while (head) {
4896	func = (void (*)(struct rq *))head->func;
4897	next = head->next;
4898	head->next = NULL;
4899	head = next;
4900
4901	func(rq);
4902	}
4903	}
4904
4905	static void balance_push(struct rq *rq);
4906
4907	/*
4908	* balance_push_callback is a right abuse of the callback interface and plays
4909	* by significantly different rules.
4910	*
4911	* Where the normal balance_callback's purpose is to be ran in the same context
4912	* that queued it (only later, when it's safe to drop rq->lock again),
4913	* balance_push_callback is specifically targeted at __schedule().
4914	*
4915	* This abuse is tolerated because it places all the unlikely/odd cases behind
4916	* a single test, namely: rq->balance_callback == NULL.
4917	*/
4918	struct balance_callback balance_push_callback = {
4919	.next = NULL,
4920	.func = balance_push,
4921	};
4922
4923	static inline struct balance_callback *
4924	__splice_balance_callbacks(struct rq *rq, bool split)
4925	{
4926	struct balance_callback *head = rq->balance_callback;
4927
4928	if (likely(!head))
4929	return NULL;
4930
4931	lockdep_assert_rq_held(rq);
4932	/*
4933	* Must not take balance_push_callback off the list when
4934	* splice_balance_callbacks() and balance_callbacks() are not
4935	* in the same rq->lock section.
4936	*
4937	* In that case it would be possible for __schedule() to interleave
4938	* and observe the list empty.
4939	*/
4940	if (split && head == &balance_push_callback)
4941	head = NULL;
4942	else
4943	rq->balance_callback = NULL;
4944
4945	return head;
4946	}
4947
4948	struct balance_callback *splice_balance_callbacks(struct rq *rq)
4949	{
4950	return __splice_balance_callbacks(rq, split: true);
4951	}
4952
4953	void __balance_callbacks(struct rq *rq, struct rq_flags *rf)
4954	{
4955	if (rf)
4956	rq_unpin_lock(rq, rf);
4957	do_balance_callbacks(rq, head: __splice_balance_callbacks(rq, split: false));
4958	if (rf)
4959	rq_repin_lock(rq, rf);
4960	}
4961
4962	void balance_callbacks(struct rq *rq, struct balance_callback *head)
4963	{
4964	unsigned long flags;
4965
4966	if (unlikely(head)) {
4967	raw_spin_rq_lock_irqsave(rq, flags);
4968	do_balance_callbacks(rq, head);
4969	raw_spin_rq_unlock_irqrestore(rq, flags);
4970	}
4971	}
4972
4973	static inline void
4974	prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf)
4975	{
4976	/*
4977	* Since the runqueue lock will be released by the next
4978	* task (which is an invalid locking op but in the case
4979	* of the scheduler it's an obvious special-case), so we
4980	* do an early lockdep release here:
4981	*/
4982	rq_unpin_lock(rq, rf);
4983	spin_release(&__rq_lockp(rq)->dep_map, _THIS_IP_);
4984	#ifdef CONFIG_DEBUG_SPINLOCK
4985	/* this is a valid case when another task releases the spinlock */
4986	rq_lockp(rq)->owner = next;
4987	#endif
4988	}
4989
4990	static inline void finish_lock_switch(struct rq *rq)
4991	{
4992	/*
4993	* If we are tracking spinlock dependencies then we have to
4994	* fix up the runqueue lock - which gets 'carried over' from
4995	* prev into current:
4996	*/
4997	spin_acquire(&__rq_lockp(rq)->dep_map, 0, 0, _THIS_IP_);
4998	__balance_callbacks(rq, NULL);
4999	raw_spin_rq_unlock_irq(rq);
5000	}
5001
5002	/*
5003	* NOP if the arch has not defined these:
5004	*/
5005
5006	#ifndef prepare_arch_switch
5007	# define prepare_arch_switch(next) do { } while (0)
5008	#endif
5009
5010	#ifndef finish_arch_post_lock_switch
5011	# define finish_arch_post_lock_switch() do { } while (0)
5012	#endif
5013
5014	static inline void kmap_local_sched_out(void)
5015	{
5016	#ifdef CONFIG_KMAP_LOCAL
5017	if (unlikely(current->kmap_ctrl.idx))
5018	__kmap_local_sched_out();
5019	#endif
5020	}
5021
5022	static inline void kmap_local_sched_in(void)
5023	{
5024	#ifdef CONFIG_KMAP_LOCAL
5025	if (unlikely(current->kmap_ctrl.idx))
5026	__kmap_local_sched_in();
5027	#endif
5028	}
5029
5030	/**
5031	* prepare_task_switch - prepare to switch tasks
5032	* @rq: the runqueue preparing to switch
5033	* @prev: the current task that is being switched out
5034	* @next: the task we are going to switch to.
5035	*
5036	* This is called with the rq lock held and interrupts off. It must
5037	* be paired with a subsequent finish_task_switch after the context
5038	* switch.
5039	*
5040	* prepare_task_switch sets up locking and calls architecture specific
5041	* hooks.
5042	*/
5043	static inline void
5044	prepare_task_switch(struct rq *rq, struct task_struct *prev,
5045	struct task_struct *next)
5046	{
5047	kcov_prepare_switch(prev);
5048	sched_info_switch(rq, prev, next);
5049	perf_event_task_sched_out(prev, next);
5050	fire_sched_out_preempt_notifiers(curr: prev, next);
5051	kmap_local_sched_out();
5052	prepare_task(next);
5053	prepare_arch_switch(next);
5054	}
5055
5056	/**
5057	* finish_task_switch - clean up after a task-switch
5058	* @prev: the thread we just switched away from.
5059	*
5060	* finish_task_switch must be called after the context switch, paired
5061	* with a prepare_task_switch call before the context switch.
5062	* finish_task_switch will reconcile locking set up by prepare_task_switch,
5063	* and do any other architecture-specific cleanup actions.
5064	*
5065	* Note that we may have delayed dropping an mm in context_switch(). If
5066	* so, we finish that here outside of the runqueue lock. (Doing it
5067	* with the lock held can cause deadlocks; see schedule() for
5068	* details.)
5069	*
5070	* The context switch have flipped the stack from under us and restored the
5071	* local variables which were saved when this task called schedule() in the
5072	* past. 'prev == current' is still correct but we need to recalculate this_rq
5073	* because prev may have moved to another CPU.
5074	*/
5075	static struct rq *finish_task_switch(struct task_struct *prev)
5076	__releases(rq->lock)
5077	{
5078	struct rq *rq = this_rq();
5079	struct mm_struct *mm = rq->prev_mm;
5080	unsigned int prev_state;
5081
5082	/*
5083	* The previous task will have left us with a preempt_count of 2
5084	* because it left us after:
5085	*
5086	* schedule()
5087	* preempt_disable(); // 1
5088	* __schedule()
5089	* raw_spin_lock_irq(&rq->lock) // 2
5090	*
5091	* Also, see FORK_PREEMPT_COUNT.
5092	*/
5093	if (WARN_ONCE(preempt_count() != 2*PREEMPT_DISABLE_OFFSET,
5094	"corrupted preempt_count: %s/%d/0x%x\n",
5095	current->comm, current->pid, preempt_count()))
5096	preempt_count_set(FORK_PREEMPT_COUNT);
5097
5098	rq->prev_mm = NULL;
5099
5100	/*
5101	* A task struct has one reference for the use as "current".
5102	* If a task dies, then it sets TASK_DEAD in tsk->state and calls
5103	* schedule one last time. The schedule call will never return, and
5104	* the scheduled task must drop that reference.
5105	*
5106	* We must observe prev->state before clearing prev->on_cpu (in
5107	* finish_task), otherwise a concurrent wakeup can get prev
5108	* running on another CPU and we could rave with its RUNNING -> DEAD
5109	* transition, resulting in a double drop.
5110	*/
5111	prev_state = READ_ONCE(prev->__state);
5112	vtime_task_switch(prev);
5113	perf_event_task_sched_in(prev, current);
5114	finish_task(prev);
5115	tick_nohz_task_switch();
5116	finish_lock_switch(rq);
5117	finish_arch_post_lock_switch();
5118	kcov_finish_switch(current);
5119	/*
5120	* kmap_local_sched_out() is invoked with rq::lock held and
5121	* interrupts disabled. There is no requirement for that, but the
5122	* sched out code does not have an interrupt enabled section.
5123	* Restoring the maps on sched in does not require interrupts being
5124	* disabled either.
5125	*/
5126	kmap_local_sched_in();
5127
5128	fire_sched_in_preempt_notifiers(current);
5129	/*
5130	* When switching through a kernel thread, the loop in
5131	* membarrier_{private,global}_expedited() may have observed that
5132	* kernel thread and not issued an IPI. It is therefore possible to
5133	* schedule between user->kernel->user threads without passing though
5134	* switch_mm(). Membarrier requires a barrier after storing to
5135	* rq->curr, before returning to userspace, so provide them here:
5136	*
5137	* - a full memory barrier for {PRIVATE,GLOBAL}_EXPEDITED, implicitly
5138	* provided by mmdrop_lazy_tlb(),
5139	* - a sync_core for SYNC_CORE.
5140	*/
5141	if (mm) {
5142	membarrier_mm_sync_core_before_usermode(mm);
5143	mmdrop_lazy_tlb_sched(mm);
5144	}
5145
5146	if (unlikely(prev_state == TASK_DEAD)) {
5147	if (prev->sched_class->task_dead)
5148	prev->sched_class->task_dead(prev);
5149
5150	/*
5151	* sched_ext_dead() must come before cgroup_task_dead() to
5152	* prevent cgroups from being removed while its member tasks are
5153	* visible to SCX schedulers.
5154	*/
5155	sched_ext_dead(p: prev);
5156	cgroup_task_dead(p: prev);
5157
5158	/* Task is done with its stack. */
5159	put_task_stack(tsk: prev);
5160
5161	put_task_struct_rcu_user(task: prev);
5162	}
5163
5164	return rq;
5165	}
5166
5167	/**
5168	* schedule_tail - first thing a freshly forked thread must call.
5169	* @prev: the thread we just switched away from.
5170	*/
5171	asmlinkage __visible void schedule_tail(struct task_struct *prev)
5172	__releases(rq->lock)
5173	{
5174	/*
5175	* New tasks start with FORK_PREEMPT_COUNT, see there and
5176	* finish_task_switch() for details.
5177	*
5178	* finish_task_switch() will drop rq->lock() and lower preempt_count
5179	* and the preempt_enable() will end up enabling preemption (on
5180	* PREEMPT_COUNT kernels).
5181	*/
5182
5183	finish_task_switch(prev);
5184	/*
5185	* This is a special case: the newly created task has just
5186	* switched the context for the first time. It is returning from
5187	* schedule for the first time in this path.
5188	*/
5189	trace_sched_exit_tp(is_switch: true);
5190	preempt_enable();
5191
5192	if (current->set_child_tid)
5193	put_user(task_pid_vnr(current), current->set_child_tid);
5194
5195	calculate_sigpending();
5196	}
5197
5198	/*
5199	* context_switch - switch to the new MM and the new thread's register state.
5200	*/
5201	static __always_inline struct rq *
5202	context_switch(struct rq *rq, struct task_struct *prev,
5203	struct task_struct *next, struct rq_flags *rf)
5204	{
5205	prepare_task_switch(rq, prev, next);
5206
5207	/*
5208	* For paravirt, this is coupled with an exit in switch_to to
5209	* combine the page table reload and the switch backend into
5210	* one hypercall.
5211	*/
5212	arch_start_context_switch(prev);
5213
5214	/*
5215	* kernel -> kernel lazy + transfer active
5216	* user -> kernel lazy + mmgrab_lazy_tlb() active
5217	*
5218	* kernel -> user switch + mmdrop_lazy_tlb() active
5219	* user -> user switch
5220	*/
5221	if (!next->mm) { // to kernel
5222	enter_lazy_tlb(mm: prev->active_mm, tsk: next);
5223
5224	next->active_mm = prev->active_mm;
5225	if (prev->mm) // from user
5226	mmgrab_lazy_tlb(mm: prev->active_mm);
5227	else
5228	prev->active_mm = NULL;
5229	} else { // to user
5230	membarrier_switch_mm(rq, prev_mm: prev->active_mm, next_mm: next->mm);
5231	/*
5232	* sys_membarrier() requires an smp_mb() between setting
5233	* rq->curr / membarrier_switch_mm() and returning to userspace.
5234	*
5235	* The below provides this either through switch_mm(), or in
5236	* case 'prev->active_mm == next->mm' through
5237	* finish_task_switch()'s mmdrop().
5238	*/
5239	switch_mm_irqs_off(prev: prev->active_mm, next: next->mm, tsk: next);
5240	lru_gen_use_mm(mm: next->mm);
5241
5242	if (!prev->mm) { // from kernel
5243	/* will mmdrop_lazy_tlb() in finish_task_switch(). */
5244	rq->prev_mm = prev->active_mm;
5245	prev->active_mm = NULL;
5246	}
5247	}
5248
5249	mm_cid_switch_to(prev, next);
5250
5251	/*
5252	* Tell rseq that the task was scheduled in. Must be after
5253	* switch_mm_cid() to get the TIF flag set.
5254	*/
5255	rseq_sched_switch_event(t: next);
5256
5257	prepare_lock_switch(rq, next, rf);
5258
5259	/* Here we just switch the register state and the stack. */
5260	switch_to(prev, next, prev);
5261	barrier();
5262
5263	return finish_task_switch(prev);
5264	}
5265
5266	/*
5267	* nr_running and nr_context_switches:
5268	*
5269	* externally visible scheduler statistics: current number of runnable
5270	* threads, total number of context switches performed since bootup.
5271	*/
5272	unsigned int nr_running(void)
5273	{
5274	unsigned int i, sum = 0;
5275
5276	for_each_online_cpu(i)
5277	sum += cpu_rq(i)->nr_running;
5278
5279	return sum;
5280	}
5281
5282	/*
5283	* Check if only the current task is running on the CPU.
5284	*
5285	* Caution: this function does not check that the caller has disabled
5286	* preemption, thus the result might have a time-of-check-to-time-of-use
5287	* race. The caller is responsible to use it correctly, for example:
5288	*
5289	* - from a non-preemptible section (of course)
5290	*
5291	* - from a thread that is bound to a single CPU
5292	*
5293	* - in a loop with very short iterations (e.g. a polling loop)
5294	*/
5295	bool single_task_running(void)
5296	{
5297	return raw_rq()->nr_running == 1;
5298	}
5299	EXPORT_SYMBOL(single_task_running);
5300
5301	unsigned long long nr_context_switches_cpu(int cpu)
5302	{
5303	return cpu_rq(cpu)->nr_switches;
5304	}
5305
5306	unsigned long long nr_context_switches(void)
5307	{
5308	int i;
5309	unsigned long long sum = 0;
5310
5311	for_each_possible_cpu(i)
5312	sum += cpu_rq(i)->nr_switches;
5313
5314	return sum;
5315	}
5316
5317	/*
5318	* Consumers of these two interfaces, like for example the cpuidle menu
5319	* governor, are using nonsensical data. Preferring shallow idle state selection
5320	* for a CPU that has IO-wait which might not even end up running the task when
5321	* it does become runnable.
5322	*/
5323
5324	unsigned int nr_iowait_cpu(int cpu)
5325	{
5326	return atomic_read(v: &cpu_rq(cpu)->nr_iowait);
5327	}
5328
5329	/*
5330	* IO-wait accounting, and how it's mostly bollocks (on SMP).
5331	*
5332	* The idea behind IO-wait account is to account the idle time that we could
5333	* have spend running if it were not for IO. That is, if we were to improve the
5334	* storage performance, we'd have a proportional reduction in IO-wait time.
5335	*
5336	* This all works nicely on UP, where, when a task blocks on IO, we account
5337	* idle time as IO-wait, because if the storage were faster, it could've been
5338	* running and we'd not be idle.
5339	*
5340	* This has been extended to SMP, by doing the same for each CPU. This however
5341	* is broken.
5342	*
5343	* Imagine for instance the case where two tasks block on one CPU, only the one
5344	* CPU will have IO-wait accounted, while the other has regular idle. Even
5345	* though, if the storage were faster, both could've ran at the same time,
5346	* utilising both CPUs.
5347	*
5348	* This means, that when looking globally, the current IO-wait accounting on
5349	* SMP is a lower bound, by reason of under accounting.
5350	*
5351	* Worse, since the numbers are provided per CPU, they are sometimes
5352	* interpreted per CPU, and that is nonsensical. A blocked task isn't strictly
5353	* associated with any one particular CPU, it can wake to another CPU than it
5354	* blocked on. This means the per CPU IO-wait number is meaningless.
5355	*
5356	* Task CPU affinities can make all that even more 'interesting'.
5357	*/
5358
5359	unsigned int nr_iowait(void)
5360	{
5361	unsigned int i, sum = 0;
5362
5363	for_each_possible_cpu(i)
5364	sum += nr_iowait_cpu(cpu: i);
5365
5366	return sum;
5367	}
5368
5369	/*
5370	* sched_exec - execve() is a valuable balancing opportunity, because at
5371	* this point the task has the smallest effective memory and cache footprint.
5372	*/
5373	void sched_exec(void)
5374	{
5375	struct task_struct *p = current;
5376	struct migration_arg arg;
5377	int dest_cpu;
5378
5379	scoped_guard (raw_spinlock_irqsave, &p->pi_lock) {
5380	dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), WF_EXEC);
5381	if (dest_cpu == smp_processor_id())
5382	return;
5383
5384	if (unlikely(!cpu_active(dest_cpu)))
5385	return;
5386
5387	arg = (struct migration_arg){ p, dest_cpu };
5388	}
5389	stop_one_cpu(cpu: task_cpu(p), fn: migration_cpu_stop, arg: &arg);
5390	}
5391
5392	DEFINE_PER_CPU(struct kernel_stat, kstat);
5393	DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat);
5394
5395	EXPORT_PER_CPU_SYMBOL(kstat);
5396	EXPORT_PER_CPU_SYMBOL(kernel_cpustat);
5397
5398	/*
5399	* The function fair_sched_class.update_curr accesses the struct curr
5400	* and its field curr->exec_start; when called from task_sched_runtime(),
5401	* we observe a high rate of cache misses in practice.
5402	* Prefetching this data results in improved performance.
5403	*/
5404	static inline void prefetch_curr_exec_start(struct task_struct *p)
5405	{
5406	#ifdef CONFIG_FAIR_GROUP_SCHED
5407	struct sched_entity *curr = p->se.cfs_rq->curr;
5408	#else
5409	struct sched_entity *curr = task_rq(p)->cfs.curr;
5410	#endif
5411	prefetch(curr);
5412	prefetch(&curr->exec_start);
5413	}
5414
5415	/*
5416	* Return accounted runtime for the task.
5417	* In case the task is currently running, return the runtime plus current's
5418	* pending runtime that have not been accounted yet.
5419	*/
5420	unsigned long long task_sched_runtime(struct task_struct *p)
5421	{
5422	struct rq_flags rf;
5423	struct rq *rq;
5424	u64 ns;
5425
5426	#ifdef CONFIG_64BIT
5427	/*
5428	* 64-bit doesn't need locks to atomically read a 64-bit value.
5429	* So we have a optimization chance when the task's delta_exec is 0.
5430	* Reading ->on_cpu is racy, but this is OK.
5431	*
5432	* If we race with it leaving CPU, we'll take a lock. So we're correct.
5433	* If we race with it entering CPU, unaccounted time is 0. This is
5434	* indistinguishable from the read occurring a few cycles earlier.
5435	* If we see ->on_cpu without ->on_rq, the task is leaving, and has
5436	* been accounted, so we're correct here as well.
5437	*/
5438	if (!p->on_cpu || !task_on_rq_queued(p))
5439	return p->se.sum_exec_runtime;
5440	#endif
5441
5442	rq = task_rq_lock(p, rf: &rf);
5443	/*
5444	* Must be ->curr _and_ ->on_rq. If dequeued, we would
5445	* project cycles that may never be accounted to this
5446	* thread, breaking clock_gettime().
5447	*/
5448	if (task_current_donor(rq, p) && task_on_rq_queued(p)) {
5449	prefetch_curr_exec_start(p);
5450	update_rq_clock(rq);
5451	p->sched_class->update_curr(rq);
5452	}
5453	ns = p->se.sum_exec_runtime;
5454	task_rq_unlock(rq, p, rf: &rf);
5455
5456	return ns;
5457	}
5458
5459	static u64 cpu_resched_latency(struct rq *rq)
5460	{
5461	int latency_warn_ms = READ_ONCE(sysctl_resched_latency_warn_ms);
5462	u64 resched_latency, now = rq_clock(rq);
5463	static bool warned_once;
5464
5465	if (sysctl_resched_latency_warn_once && warned_once)
5466	return 0;
5467
5468	if (!need_resched() || !latency_warn_ms)
5469	return 0;
5470
5471	if (system_state == SYSTEM_BOOTING)
5472	return 0;
5473
5474	if (!rq->last_seen_need_resched_ns) {
5475	rq->last_seen_need_resched_ns = now;
5476	rq->ticks_without_resched = 0;
5477	return 0;
5478	}
5479
5480	rq->ticks_without_resched++;
5481	resched_latency = now - rq->last_seen_need_resched_ns;
5482	if (resched_latency <= latency_warn_ms * NSEC_PER_MSEC)
5483	return 0;
5484
5485	warned_once = true;
5486
5487	return resched_latency;
5488	}
5489
5490	static int __init setup_resched_latency_warn_ms(char *str)
5491	{
5492	long val;
5493
5494	if ((kstrtol(s: str, base: 0, res: &val))) {
5495	pr_warn("Unable to set resched_latency_warn_ms\n");
5496	return 1;
5497	}
5498
5499	sysctl_resched_latency_warn_ms = val;
5500	return 1;
5501	}
5502	__setup("resched_latency_warn_ms=", setup_resched_latency_warn_ms);
5503
5504	/*
5505	* This function gets called by the timer code, with HZ frequency.
5506	* We call it with interrupts disabled.
5507	*/
5508	void sched_tick(void)
5509	{
5510	int cpu = smp_processor_id();
5511	struct rq *rq = cpu_rq(cpu);
5512	/* accounting goes to the donor task */
5513	struct task_struct *donor;
5514	struct rq_flags rf;
5515	unsigned long hw_pressure;
5516	u64 resched_latency;
5517
5518	if (housekeeping_cpu(cpu, type: HK_TYPE_KERNEL_NOISE))
5519	arch_scale_freq_tick();
5520
5521	sched_clock_tick();
5522
5523	rq_lock(rq, rf: &rf);
5524	donor = rq->donor;
5525
5526	psi_account_irqtime(rq, curr: donor, NULL);
5527
5528	update_rq_clock(rq);
5529	hw_pressure = arch_scale_hw_pressure(cpu: cpu_of(rq));
5530	update_hw_load_avg(now: rq_clock_task(rq), rq, capacity: hw_pressure);
5531
5532	if (dynamic_preempt_lazy() && tif_test_bit(TIF_NEED_RESCHED_LAZY))
5533	resched_curr(rq);
5534
5535	donor->sched_class->task_tick(rq, donor, 0);
5536	if (sched_feat(LATENCY_WARN))
5537	resched_latency = cpu_resched_latency(rq);
5538	calc_global_load_tick(this_rq: rq);
5539	sched_core_tick(rq);
5540	scx_tick(rq);
5541
5542	rq_unlock(rq, rf: &rf);
5543
5544	if (sched_feat(LATENCY_WARN) && resched_latency)
5545	resched_latency_warn(cpu, latency: resched_latency);
5546
5547	perf_event_task_tick();
5548
5549	if (donor->flags & PF_WQ_WORKER)
5550	wq_worker_tick(task: donor);
5551
5552	if (!scx_switched_all()) {
5553	rq->idle_balance = idle_cpu(cpu);
5554	sched_balance_trigger(rq);
5555	}
5556	}
5557
5558	#ifdef CONFIG_NO_HZ_FULL
5559
5560	struct tick_work {
5561	int cpu;
5562	atomic_t state;
5563	struct delayed_work work;
5564	};
5565	/* Values for ->state, see diagram below. */
5566	#define TICK_SCHED_REMOTE_OFFLINE 0
5567	#define TICK_SCHED_REMOTE_OFFLINING 1
5568	#define TICK_SCHED_REMOTE_RUNNING 2
5569
5570	/*
5571	* State diagram for ->state:
5572	*
5573	*
5574	* TICK_SCHED_REMOTE_OFFLINE
5575	* | ^
5576	* | |
5577	* | | sched_tick_remote()
5578	* | |
5579	* | |
5580	* +--TICK_SCHED_REMOTE_OFFLINING
5581	* | ^
5582	* | |
5583	* sched_tick_start() | | sched_tick_stop()
5584	* | |
5585	* V |
5586	* TICK_SCHED_REMOTE_RUNNING
5587	*
5588	*
5589	* Other transitions get WARN_ON_ONCE(), except that sched_tick_remote()
5590	* and sched_tick_start() are happy to leave the state in RUNNING.
5591	*/
5592
5593	static struct tick_work __percpu *tick_work_cpu;
5594
5595	static void sched_tick_remote(struct work_struct *work)
5596	{
5597	struct delayed_work *dwork = to_delayed_work(work);
5598	struct tick_work *twork = container_of(dwork, struct tick_work, work);
5599	int cpu = twork->cpu;
5600	struct rq *rq = cpu_rq(cpu);
5601	int os;
5602
5603	/*
5604	* Handle the tick only if it appears the remote CPU is running in full
5605	* dynticks mode. The check is racy by nature, but missing a tick or
5606	* having one too much is no big deal because the scheduler tick updates
5607	* statistics and checks timeslices in a time-independent way, regardless
5608	* of when exactly it is running.
5609	*/
5610	if (tick_nohz_tick_stopped_cpu(cpu)) {
5611	guard(rq_lock_irq)(rq);
5612	struct task_struct *curr = rq->curr;
5613
5614	if (cpu_online(cpu)) {
5615	/*
5616	* Since this is a remote tick for full dynticks mode,
5617	* we are always sure that there is no proxy (only a
5618	* single task is running).
5619	*/
5620	WARN_ON_ONCE(rq->curr != rq->donor);
5621	update_rq_clock(rq);
5622
5623	if (!is_idle_task(curr)) {
5624	/*
5625	* Make sure the next tick runs within a
5626	* reasonable amount of time.
5627	*/
5628	u64 delta = rq_clock_task(rq) - curr->se.exec_start;
5629	WARN_ON_ONCE(delta > (u64)NSEC_PER_SEC * 30);
5630	}
5631	curr->sched_class->task_tick(rq, curr, 0);
5632
5633	calc_load_nohz_remote(rq);
5634	}
5635	}
5636
5637	/*
5638	* Run the remote tick once per second (1Hz). This arbitrary
5639	* frequency is large enough to avoid overload but short enough
5640	* to keep scheduler internal stats reasonably up to date. But
5641	* first update state to reflect hotplug activity if required.
5642	*/
5643	os = atomic_fetch_add_unless(&twork->state, -1, TICK_SCHED_REMOTE_RUNNING);
5644	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_OFFLINE);
5645	if (os == TICK_SCHED_REMOTE_RUNNING)
5646	queue_delayed_work(system_unbound_wq, dwork, HZ);
5647	}
5648
5649	static void sched_tick_start(int cpu)
5650	{
5651	int os;
5652	struct tick_work *twork;
5653
5654	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
5655	return;
5656
5657	WARN_ON_ONCE(!tick_work_cpu);
5658
5659	twork = per_cpu_ptr(tick_work_cpu, cpu);
5660	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_RUNNING);
5661	WARN_ON_ONCE(os == TICK_SCHED_REMOTE_RUNNING);
5662	if (os == TICK_SCHED_REMOTE_OFFLINE) {
5663	twork->cpu = cpu;
5664	INIT_DELAYED_WORK(&twork->work, sched_tick_remote);
5665	queue_delayed_work(system_unbound_wq, &twork->work, HZ);
5666	}
5667	}
5668
5669	#ifdef CONFIG_HOTPLUG_CPU
5670	static void sched_tick_stop(int cpu)
5671	{
5672	struct tick_work *twork;
5673	int os;
5674
5675	if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
5676	return;
5677
5678	WARN_ON_ONCE(!tick_work_cpu);
5679
5680	twork = per_cpu_ptr(tick_work_cpu, cpu);
5681	/* There cannot be competing actions, but don't rely on stop-machine. */
5682	os = atomic_xchg(&twork->state, TICK_SCHED_REMOTE_OFFLINING);
5683	WARN_ON_ONCE(os != TICK_SCHED_REMOTE_RUNNING);
5684	/* Don't cancel, as this would mess up the state machine. */
5685	}
5686	#endif /* CONFIG_HOTPLUG_CPU */
5687
5688	int __init sched_tick_offload_init(void)
5689	{
5690	tick_work_cpu = alloc_percpu(struct tick_work);
5691	BUG_ON(!tick_work_cpu);
5692	return 0;
5693	}
5694
5695	#else /* !CONFIG_NO_HZ_FULL: */
5696	static inline void sched_tick_start(int cpu) { }
5697	static inline void sched_tick_stop(int cpu) { }
5698	#endif /* !CONFIG_NO_HZ_FULL */
5699
5700	#if defined(CONFIG_PREEMPTION) && (defined(CONFIG_DEBUG_PREEMPT) || \
5701	defined(CONFIG_TRACE_PREEMPT_TOGGLE))
5702	/*
5703	* If the value passed in is equal to the current preempt count
5704	* then we just disabled preemption. Start timing the latency.
5705	*/
5706	static inline void preempt_latency_start(int val)
5707	{
5708	if (preempt_count() == val) {
5709	unsigned long ip = get_lock_parent_ip();
5710	#ifdef CONFIG_DEBUG_PREEMPT
5711	current->preempt_disable_ip = ip;
5712	#endif
5713	trace_preempt_off(CALLER_ADDR0, a1: ip);
5714	}
5715	}
5716
5717	void preempt_count_add(int val)
5718	{
5719	#ifdef CONFIG_DEBUG_PREEMPT
5720	/*
5721	* Underflow?
5722	*/
5723	if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
5724	return;
5725	#endif
5726	__preempt_count_add(val);
5727	#ifdef CONFIG_DEBUG_PREEMPT
5728	/*
5729	* Spinlock count overflowing soon?
5730	*/
5731	DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >=
5732	PREEMPT_MASK - 10);
5733	#endif
5734	preempt_latency_start(val);
5735	}
5736	EXPORT_SYMBOL(preempt_count_add);
5737	NOKPROBE_SYMBOL(preempt_count_add);
5738
5739	/*
5740	* If the value passed in equals to the current preempt count
5741	* then we just enabled preemption. Stop timing the latency.
5742	*/
5743	static inline void preempt_latency_stop(int val)
5744	{
5745	if (preempt_count() == val)
5746	trace_preempt_on(CALLER_ADDR0, a1: get_lock_parent_ip());
5747	}
5748
5749	void preempt_count_sub(int val)
5750	{
5751	#ifdef CONFIG_DEBUG_PREEMPT
5752	/*
5753	* Underflow?
5754	*/
5755	if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
5756	return;
5757	/*
5758	* Is the spinlock portion underflowing?
5759	*/
5760	if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) &&
5761	!(preempt_count() & PREEMPT_MASK)))
5762	return;
5763	#endif
5764
5765	preempt_latency_stop(val);
5766	__preempt_count_sub(val);
5767	}
5768	EXPORT_SYMBOL(preempt_count_sub);
5769	NOKPROBE_SYMBOL(preempt_count_sub);
5770
5771	#else
5772	static inline void preempt_latency_start(int val) { }
5773	static inline void preempt_latency_stop(int val) { }
5774	#endif
5775
5776	static inline unsigned long get_preempt_disable_ip(struct task_struct *p)
5777	{
5778	#ifdef CONFIG_DEBUG_PREEMPT
5779	return p->preempt_disable_ip;
5780	#else
5781	return 0;
5782	#endif
5783	}
5784
5785	/*
5786	* Print scheduling while atomic bug:
5787	*/
5788	static noinline void __schedule_bug(struct task_struct *prev)
5789	{
5790	/* Save this before calling printk(), since that will clobber it */
5791	unsigned long preempt_disable_ip = get_preempt_disable_ip(current);
5792
5793	if (oops_in_progress)
5794	return;
5795
5796	printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n",
5797	prev->comm, prev->pid, preempt_count());
5798
5799	debug_show_held_locks(task: prev);
5800	print_modules();
5801	if (irqs_disabled())
5802	print_irqtrace_events(curr: prev);
5803	if (IS_ENABLED(CONFIG_DEBUG_PREEMPT)) {
5804	pr_err("Preemption disabled at:");
5805	print_ip_sym(KERN_ERR, ip: preempt_disable_ip);
5806	}
5807	check_panic_on_warn(origin: "scheduling while atomic");
5808
5809	dump_stack();
5810	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
5811	}
5812
5813	/*
5814	* Various schedule()-time debugging checks and statistics:
5815	*/
5816	static inline void schedule_debug(struct task_struct *prev, bool preempt)
5817	{
5818	#ifdef CONFIG_SCHED_STACK_END_CHECK
5819	if (task_stack_end_corrupted(prev))
5820	panic(fmt: "corrupted stack end detected inside scheduler\n");
5821
5822	if (task_scs_end_corrupted(tsk: prev))
5823	panic(fmt: "corrupted shadow stack detected inside scheduler\n");
5824	#endif
5825
5826	#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
5827	if (!preempt && READ_ONCE(prev->__state) && prev->non_block_count) {
5828	printk(KERN_ERR "BUG: scheduling in a non-blocking section: %s/%d/%i\n",
5829	prev->comm, prev->pid, prev->non_block_count);
5830	dump_stack();
5831	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
5832	}
5833	#endif
5834
5835	if (unlikely(in_atomic_preempt_off())) {
5836	__schedule_bug(prev);
5837	preempt_count_set(PREEMPT_DISABLED);
5838	}
5839	rcu_sleep_check();
5840	WARN_ON_ONCE(ct_state() == CT_STATE_USER);
5841
5842	profile_hit(SCHED_PROFILING, ip: __builtin_return_address(0));
5843
5844	schedstat_inc(this_rq()->sched_count);
5845	}
5846
5847	static void prev_balance(struct rq *rq, struct task_struct *prev,
5848	struct rq_flags *rf)
5849	{
5850	const struct sched_class *start_class = prev->sched_class;
5851	const struct sched_class *class;
5852
5853	/*
5854	* We must do the balancing pass before put_prev_task(), such
5855	* that when we release the rq->lock the task is in the same
5856	* state as before we took rq->lock.
5857	*
5858	* We can terminate the balance pass as soon as we know there is
5859	* a runnable task of @class priority or higher.
5860	*/
5861	for_active_class_range(class, start_class, &idle_sched_class) {
5862	if (class->balance && class->balance(rq, prev, rf))
5863	break;
5864	}
5865	}
5866
5867	/*
5868	* Pick up the highest-prio task:
5869	*/
5870	static inline struct task_struct *
5871	__pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
5872	{
5873	const struct sched_class *class;
5874	struct task_struct *p;
5875
5876	rq->dl_server = NULL;
5877
5878	if (scx_enabled())
5879	goto restart;
5880
5881	/*
5882	* Optimization: we know that if all tasks are in the fair class we can
5883	* call that function directly, but only if the @prev task wasn't of a
5884	* higher scheduling class, because otherwise those lose the
5885	* opportunity to pull in more work from other CPUs.
5886	*/
5887	if (likely(!sched_class_above(prev->sched_class, &fair_sched_class) &&
5888	rq->nr_running == rq->cfs.h_nr_queued)) {
5889
5890	p = pick_next_task_fair(rq, prev, rf);
5891	if (unlikely(p == RETRY_TASK))
5892	goto restart;
5893
5894	/* Assume the next prioritized class is idle_sched_class */
5895	if (!p) {
5896	p = pick_task_idle(rq, rf);
5897	put_prev_set_next_task(rq, prev, next: p);
5898	}
5899
5900	return p;
5901	}
5902
5903	restart:
5904	prev_balance(rq, prev, rf);
5905
5906	for_each_active_class(class) {
5907	if (class->pick_next_task) {
5908	p = class->pick_next_task(rq, prev, rf);
5909	if (unlikely(p == RETRY_TASK))
5910	goto restart;
5911	if (p)
5912	return p;
5913	} else {
5914	p = class->pick_task(rq, rf);
5915	if (unlikely(p == RETRY_TASK))
5916	goto restart;
5917	if (p) {
5918	put_prev_set_next_task(rq, prev, next: p);
5919	return p;
5920	}
5921	}
5922	}
5923
5924	BUG(); /* The idle class should always have a runnable task. */
5925	}
5926
5927	#ifdef CONFIG_SCHED_CORE
5928	static inline bool is_task_rq_idle(struct task_struct *t)
5929	{
5930	return (task_rq(t)->idle == t);
5931	}
5932
5933	static inline bool cookie_equals(struct task_struct *a, unsigned long cookie)
5934	{
5935	return is_task_rq_idle(t: a) || (a->core_cookie == cookie);
5936	}
5937
5938	static inline bool cookie_match(struct task_struct *a, struct task_struct *b)
5939	{
5940	if (is_task_rq_idle(t: a) || is_task_rq_idle(t: b))
5941	return true;
5942
5943	return a->core_cookie == b->core_cookie;
5944	}
5945
5946	/*
5947	* Careful; this can return RETRY_TASK, it does not include the retry-loop
5948	* itself due to the whole SMT pick retry thing below.
5949	*/
5950	static inline struct task_struct *pick_task(struct rq *rq, struct rq_flags *rf)
5951	{
5952	const struct sched_class *class;
5953	struct task_struct *p;
5954
5955	rq->dl_server = NULL;
5956
5957	for_each_active_class(class) {
5958	p = class->pick_task(rq, rf);
5959	if (p)
5960	return p;
5961	}
5962
5963	BUG(); /* The idle class should always have a runnable task. */
5964	}
5965
5966	extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi);
5967
5968	static void queue_core_balance(struct rq *rq);
5969
5970	static struct task_struct *
5971	pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
5972	{
5973	struct task_struct *next, *p, *max;
5974	const struct cpumask *smt_mask;
5975	bool fi_before = false;
5976	bool core_clock_updated = (rq == rq->core);
5977	unsigned long cookie;
5978	int i, cpu, occ = 0;
5979	struct rq *rq_i;
5980	bool need_sync;
5981
5982	if (!sched_core_enabled(rq))
5983	return __pick_next_task(rq, prev, rf);
5984
5985	cpu = cpu_of(rq);
5986
5987	/* Stopper task is switching into idle, no need core-wide selection. */
5988	if (cpu_is_offline(cpu)) {
5989	/*
5990	* Reset core_pick so that we don't enter the fastpath when
5991	* coming online. core_pick would already be migrated to
5992	* another cpu during offline.
5993	*/
5994	rq->core_pick = NULL;
5995	rq->core_dl_server = NULL;
5996	return __pick_next_task(rq, prev, rf);
5997	}
5998
5999	/*
6000	* If there were no {en,de}queues since we picked (IOW, the task
6001	* pointers are all still valid), and we haven't scheduled the last
6002	* pick yet, do so now.
6003	*
6004	* rq->core_pick can be NULL if no selection was made for a CPU because
6005	* it was either offline or went offline during a sibling's core-wide
6006	* selection. In this case, do a core-wide selection.
6007	*/
6008	if (rq->core->core_pick_seq == rq->core->core_task_seq &&
6009	rq->core->core_pick_seq != rq->core_sched_seq &&
6010	rq->core_pick) {
6011	WRITE_ONCE(rq->core_sched_seq, rq->core->core_pick_seq);
6012
6013	next = rq->core_pick;
6014	rq->dl_server = rq->core_dl_server;
6015	rq->core_pick = NULL;
6016	rq->core_dl_server = NULL;
6017	goto out_set_next;
6018	}
6019
6020	prev_balance(rq, prev, rf);
6021
6022	smt_mask = cpu_smt_mask(cpu);
6023	need_sync = !!rq->core->core_cookie;
6024
6025	/* reset state */
6026	rq->core->core_cookie = 0UL;
6027	if (rq->core->core_forceidle_count) {
6028	if (!core_clock_updated) {
6029	update_rq_clock(rq: rq->core);
6030	core_clock_updated = true;
6031	}
6032	sched_core_account_forceidle(rq);
6033	/* reset after accounting force idle */
6034	rq->core->core_forceidle_start = 0;
6035	rq->core->core_forceidle_count = 0;
6036	rq->core->core_forceidle_occupation = 0;
6037	need_sync = true;
6038	fi_before = true;
6039	}
6040
6041	/*
6042	* core->core_task_seq, core->core_pick_seq, rq->core_sched_seq
6043	*
6044	* @task_seq guards the task state ({en,de}queues)
6045	* @pick_seq is the @task_seq we did a selection on
6046	* @sched_seq is the @pick_seq we scheduled
6047	*
6048	* However, preemptions can cause multiple picks on the same task set.
6049	* 'Fix' this by also increasing @task_seq for every pick.
6050	*/
6051	rq->core->core_task_seq++;
6052
6053	/*
6054	* Optimize for common case where this CPU has no cookies
6055	* and there are no cookied tasks running on siblings.
6056	*/
6057	if (!need_sync) {
6058	restart_single:
6059	next = pick_task(rq, rf);
6060	if (unlikely(next == RETRY_TASK))
6061	goto restart_single;
6062	if (!next->core_cookie) {
6063	rq->core_pick = NULL;
6064	rq->core_dl_server = NULL;
6065	/*
6066	* For robustness, update the min_vruntime_fi for
6067	* unconstrained picks as well.
6068	*/
6069	WARN_ON_ONCE(fi_before);
6070	task_vruntime_update(rq, p: next, in_fi: false);
6071	goto out_set_next;
6072	}
6073	}
6074
6075	/*
6076	* For each thread: do the regular task pick and find the max prio task
6077	* amongst them.
6078	*
6079	* Tie-break prio towards the current CPU
6080	*/
6081	restart_multi:
6082	max = NULL;
6083	for_each_cpu_wrap(i, smt_mask, cpu) {
6084	rq_i = cpu_rq(i);
6085
6086	/*
6087	* Current cpu always has its clock updated on entrance to
6088	* pick_next_task(). If the current cpu is not the core,
6089	* the core may also have been updated above.
6090	*/
6091	if (i != cpu && (rq_i != rq->core || !core_clock_updated))
6092	update_rq_clock(rq: rq_i);
6093
6094	p = pick_task(rq: rq_i, rf);
6095	if (unlikely(p == RETRY_TASK))
6096	goto restart_multi;
6097
6098	rq_i->core_pick = p;
6099	rq_i->core_dl_server = rq_i->dl_server;
6100
6101	if (!max || prio_less(a: max, b: p, in_fi: fi_before))
6102	max = p;
6103	}
6104
6105	cookie = rq->core->core_cookie = max->core_cookie;
6106
6107	/*
6108	* For each thread: try and find a runnable task that matches @max or
6109	* force idle.
6110	*/
6111	for_each_cpu(i, smt_mask) {
6112	rq_i = cpu_rq(i);
6113	p = rq_i->core_pick;
6114
6115	if (!cookie_equals(a: p, cookie)) {
6116	p = NULL;
6117	if (cookie)
6118	p = sched_core_find(rq: rq_i, cookie);
6119	if (!p)
6120	p = idle_sched_class.pick_task(rq_i, rf);
6121	}
6122
6123	rq_i->core_pick = p;
6124	rq_i->core_dl_server = NULL;
6125
6126	if (p == rq_i->idle) {
6127	if (rq_i->nr_running) {
6128	rq->core->core_forceidle_count++;
6129	if (!fi_before)
6130	rq->core->core_forceidle_seq++;
6131	}
6132	} else {
6133	occ++;
6134	}
6135	}
6136
6137	if (schedstat_enabled() && rq->core->core_forceidle_count) {
6138	rq->core->core_forceidle_start = rq_clock(rq: rq->core);
6139	rq->core->core_forceidle_occupation = occ;
6140	}
6141
6142	rq->core->core_pick_seq = rq->core->core_task_seq;
6143	next = rq->core_pick;
6144	rq->core_sched_seq = rq->core->core_pick_seq;
6145
6146	/* Something should have been selected for current CPU */
6147	WARN_ON_ONCE(!next);
6148
6149	/*
6150	* Reschedule siblings
6151	*
6152	* NOTE: L1TF -- at this point we're no longer running the old task and
6153	* sending an IPI (below) ensures the sibling will no longer be running
6154	* their task. This ensures there is no inter-sibling overlap between
6155	* non-matching user state.
6156	*/
6157	for_each_cpu(i, smt_mask) {
6158	rq_i = cpu_rq(i);
6159
6160	/*
6161	* An online sibling might have gone offline before a task
6162	* could be picked for it, or it might be offline but later
6163	* happen to come online, but its too late and nothing was
6164	* picked for it. That's Ok - it will pick tasks for itself,
6165	* so ignore it.
6166	*/
6167	if (!rq_i->core_pick)
6168	continue;
6169
6170	/*
6171	* Update for new !FI->FI transitions, or if continuing to be in !FI:
6172	* fi_before fi update?
6173	* 0 0 1
6174	* 0 1 1
6175	* 1 0 1
6176	* 1 1 0
6177	*/
6178	if (!(fi_before && rq->core->core_forceidle_count))
6179	task_vruntime_update(rq: rq_i, p: rq_i->core_pick, in_fi: !!rq->core->core_forceidle_count);
6180
6181	rq_i->core_pick->core_occupation = occ;
6182
6183	if (i == cpu) {
6184	rq_i->core_pick = NULL;
6185	rq_i->core_dl_server = NULL;
6186	continue;
6187	}
6188
6189	/* Did we break L1TF mitigation requirements? */
6190	WARN_ON_ONCE(!cookie_match(next, rq_i->core_pick));
6191
6192	if (rq_i->curr == rq_i->core_pick) {
6193	rq_i->core_pick = NULL;
6194	rq_i->core_dl_server = NULL;
6195	continue;
6196	}
6197
6198	resched_curr(rq: rq_i);
6199	}
6200
6201	out_set_next:
6202	put_prev_set_next_task(rq, prev, next);
6203	if (rq->core->core_forceidle_count && next == rq->idle)
6204	queue_core_balance(rq);
6205
6206	return next;
6207	}
6208
6209	static bool try_steal_cookie(int this, int that)
6210	{
6211	struct rq *dst = cpu_rq(this), *src = cpu_rq(that);
6212	struct task_struct *p;
6213	unsigned long cookie;
6214	bool success = false;
6215
6216	guard(irq)();
6217	guard(double_rq_lock)(lock: dst, lock2: src);
6218
6219	cookie = dst->core->core_cookie;
6220	if (!cookie)
6221	return false;
6222
6223	if (dst->curr != dst->idle)
6224	return false;
6225
6226	p = sched_core_find(rq: src, cookie);
6227	if (!p)
6228	return false;
6229
6230	do {
6231	if (p == src->core_pick || p == src->curr)
6232	goto next;
6233
6234	if (!is_cpu_allowed(p, cpu: this))
6235	goto next;
6236
6237	if (p->core_occupation > dst->idle->core_occupation)
6238	goto next;
6239	/*
6240	* sched_core_find() and sched_core_next() will ensure
6241	* that task @p is not throttled now, we also need to
6242	* check whether the runqueue of the destination CPU is
6243	* being throttled.
6244	*/
6245	if (sched_task_is_throttled(p, cpu: this))
6246	goto next;
6247
6248	move_queued_task_locked(src_rq: src, dst_rq: dst, task: p);
6249	resched_curr(rq: dst);
6250
6251	success = true;
6252	break;
6253
6254	next:
6255	p = sched_core_next(p, cookie);
6256	} while (p);
6257
6258	return success;
6259	}
6260
6261	static bool steal_cookie_task(int cpu, struct sched_domain *sd)
6262	{
6263	int i;
6264
6265	for_each_cpu_wrap(i, sched_domain_span(sd), cpu + 1) {
6266	if (i == cpu)
6267	continue;
6268
6269	if (need_resched())
6270	break;
6271
6272	if (try_steal_cookie(this: cpu, that: i))
6273	return true;
6274	}
6275
6276	return false;
6277	}
6278
6279	static void sched_core_balance(struct rq *rq)
6280	{
6281	struct sched_domain *sd;
6282	int cpu = cpu_of(rq);
6283
6284	guard(preempt)();
6285	guard(rcu)();
6286
6287	raw_spin_rq_unlock_irq(rq);
6288	for_each_domain(cpu, sd) {
6289	if (need_resched())
6290	break;
6291
6292	if (steal_cookie_task(cpu, sd))
6293	break;
6294	}
6295	raw_spin_rq_lock_irq(rq);
6296	}
6297
6298	static DEFINE_PER_CPU(struct balance_callback, core_balance_head);
6299
6300	static void queue_core_balance(struct rq *rq)
6301	{
6302	if (!sched_core_enabled(rq))
6303	return;
6304
6305	if (!rq->core->core_cookie)
6306	return;
6307
6308	if (!rq->nr_running) /* not forced idle */
6309	return;
6310
6311	queue_balance_callback(rq, head: &per_cpu(core_balance_head, rq->cpu), func: sched_core_balance);
6312	}
6313
6314	DEFINE_LOCK_GUARD_1(core_lock, int,
6315	sched_core_lock(*_T->lock, &_T->flags),
6316	sched_core_unlock(*_T->lock, &_T->flags),
6317	unsigned long flags)
6318
6319	static void sched_core_cpu_starting(unsigned int cpu)
6320	{
6321	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
6322	struct rq *rq = cpu_rq(cpu), *core_rq = NULL;
6323	int t;
6324
6325	guard(core_lock)(l: &cpu);
6326
6327	WARN_ON_ONCE(rq->core != rq);
6328
6329	/* if we're the first, we'll be our own leader */
6330	if (cpumask_weight(srcp: smt_mask) == 1)
6331	return;
6332
6333	/* find the leader */
6334	for_each_cpu(t, smt_mask) {
6335	if (t == cpu)
6336	continue;
6337	rq = cpu_rq(t);
6338	if (rq->core == rq) {
6339	core_rq = rq;
6340	break;
6341	}
6342	}
6343
6344	if (WARN_ON_ONCE(!core_rq)) /* whoopsie */
6345	return;
6346
6347	/* install and validate core_rq */
6348	for_each_cpu(t, smt_mask) {
6349	rq = cpu_rq(t);
6350
6351	if (t == cpu)
6352	rq->core = core_rq;
6353
6354	WARN_ON_ONCE(rq->core != core_rq);
6355	}
6356	}
6357
6358	static void sched_core_cpu_deactivate(unsigned int cpu)
6359	{
6360	const struct cpumask *smt_mask = cpu_smt_mask(cpu);
6361	struct rq *rq = cpu_rq(cpu), *core_rq = NULL;
6362	int t;
6363
6364	guard(core_lock)(l: &cpu);
6365
6366	/* if we're the last man standing, nothing to do */
6367	if (cpumask_weight(srcp: smt_mask) == 1) {
6368	WARN_ON_ONCE(rq->core != rq);
6369	return;
6370	}
6371
6372	/* if we're not the leader, nothing to do */
6373	if (rq->core != rq)
6374	return;
6375
6376	/* find a new leader */
6377	for_each_cpu(t, smt_mask) {
6378	if (t == cpu)
6379	continue;
6380	core_rq = cpu_rq(t);
6381	break;
6382	}
6383
6384	if (WARN_ON_ONCE(!core_rq)) /* impossible */
6385	return;
6386
6387	/* copy the shared state to the new leader */
6388	core_rq->core_task_seq = rq->core_task_seq;
6389	core_rq->core_pick_seq = rq->core_pick_seq;
6390	core_rq->core_cookie = rq->core_cookie;
6391	core_rq->core_forceidle_count = rq->core_forceidle_count;
6392	core_rq->core_forceidle_seq = rq->core_forceidle_seq;
6393	core_rq->core_forceidle_occupation = rq->core_forceidle_occupation;
6394
6395	/*
6396	* Accounting edge for forced idle is handled in pick_next_task().
6397	* Don't need another one here, since the hotplug thread shouldn't
6398	* have a cookie.
6399	*/
6400	core_rq->core_forceidle_start = 0;
6401
6402	/* install new leader */
6403	for_each_cpu(t, smt_mask) {
6404	rq = cpu_rq(t);
6405	rq->core = core_rq;
6406	}
6407	}
6408
6409	static inline void sched_core_cpu_dying(unsigned int cpu)
6410	{
6411	struct rq *rq = cpu_rq(cpu);
6412
6413	if (rq->core != rq)
6414	rq->core = rq;
6415	}
6416
6417	#else /* !CONFIG_SCHED_CORE: */
6418
6419	static inline void sched_core_cpu_starting(unsigned int cpu) {}
6420	static inline void sched_core_cpu_deactivate(unsigned int cpu) {}
6421	static inline void sched_core_cpu_dying(unsigned int cpu) {}
6422
6423	static struct task_struct *
6424	pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
6425	{
6426	return __pick_next_task(rq, prev, rf);
6427	}
6428
6429	#endif /* !CONFIG_SCHED_CORE */
6430
6431	/*
6432	* Constants for the sched_mode argument of __schedule().
6433	*
6434	* The mode argument allows RT enabled kernels to differentiate a
6435	* preemption from blocking on an 'sleeping' spin/rwlock.
6436	*/
6437	#define SM_IDLE (-1)
6438	#define SM_NONE 0
6439	#define SM_PREEMPT 1
6440	#define SM_RTLOCK_WAIT 2
6441
6442	/*
6443	* Helper function for __schedule()
6444	*
6445	* Tries to deactivate the task, unless the should_block arg
6446	* is false or if a signal is pending. In the case a signal
6447	* is pending, marks the task's __state as RUNNING (and clear
6448	* blocked_on).
6449	*/
6450	static bool try_to_block_task(struct rq *rq, struct task_struct *p,
6451	unsigned long *task_state_p, bool should_block)
6452	{
6453	unsigned long task_state = *task_state_p;
6454	int flags = DEQUEUE_NOCLOCK;
6455
6456	if (signal_pending_state(state: task_state, p)) {
6457	WRITE_ONCE(p->__state, TASK_RUNNING);
6458	*task_state_p = TASK_RUNNING;
6459	return false;
6460	}
6461
6462	/*
6463	* We check should_block after signal_pending because we
6464	* will want to wake the task in that case. But if
6465	* should_block is false, its likely due to the task being
6466	* blocked on a mutex, and we want to keep it on the runqueue
6467	* to be selectable for proxy-execution.
6468	*/
6469	if (!should_block)
6470	return false;
6471
6472	p->sched_contributes_to_load =
6473	(task_state & TASK_UNINTERRUPTIBLE) &&
6474	!(task_state & TASK_NOLOAD) &&
6475	!(task_state & TASK_FROZEN);
6476
6477	if (unlikely(is_special_task_state(task_state)))
6478	flags |= DEQUEUE_SPECIAL;
6479
6480	/*
6481	* __schedule() ttwu()
6482	* prev_state = prev->state; if (p->on_rq && ...)
6483	* if (prev_state) goto out;
6484	* p->on_rq = 0; smp_acquire__after_ctrl_dep();
6485	* p->state = TASK_WAKING
6486	*
6487	* Where __schedule() and ttwu() have matching control dependencies.
6488	*
6489	* After this, schedule() must not care about p->state any more.
6490	*/
6491	block_task(rq, p, flags);
6492	return true;
6493	}
6494
6495	#ifdef CONFIG_SCHED_PROXY_EXEC
6496	static inline struct task_struct *proxy_resched_idle(struct rq *rq)
6497	{
6498	put_prev_set_next_task(rq, prev: rq->donor, next: rq->idle);
6499	rq_set_donor(rq, t: rq->idle);
6500	set_tsk_need_resched(rq->idle);
6501	return rq->idle;
6502	}
6503
6504	static bool __proxy_deactivate(struct rq *rq, struct task_struct *donor)
6505	{
6506	unsigned long state = READ_ONCE(donor->__state);
6507
6508	/* Don't deactivate if the state has been changed to TASK_RUNNING */
6509	if (state == TASK_RUNNING)
6510	return false;
6511	/*
6512	* Because we got donor from pick_next_task(), it is *crucial*
6513	* that we call proxy_resched_idle() before we deactivate it.
6514	* As once we deactivate donor, donor->on_rq is set to zero,
6515	* which allows ttwu() to immediately try to wake the task on
6516	* another rq. So we cannot use *any* references to donor
6517	* after that point. So things like cfs_rq->curr or rq->donor
6518	* need to be changed from next *before* we deactivate.
6519	*/
6520	proxy_resched_idle(rq);
6521	return try_to_block_task(rq, p: donor, task_state_p: &state, should_block: true);
6522	}
6523
6524	static struct task_struct *proxy_deactivate(struct rq *rq, struct task_struct *donor)
6525	{
6526	if (!__proxy_deactivate(rq, donor)) {
6527	/*
6528	* XXX: For now, if deactivation failed, set donor
6529	* as unblocked, as we aren't doing proxy-migrations
6530	* yet (more logic will be needed then).
6531	*/
6532	donor->blocked_on = NULL;
6533	}
6534	return NULL;
6535	}
6536
6537	/*
6538	* Find runnable lock owner to proxy for mutex blocked donor
6539	*
6540	* Follow the blocked-on relation:
6541	* task->blocked_on -> mutex->owner -> task...
6542	*
6543	* Lock order:
6544	*
6545	* p->pi_lock
6546	* rq->lock
6547	* mutex->wait_lock
6548	*
6549	* Returns the task that is going to be used as execution context (the one
6550	* that is actually going to be run on cpu_of(rq)).
6551	*/
6552	static struct task_struct *
6553	find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
6554	{
6555	struct task_struct *owner = NULL;
6556	int this_cpu = cpu_of(rq);
6557	struct task_struct *p;
6558	struct mutex *mutex;
6559
6560	/* Follow blocked_on chain. */
6561	for (p = donor; task_is_blocked(p); p = owner) {
6562	mutex = p->blocked_on;
6563	/* Something changed in the chain, so pick again */
6564	if (!mutex)
6565	return NULL;
6566	/*
6567	* By taking mutex->wait_lock we hold off concurrent mutex_unlock()
6568	* and ensure @owner sticks around.
6569	*/
6570	guard(raw_spinlock)(l: &mutex->wait_lock);
6571
6572	/* Check again that p is blocked with wait_lock held */
6573	if (mutex != __get_task_blocked_on(p)) {
6574	/*
6575	* Something changed in the blocked_on chain and
6576	* we don't know if only at this level. So, let's
6577	* just bail out completely and let __schedule()
6578	* figure things out (pick_again loop).
6579	*/
6580	return NULL;
6581	}
6582
6583	owner = __mutex_owner(lock: mutex);
6584	if (!owner) {
6585	__clear_task_blocked_on(p, m: mutex);
6586	return p;
6587	}
6588
6589	if (!READ_ONCE(owner->on_rq) || owner->se.sched_delayed) {
6590	/* XXX Don't handle blocked owners/delayed dequeue yet */
6591	return proxy_deactivate(rq, donor);
6592	}
6593
6594	if (task_cpu(p: owner) != this_cpu) {
6595	/* XXX Don't handle migrations yet */
6596	return proxy_deactivate(rq, donor);
6597	}
6598
6599	if (task_on_rq_migrating(p: owner)) {
6600	/*
6601	* One of the chain of mutex owners is currently migrating to this
6602	* CPU, but has not yet been enqueued because we are holding the
6603	* rq lock. As a simple solution, just schedule rq->idle to give
6604	* the migration a chance to complete. Much like the migrate_task
6605	* case we should end up back in find_proxy_task(), this time
6606	* hopefully with all relevant tasks already enqueued.
6607	*/
6608	return proxy_resched_idle(rq);
6609	}
6610
6611	/*
6612	* Its possible to race where after we check owner->on_rq
6613	* but before we check (owner_cpu != this_cpu) that the
6614	* task on another cpu was migrated back to this cpu. In
6615	* that case it could slip by our checks. So double check
6616	* we are still on this cpu and not migrating. If we get
6617	* inconsistent results, try again.
6618	*/
6619	if (!task_on_rq_queued(p: owner) || task_cpu(p: owner) != this_cpu)
6620	return NULL;
6621
6622	if (owner == p) {
6623	/*
6624	* It's possible we interleave with mutex_unlock like:
6625	*
6626	* lock(&rq->lock);
6627	* find_proxy_task()
6628	* mutex_unlock()
6629	* lock(&wait_lock);
6630	* donor(owner) = current->blocked_donor;
6631	* unlock(&wait_lock);
6632	*
6633	* wake_up_q();
6634	* ...
6635	* ttwu_runnable()
6636	* __task_rq_lock()
6637	* lock(&wait_lock);
6638	* owner == p
6639	*
6640	* Which leaves us to finish the ttwu_runnable() and make it go.
6641	*
6642	* So schedule rq->idle so that ttwu_runnable() can get the rq
6643	* lock and mark owner as running.
6644	*/
6645	return proxy_resched_idle(rq);
6646	}
6647	/*
6648	* OK, now we're absolutely sure @owner is on this
6649	* rq, therefore holding @rq->lock is sufficient to
6650	* guarantee its existence, as per ttwu_remote().
6651	*/
6652	}
6653
6654	WARN_ON_ONCE(owner && !owner->on_rq);
6655	return owner;
6656	}
6657	#else /* SCHED_PROXY_EXEC */
6658	static struct task_struct *
6659	find_proxy_task(struct rq *rq, struct task_struct *donor, struct rq_flags *rf)
6660	{
6661	WARN_ONCE(1, "This should never be called in the !SCHED_PROXY_EXEC case\n");
6662	return donor;
6663	}
6664	#endif /* SCHED_PROXY_EXEC */
6665
6666	static inline void proxy_tag_curr(struct rq *rq, struct task_struct *owner)
6667	{
6668	if (!sched_proxy_exec())
6669	return;
6670	/*
6671	* pick_next_task() calls set_next_task() on the chosen task
6672	* at some point, which ensures it is not push/pullable.
6673	* However, the chosen/donor task *and* the mutex owner form an
6674	* atomic pair wrt push/pull.
6675	*
6676	* Make sure owner we run is not pushable. Unfortunately we can
6677	* only deal with that by means of a dequeue/enqueue cycle. :-/
6678	*/
6679	dequeue_task(rq, p: owner, DEQUEUE_NOCLOCK | DEQUEUE_SAVE);
6680	enqueue_task(rq, p: owner, ENQUEUE_NOCLOCK | ENQUEUE_RESTORE);
6681	}
6682
6683	/*
6684	* __schedule() is the main scheduler function.
6685	*
6686	* The main means of driving the scheduler and thus entering this function are:
6687	*
6688	* 1. Explicit blocking: mutex, semaphore, waitqueue, etc.
6689	*
6690	* 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return
6691	* paths. For example, see arch/x86/entry_64.S.
6692	*
6693	* To drive preemption between tasks, the scheduler sets the flag in timer
6694	* interrupt handler sched_tick().
6695	*
6696	* 3. Wakeups don't really cause entry into schedule(). They add a
6697	* task to the run-queue and that's it.
6698	*
6699	* Now, if the new task added to the run-queue preempts the current
6700	* task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets
6701	* called on the nearest possible occasion:
6702	*
6703	* - If the kernel is preemptible (CONFIG_PREEMPTION=y):
6704	*
6705	* - in syscall or exception context, at the next outmost
6706	* preempt_enable(). (this might be as soon as the wake_up()'s
6707	* spin_unlock()!)
6708	*
6709	* - in IRQ context, return from interrupt-handler to
6710	* preemptible context
6711	*
6712	* - If the kernel is not preemptible (CONFIG_PREEMPTION is not set)
6713	* then at the next:
6714	*
6715	* - cond_resched() call
6716	* - explicit schedule() call
6717	* - return from syscall or exception to user-space
6718	* - return from interrupt-handler to user-space
6719	*
6720	* WARNING: must be called with preemption disabled!
6721	*/
6722	static void __sched notrace __schedule(int sched_mode)
6723	{
6724	struct task_struct *prev, *next;
6725	/*
6726	* On PREEMPT_RT kernel, SM_RTLOCK_WAIT is noted
6727	* as a preemption by schedule_debug() and RCU.
6728	*/
6729	bool preempt = sched_mode > SM_NONE;
6730	bool is_switch = false;
6731	unsigned long *switch_count;
6732	unsigned long prev_state;
6733	struct rq_flags rf;
6734	struct rq *rq;
6735	int cpu;
6736
6737	/* Trace preemptions consistently with task switches */
6738	trace_sched_entry_tp(preempt: sched_mode == SM_PREEMPT);
6739
6740	cpu = smp_processor_id();
6741	rq = cpu_rq(cpu);
6742	prev = rq->curr;
6743
6744	schedule_debug(prev, preempt);
6745
6746	if (sched_feat(HRTICK) || sched_feat(HRTICK_DL))
6747	hrtick_clear(rq);
6748
6749	klp_sched_try_switch(curr: prev);
6750
6751	local_irq_disable();
6752	rcu_note_context_switch(preempt);
6753	migrate_disable_switch(rq, p: prev);
6754
6755	/*
6756	* Make sure that signal_pending_state()->signal_pending() below
6757	* can't be reordered with __set_current_state(TASK_INTERRUPTIBLE)
6758	* done by the caller to avoid the race with signal_wake_up():
6759	*
6760	* __set_current_state(@state) signal_wake_up()
6761	* schedule() set_tsk_thread_flag(p, TIF_SIGPENDING)
6762	* wake_up_state(p, state)
6763	* LOCK rq->lock LOCK p->pi_state
6764	* smp_mb__after_spinlock() smp_mb__after_spinlock()
6765	* if (signal_pending_state()) if (p->state & @state)
6766	*
6767	* Also, the membarrier system call requires a full memory barrier
6768	* after coming from user-space, before storing to rq->curr; this
6769	* barrier matches a full barrier in the proximity of the membarrier
6770	* system call exit.
6771	*/
6772	rq_lock(rq, rf: &rf);
6773	smp_mb__after_spinlock();
6774
6775	/* Promote REQ to ACT */
6776	rq->clock_update_flags <<= 1;
6777	update_rq_clock(rq);
6778	rq->clock_update_flags = RQCF_UPDATED;
6779
6780	switch_count = &prev->nivcsw;
6781
6782	/* Task state changes only considers SM_PREEMPT as preemption */
6783	preempt = sched_mode == SM_PREEMPT;
6784
6785	/*
6786	* We must load prev->state once (task_struct::state is volatile), such
6787	* that we form a control dependency vs deactivate_task() below.
6788	*/
6789	prev_state = READ_ONCE(prev->__state);
6790	if (sched_mode == SM_IDLE) {
6791	/* SCX must consult the BPF scheduler to tell if rq is empty */
6792	if (!rq->nr_running && !scx_enabled()) {
6793	next = prev;
6794	goto picked;
6795	}
6796	} else if (!preempt && prev_state) {
6797	/*
6798	* We pass task_is_blocked() as the should_block arg
6799	* in order to keep mutex-blocked tasks on the runqueue
6800	* for slection with proxy-exec (without proxy-exec
6801	* task_is_blocked() will always be false).
6802	*/
6803	try_to_block_task(rq, p: prev, task_state_p: &prev_state,
6804	should_block: !task_is_blocked(p: prev));
6805	switch_count = &prev->nvcsw;
6806	}
6807
6808	pick_again:
6809	next = pick_next_task(rq, prev: rq->donor, rf: &rf);
6810	rq_set_donor(rq, t: next);
6811	if (unlikely(task_is_blocked(next))) {
6812	next = find_proxy_task(rq, donor: next, rf: &rf);
6813	if (!next)
6814	goto pick_again;
6815	if (next == rq->idle)
6816	goto keep_resched;
6817	}
6818	picked:
6819	clear_tsk_need_resched(tsk: prev);
6820	clear_preempt_need_resched();
6821	keep_resched:
6822	rq->last_seen_need_resched_ns = 0;
6823
6824	is_switch = prev != next;
6825	if (likely(is_switch)) {
6826	rq->nr_switches++;
6827	/*
6828	* RCU users of rcu_dereference(rq->curr) may not see
6829	* changes to task_struct made by pick_next_task().
6830	*/
6831	RCU_INIT_POINTER(rq->curr, next);
6832
6833	if (!task_current_donor(rq, p: next))
6834	proxy_tag_curr(rq, owner: next);
6835
6836	/*
6837	* The membarrier system call requires each architecture
6838	* to have a full memory barrier after updating
6839	* rq->curr, before returning to user-space.
6840	*
6841	* Here are the schemes providing that barrier on the
6842	* various architectures:
6843	* - mm ? switch_mm() : mmdrop() for x86, s390, sparc, PowerPC,
6844	* RISC-V. switch_mm() relies on membarrier_arch_switch_mm()
6845	* on PowerPC and on RISC-V.
6846	* - finish_lock_switch() for weakly-ordered
6847	* architectures where spin_unlock is a full barrier,
6848	* - switch_to() for arm64 (weakly-ordered, spin_unlock
6849	* is a RELEASE barrier),
6850	*
6851	* The barrier matches a full barrier in the proximity of
6852	* the membarrier system call entry.
6853	*
6854	* On RISC-V, this barrier pairing is also needed for the
6855	* SYNC_CORE command when switching between processes, cf.
6856	* the inline comments in membarrier_arch_switch_mm().
6857	*/
6858	++*switch_count;
6859
6860	psi_account_irqtime(rq, curr: prev, prev: next);
6861	psi_sched_switch(prev, next, sleep: !task_on_rq_queued(p: prev) ||
6862	prev->se.sched_delayed);
6863
6864	trace_sched_switch(preempt, prev, next, prev_state);
6865
6866	/* Also unlocks the rq: */
6867	rq = context_switch(rq, prev, next, rf: &rf);
6868	} else {
6869	/* In case next was already curr but just got blocked_donor */
6870	if (!task_current_donor(rq, p: next))
6871	proxy_tag_curr(rq, owner: next);
6872
6873	rq_unpin_lock(rq, rf: &rf);
6874	__balance_callbacks(rq, NULL);
6875	raw_spin_rq_unlock_irq(rq);
6876	}
6877	trace_sched_exit_tp(is_switch);
6878	}
6879
6880	void __noreturn do_task_dead(void)
6881	{
6882	/* Causes final put_task_struct in finish_task_switch(): */
6883	set_special_state(TASK_DEAD);
6884
6885	/* Tell freezer to ignore us: */
6886	current->flags |= PF_NOFREEZE;
6887
6888	__schedule(SM_NONE);
6889	BUG();
6890
6891	/* Avoid "noreturn function does return" - but don't continue if BUG() is a NOP: */
6892	for (;;)
6893	cpu_relax();
6894	}
6895
6896	static inline void sched_submit_work(struct task_struct *tsk)
6897	{
6898	static DEFINE_WAIT_OVERRIDE_MAP(sched_map, LD_WAIT_CONFIG);
6899	unsigned int task_flags;
6900
6901	/*
6902	* Establish LD_WAIT_CONFIG context to ensure none of the code called
6903	* will use a blocking primitive -- which would lead to recursion.
6904	*/
6905	lock_map_acquire_try(&sched_map);
6906
6907	task_flags = tsk->flags;
6908	/*
6909	* If a worker goes to sleep, notify and ask workqueue whether it
6910	* wants to wake up a task to maintain concurrency.
6911	*/
6912	if (task_flags & PF_WQ_WORKER)
6913	wq_worker_sleeping(task: tsk);
6914	else if (task_flags & PF_IO_WORKER)
6915	io_wq_worker_sleeping(tsk);
6916
6917	/*
6918	* spinlock and rwlock must not flush block requests. This will
6919	* deadlock if the callback attempts to acquire a lock which is
6920	* already acquired.
6921	*/
6922	WARN_ON_ONCE(current->__state & TASK_RTLOCK_WAIT);
6923
6924	/*
6925	* If we are going to sleep and we have plugged IO queued,
6926	* make sure to submit it to avoid deadlocks.
6927	*/
6928	blk_flush_plug(plug: tsk->plug, async: true);
6929
6930	lock_map_release(&sched_map);
6931	}
6932
6933	static void sched_update_worker(struct task_struct *tsk)
6934	{
6935	if (tsk->flags & (PF_WQ_WORKER | PF_IO_WORKER | PF_BLOCK_TS)) {
6936	if (tsk->flags & PF_BLOCK_TS)
6937	blk_plug_invalidate_ts(tsk);
6938	if (tsk->flags & PF_WQ_WORKER)
6939	wq_worker_running(task: tsk);
6940	else if (tsk->flags & PF_IO_WORKER)
6941	io_wq_worker_running(tsk);
6942	}
6943	}
6944
6945	static __always_inline void __schedule_loop(int sched_mode)
6946	{
6947	do {
6948	preempt_disable();
6949	__schedule(sched_mode);
6950	sched_preempt_enable_no_resched();
6951	} while (need_resched());
6952	}
6953
6954	asmlinkage __visible void __sched schedule(void)
6955	{
6956	struct task_struct *tsk = current;
6957
6958	#ifdef CONFIG_RT_MUTEXES
6959	lockdep_assert(!tsk->sched_rt_mutex);
6960	#endif
6961
6962	if (!task_is_running(tsk))
6963	sched_submit_work(tsk);
6964	__schedule_loop(SM_NONE);
6965	sched_update_worker(tsk);
6966	}
6967	EXPORT_SYMBOL(schedule);
6968
6969	/*
6970	* synchronize_rcu_tasks() makes sure that no task is stuck in preempted
6971	* state (have scheduled out non-voluntarily) by making sure that all
6972	* tasks have either left the run queue or have gone into user space.
6973	* As idle tasks do not do either, they must not ever be preempted
6974	* (schedule out non-voluntarily).
6975	*
6976	* schedule_idle() is similar to schedule_preempt_disable() except that it
6977	* never enables preemption because it does not call sched_submit_work().
6978	*/
6979	void __sched schedule_idle(void)
6980	{
6981	/*
6982	* As this skips calling sched_submit_work(), which the idle task does
6983	* regardless because that function is a NOP when the task is in a
6984	* TASK_RUNNING state, make sure this isn't used someplace that the
6985	* current task can be in any other state. Note, idle is always in the
6986	* TASK_RUNNING state.
6987	*/
6988	WARN_ON_ONCE(current->__state);
6989	do {
6990	__schedule(SM_IDLE);
6991	} while (need_resched());
6992	}
6993
6994	#if defined(CONFIG_CONTEXT_TRACKING_USER) && !defined(CONFIG_HAVE_CONTEXT_TRACKING_USER_OFFSTACK)
6995	asmlinkage __visible void __sched schedule_user(void)
6996	{
6997	/*
6998	* If we come here after a random call to set_need_resched(),
6999	* or we have been woken up remotely but the IPI has not yet arrived,
7000	* we haven't yet exited the RCU idle mode. Do it here manually until
7001	* we find a better solution.
7002	*
7003	* NB: There are buggy callers of this function. Ideally we
7004	* should warn if prev_state != CT_STATE_USER, but that will trigger
7005	* too frequently to make sense yet.
7006	*/
7007	enum ctx_state prev_state = exception_enter();
7008	schedule();
7009	exception_exit(prev_state);
7010	}
7011	#endif
7012
7013	/**
7014	* schedule_preempt_disabled - called with preemption disabled
7015	*
7016	* Returns with preemption disabled. Note: preempt_count must be 1
7017	*/
7018	void __sched schedule_preempt_disabled(void)
7019	{
7020	sched_preempt_enable_no_resched();
7021	schedule();
7022	preempt_disable();
7023	}
7024
7025	#ifdef CONFIG_PREEMPT_RT
7026	void __sched notrace schedule_rtlock(void)
7027	{
7028	__schedule_loop(SM_RTLOCK_WAIT);
7029	}
7030	NOKPROBE_SYMBOL(schedule_rtlock);
7031	#endif
7032
7033	static void __sched notrace preempt_schedule_common(void)
7034	{
7035	do {
7036	/*
7037	* Because the function tracer can trace preempt_count_sub()
7038	* and it also uses preempt_enable/disable_notrace(), if
7039	* NEED_RESCHED is set, the preempt_enable_notrace() called
7040	* by the function tracer will call this function again and
7041	* cause infinite recursion.
7042	*
7043	* Preemption must be disabled here before the function
7044	* tracer can trace. Break up preempt_disable() into two
7045	* calls. One to disable preemption without fear of being
7046	* traced. The other to still record the preemption latency,
7047	* which can also be traced by the function tracer.
7048	*/
7049	preempt_disable_notrace();
7050	preempt_latency_start(val: 1);
7051	__schedule(SM_PREEMPT);
7052	preempt_latency_stop(val: 1);
7053	preempt_enable_no_resched_notrace();
7054
7055	/*
7056	* Check again in case we missed a preemption opportunity
7057	* between schedule and now.
7058	*/
7059	} while (need_resched());
7060	}
7061
7062	#ifdef CONFIG_PREEMPTION
7063	/*
7064	* This is the entry point to schedule() from in-kernel preemption
7065	* off of preempt_enable.
7066	*/
7067	asmlinkage __visible void __sched notrace preempt_schedule(void)
7068	{
7069	/*
7070	* If there is a non-zero preempt_count or interrupts are disabled,
7071	* we do not want to preempt the current task. Just return..
7072	*/
7073	if (likely(!preemptible()))
7074	return;
7075	preempt_schedule_common();
7076	}
7077	NOKPROBE_SYMBOL(preempt_schedule);
7078	EXPORT_SYMBOL(preempt_schedule);
7079
7080	#ifdef CONFIG_PREEMPT_DYNAMIC
7081	# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
7082	# ifndef preempt_schedule_dynamic_enabled
7083	# define preempt_schedule_dynamic_enabled preempt_schedule
7084	# define preempt_schedule_dynamic_disabled NULL
7085	# endif
7086	DEFINE_STATIC_CALL(preempt_schedule, preempt_schedule_dynamic_enabled);
7087	EXPORT_STATIC_CALL_TRAMP(preempt_schedule);
7088	# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
7089	static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule);
7090	void __sched notrace dynamic_preempt_schedule(void)
7091	{
7092	if (!static_branch_unlikely(&sk_dynamic_preempt_schedule))
7093	return;
7094	preempt_schedule();
7095	}
7096	NOKPROBE_SYMBOL(dynamic_preempt_schedule);
7097	EXPORT_SYMBOL(dynamic_preempt_schedule);
7098	# endif
7099	#endif /* CONFIG_PREEMPT_DYNAMIC */
7100
7101	/**
7102	* preempt_schedule_notrace - preempt_schedule called by tracing
7103	*
7104	* The tracing infrastructure uses preempt_enable_notrace to prevent
7105	* recursion and tracing preempt enabling caused by the tracing
7106	* infrastructure itself. But as tracing can happen in areas coming
7107	* from userspace or just about to enter userspace, a preempt enable
7108	* can occur before user_exit() is called. This will cause the scheduler
7109	* to be called when the system is still in usermode.
7110	*
7111	* To prevent this, the preempt_enable_notrace will use this function
7112	* instead of preempt_schedule() to exit user context if needed before
7113	* calling the scheduler.
7114	*/
7115	asmlinkage __visible void __sched notrace preempt_schedule_notrace(void)
7116	{
7117	enum ctx_state prev_ctx;
7118
7119	if (likely(!preemptible()))
7120	return;
7121
7122	do {
7123	/*
7124	* Because the function tracer can trace preempt_count_sub()
7125	* and it also uses preempt_enable/disable_notrace(), if
7126	* NEED_RESCHED is set, the preempt_enable_notrace() called
7127	* by the function tracer will call this function again and
7128	* cause infinite recursion.
7129	*
7130	* Preemption must be disabled here before the function
7131	* tracer can trace. Break up preempt_disable() into two
7132	* calls. One to disable preemption without fear of being
7133	* traced. The other to still record the preemption latency,
7134	* which can also be traced by the function tracer.
7135	*/
7136	preempt_disable_notrace();
7137	preempt_latency_start(val: 1);
7138	/*
7139	* Needs preempt disabled in case user_exit() is traced
7140	* and the tracer calls preempt_enable_notrace() causing
7141	* an infinite recursion.
7142	*/
7143	prev_ctx = exception_enter();
7144	__schedule(SM_PREEMPT);
7145	exception_exit(prev_ctx);
7146
7147	preempt_latency_stop(val: 1);
7148	preempt_enable_no_resched_notrace();
7149	} while (need_resched());
7150	}
7151	EXPORT_SYMBOL_GPL(preempt_schedule_notrace);
7152
7153	#ifdef CONFIG_PREEMPT_DYNAMIC
7154	# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
7155	# ifndef preempt_schedule_notrace_dynamic_enabled
7156	# define preempt_schedule_notrace_dynamic_enabled preempt_schedule_notrace
7157	# define preempt_schedule_notrace_dynamic_disabled NULL
7158	# endif
7159	DEFINE_STATIC_CALL(preempt_schedule_notrace, preempt_schedule_notrace_dynamic_enabled);
7160	EXPORT_STATIC_CALL_TRAMP(preempt_schedule_notrace);
7161	# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
7162	static DEFINE_STATIC_KEY_TRUE(sk_dynamic_preempt_schedule_notrace);
7163	void __sched notrace dynamic_preempt_schedule_notrace(void)
7164	{
7165	if (!static_branch_unlikely(&sk_dynamic_preempt_schedule_notrace))
7166	return;
7167	preempt_schedule_notrace();
7168	}
7169	NOKPROBE_SYMBOL(dynamic_preempt_schedule_notrace);
7170	EXPORT_SYMBOL(dynamic_preempt_schedule_notrace);
7171	# endif
7172	#endif
7173
7174	#endif /* CONFIG_PREEMPTION */
7175
7176	/*
7177	* This is the entry point to schedule() from kernel preemption
7178	* off of IRQ context.
7179	* Note, that this is called and return with IRQs disabled. This will
7180	* protect us against recursive calling from IRQ contexts.
7181	*/
7182	asmlinkage __visible void __sched preempt_schedule_irq(void)
7183	{
7184	enum ctx_state prev_state;
7185
7186	/* Catch callers which need to be fixed */
7187	BUG_ON(preempt_count() || !irqs_disabled());
7188
7189	prev_state = exception_enter();
7190
7191	do {
7192	preempt_disable();
7193	local_irq_enable();
7194	__schedule(SM_PREEMPT);
7195	local_irq_disable();
7196	sched_preempt_enable_no_resched();
7197	} while (need_resched());
7198
7199	exception_exit(prev_ctx: prev_state);
7200	}
7201
7202	int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flags,
7203	void *key)
7204	{
7205	WARN_ON_ONCE(wake_flags & ~(WF_SYNC|WF_CURRENT_CPU));
7206	return try_to_wake_up(p: curr->private, state: mode, wake_flags);
7207	}
7208	EXPORT_SYMBOL(default_wake_function);
7209
7210	const struct sched_class *__setscheduler_class(int policy, int prio)
7211	{
7212	if (dl_prio(prio))
7213	return &dl_sched_class;
7214
7215	if (rt_prio(prio))
7216	return &rt_sched_class;
7217
7218	#ifdef CONFIG_SCHED_CLASS_EXT
7219	if (task_should_scx(policy))
7220	return &ext_sched_class;
7221	#endif
7222
7223	return &fair_sched_class;
7224	}
7225
7226	#ifdef CONFIG_RT_MUTEXES
7227
7228	/*
7229	* Would be more useful with typeof()/auto_type but they don't mix with
7230	* bit-fields. Since it's a local thing, use int. Keep the generic sounding
7231	* name such that if someone were to implement this function we get to compare
7232	* notes.
7233	*/
7234	#define fetch_and_set(x, v) ({ int _x = (x); (x) = (v); _x; })
7235
7236	void rt_mutex_pre_schedule(void)
7237	{
7238	lockdep_assert(!fetch_and_set(current->sched_rt_mutex, 1));
7239	sched_submit_work(current);
7240	}
7241
7242	void rt_mutex_schedule(void)
7243	{
7244	lockdep_assert(current->sched_rt_mutex);
7245	__schedule_loop(SM_NONE);
7246	}
7247
7248	void rt_mutex_post_schedule(void)
7249	{
7250	sched_update_worker(current);
7251	lockdep_assert(fetch_and_set(current->sched_rt_mutex, 0));
7252	}
7253
7254	/*
7255	* rt_mutex_setprio - set the current priority of a task
7256	* @p: task to boost
7257	* @pi_task: donor task
7258	*
7259	* This function changes the 'effective' priority of a task. It does
7260	* not touch ->normal_prio like __setscheduler().
7261	*
7262	* Used by the rt_mutex code to implement priority inheritance
7263	* logic. Call site only calls if the priority of the task changed.
7264	*/
7265	void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
7266	{
7267	int prio, oldprio, queue_flag =
7268	DEQUEUE_SAVE | DEQUEUE_MOVE | DEQUEUE_NOCLOCK;
7269	const struct sched_class *prev_class, *next_class;
7270	struct rq_flags rf;
7271	struct rq *rq;
7272
7273	/* XXX used to be waiter->prio, not waiter->task->prio */
7274	prio = __rt_effective_prio(pi_task, prio: p->normal_prio);
7275
7276	/*
7277	* If nothing changed; bail early.
7278	*/
7279	if (p->pi_top_task == pi_task && prio == p->prio && !dl_prio(prio))
7280	return;
7281
7282	rq = __task_rq_lock(p, rf: &rf);
7283	update_rq_clock(rq);
7284	/*
7285	* Set under pi_lock && rq->lock, such that the value can be used under
7286	* either lock.
7287	*
7288	* Note that there is loads of tricky to make this pointer cache work
7289	* right. rt_mutex_slowunlock()+rt_mutex_postunlock() work together to
7290	* ensure a task is de-boosted (pi_task is set to NULL) before the
7291	* task is allowed to run again (and can exit). This ensures the pointer
7292	* points to a blocked task -- which guarantees the task is present.
7293	*/
7294	p->pi_top_task = pi_task;
7295
7296	/*
7297	* For FIFO/RR we only need to set prio, if that matches we're done.
7298	*/
7299	if (prio == p->prio && !dl_prio(prio))
7300	goto out_unlock;
7301
7302	/*
7303	* Idle task boosting is a no-no in general. There is one
7304	* exception, when PREEMPT_RT and NOHZ is active:
7305	*
7306	* The idle task calls get_next_timer_interrupt() and holds
7307	* the timer wheel base->lock on the CPU and another CPU wants
7308	* to access the timer (probably to cancel it). We can safely
7309	* ignore the boosting request, as the idle CPU runs this code
7310	* with interrupts disabled and will complete the lock
7311	* protected section without being interrupted. So there is no
7312	* real need to boost.
7313	*/
7314	if (unlikely(p == rq->idle)) {
7315	WARN_ON(p != rq->curr);
7316	WARN_ON(p->pi_blocked_on);
7317	goto out_unlock;
7318	}
7319
7320	trace_sched_pi_setprio(tsk: p, pi_task);
7321	oldprio = p->prio;
7322
7323	if (oldprio == prio && !dl_prio(prio))
7324	queue_flag &= ~DEQUEUE_MOVE;
7325
7326	prev_class = p->sched_class;
7327	next_class = __setscheduler_class(policy: p->policy, prio);
7328
7329	if (prev_class != next_class)
7330	queue_flag |= DEQUEUE_CLASS;
7331
7332	scoped_guard (sched_change, p, queue_flag) {
7333	/*
7334	* Boosting condition are:
7335	* 1. -rt task is running and holds mutex A
7336	* --> -dl task blocks on mutex A
7337	*
7338	* 2. -dl task is running and holds mutex A
7339	* --> -dl task blocks on mutex A and could preempt the
7340	* running task
7341	*/
7342	if (dl_prio(prio)) {
7343	if (!dl_prio(prio: p->normal_prio) ||
7344	(pi_task && dl_prio(prio: pi_task->prio) &&
7345	dl_entity_preempt(a: &pi_task->dl, b: &p->dl))) {
7346	p->dl.pi_se = pi_task->dl.pi_se;
7347	scope->flags |= ENQUEUE_REPLENISH;
7348	} else {
7349	p->dl.pi_se = &p->dl;
7350	}
7351	} else if (rt_prio(prio)) {
7352	if (dl_prio(prio: oldprio))
7353	p->dl.pi_se = &p->dl;
7354	if (oldprio < prio)
7355	scope->flags |= ENQUEUE_HEAD;
7356	} else {
7357	if (dl_prio(prio: oldprio))
7358	p->dl.pi_se = &p->dl;
7359	if (rt_prio(prio: oldprio))
7360	p->rt.timeout = 0;
7361	}
7362
7363	p->sched_class = next_class;
7364	p->prio = prio;
7365	}
7366	out_unlock:
7367	/* Caller holds task_struct::pi_lock, IRQs are still disabled */
7368
7369	__balance_callbacks(rq, rf: &rf);
7370	__task_rq_unlock(rq, p, rf: &rf);
7371	}
7372	#endif /* CONFIG_RT_MUTEXES */
7373
7374	#if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC)
7375	int __sched __cond_resched(void)
7376	{
7377	if (should_resched(preempt_offset: 0) && !irqs_disabled()) {
7378	preempt_schedule_common();
7379	return 1;
7380	}
7381	/*
7382	* In PREEMPT_RCU kernels, ->rcu_read_lock_nesting tells the tick
7383	* whether the current CPU is in an RCU read-side critical section,
7384	* so the tick can report quiescent states even for CPUs looping
7385	* in kernel context. In contrast, in non-preemptible kernels,
7386	* RCU readers leave no in-memory hints, which means that CPU-bound
7387	* processes executing in kernel context might never report an
7388	* RCU quiescent state. Therefore, the following code causes
7389	* cond_resched() to report a quiescent state, but only when RCU
7390	* is in urgent need of one.
7391	* A third case, preemptible, but non-PREEMPT_RCU provides for
7392	* urgently needed quiescent states via rcu_flavor_sched_clock_irq().
7393	*/
7394	#ifndef CONFIG_PREEMPT_RCU
7395	rcu_all_qs();
7396	#endif
7397	return 0;
7398	}
7399	EXPORT_SYMBOL(__cond_resched);
7400	#endif
7401
7402	#ifdef CONFIG_PREEMPT_DYNAMIC
7403	# ifdef CONFIG_HAVE_PREEMPT_DYNAMIC_CALL
7404	# define cond_resched_dynamic_enabled __cond_resched
7405	# define cond_resched_dynamic_disabled ((void *)&__static_call_return0)
7406	DEFINE_STATIC_CALL_RET0(cond_resched, __cond_resched);
7407	EXPORT_STATIC_CALL_TRAMP(cond_resched);
7408
7409	# define might_resched_dynamic_enabled __cond_resched
7410	# define might_resched_dynamic_disabled ((void *)&__static_call_return0)
7411	DEFINE_STATIC_CALL_RET0(might_resched, __cond_resched);
7412	EXPORT_STATIC_CALL_TRAMP(might_resched);
7413	# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
7414	static DEFINE_STATIC_KEY_FALSE(sk_dynamic_cond_resched);
7415	int __sched dynamic_cond_resched(void)
7416	{
7417	if (!static_branch_unlikely(&sk_dynamic_cond_resched))
7418	return 0;
7419	return __cond_resched();
7420	}
7421	EXPORT_SYMBOL(dynamic_cond_resched);
7422
7423	static DEFINE_STATIC_KEY_FALSE(sk_dynamic_might_resched);
7424	int __sched dynamic_might_resched(void)
7425	{
7426	if (!static_branch_unlikely(&sk_dynamic_might_resched))
7427	return 0;
7428	return __cond_resched();
7429	}
7430	EXPORT_SYMBOL(dynamic_might_resched);
7431	# endif
7432	#endif /* CONFIG_PREEMPT_DYNAMIC */
7433
7434	/*
7435	* __cond_resched_lock() - if a reschedule is pending, drop the given lock,
7436	* call schedule, and on return reacquire the lock.
7437	*
7438	* This works OK both with and without CONFIG_PREEMPTION. We do strange low-level
7439	* operations here to prevent schedule() from being called twice (once via
7440	* spin_unlock(), once by hand).
7441	*/
7442	int __cond_resched_lock(spinlock_t *lock)
7443	{
7444	int resched = should_resched(PREEMPT_LOCK_OFFSET);
7445	int ret = 0;
7446
7447	lockdep_assert_held(lock);
7448
7449	if (spin_needbreak(lock) || resched) {
7450	spin_unlock(lock);
7451	if (!_cond_resched())
7452	cpu_relax();
7453	ret = 1;
7454	spin_lock(lock);
7455	}
7456	return ret;
7457	}
7458	EXPORT_SYMBOL(__cond_resched_lock);
7459
7460	int __cond_resched_rwlock_read(rwlock_t *lock)
7461	{
7462	int resched = should_resched(PREEMPT_LOCK_OFFSET);
7463	int ret = 0;
7464
7465	lockdep_assert_held_read(lock);
7466
7467	if (rwlock_needbreak(lock) || resched) {
7468	read_unlock(lock);
7469	if (!_cond_resched())
7470	cpu_relax();
7471	ret = 1;
7472	read_lock(lock);
7473	}
7474	return ret;
7475	}
7476	EXPORT_SYMBOL(__cond_resched_rwlock_read);
7477
7478	int __cond_resched_rwlock_write(rwlock_t *lock)
7479	{
7480	int resched = should_resched(PREEMPT_LOCK_OFFSET);
7481	int ret = 0;
7482
7483	lockdep_assert_held_write(lock);
7484
7485	if (rwlock_needbreak(lock) || resched) {
7486	write_unlock(lock);
7487	if (!_cond_resched())
7488	cpu_relax();
7489	ret = 1;
7490	write_lock(lock);
7491	}
7492	return ret;
7493	}
7494	EXPORT_SYMBOL(__cond_resched_rwlock_write);
7495
7496	#ifdef CONFIG_PREEMPT_DYNAMIC
7497
7498	# ifdef CONFIG_GENERIC_IRQ_ENTRY
7499	# include <linux/irq-entry-common.h>
7500	# endif
7501
7502	/*
7503	* SC:cond_resched
7504	* SC:might_resched
7505	* SC:preempt_schedule
7506	* SC:preempt_schedule_notrace
7507	* SC:irqentry_exit_cond_resched
7508	*
7509	*
7510	* NONE:
7511	* cond_resched <- __cond_resched
7512	* might_resched <- RET0
7513	* preempt_schedule <- NOP
7514	* preempt_schedule_notrace <- NOP
7515	* irqentry_exit_cond_resched <- NOP
7516	* dynamic_preempt_lazy <- false
7517	*
7518	* VOLUNTARY:
7519	* cond_resched <- __cond_resched
7520	* might_resched <- __cond_resched
7521	* preempt_schedule <- NOP
7522	* preempt_schedule_notrace <- NOP
7523	* irqentry_exit_cond_resched <- NOP
7524	* dynamic_preempt_lazy <- false
7525	*
7526	* FULL:
7527	* cond_resched <- RET0
7528	* might_resched <- RET0
7529	* preempt_schedule <- preempt_schedule
7530	* preempt_schedule_notrace <- preempt_schedule_notrace
7531	* irqentry_exit_cond_resched <- irqentry_exit_cond_resched
7532	* dynamic_preempt_lazy <- false
7533	*
7534	* LAZY:
7535	* cond_resched <- RET0
7536	* might_resched <- RET0
7537	* preempt_schedule <- preempt_schedule
7538	* preempt_schedule_notrace <- preempt_schedule_notrace
7539	* irqentry_exit_cond_resched <- irqentry_exit_cond_resched
7540	* dynamic_preempt_lazy <- true
7541	*/
7542
7543	enum {
7544	preempt_dynamic_undefined = -1,
7545	preempt_dynamic_none,
7546	preempt_dynamic_voluntary,
7547	preempt_dynamic_full,
7548	preempt_dynamic_lazy,
7549	};
7550
7551	int preempt_dynamic_mode = preempt_dynamic_undefined;
7552
7553	int sched_dynamic_mode(const char *str)
7554	{
7555	# ifndef CONFIG_PREEMPT_RT
7556	if (!strcmp(str, "none"))
7557	return preempt_dynamic_none;
7558
7559	if (!strcmp(str, "voluntary"))
7560	return preempt_dynamic_voluntary;
7561	# endif
7562
7563	if (!strcmp(str, "full"))
7564	return preempt_dynamic_full;
7565
7566	# ifdef CONFIG_ARCH_HAS_PREEMPT_LAZY
7567	if (!strcmp(str, "lazy"))
7568	return preempt_dynamic_lazy;
7569	# endif
7570
7571	return -EINVAL;
7572	}
7573
7574	# define preempt_dynamic_key_enable(f) static_key_enable(&sk_dynamic_##f.key)
7575	# define preempt_dynamic_key_disable(f) static_key_disable(&sk_dynamic_##f.key)
7576
7577	# if defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL)
7578	# define preempt_dynamic_enable(f) static_call_update(f, f##_dynamic_enabled)
7579	# define preempt_dynamic_disable(f) static_call_update(f, f##_dynamic_disabled)
7580	# elif defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY)
7581	# define preempt_dynamic_enable(f) preempt_dynamic_key_enable(f)
7582	# define preempt_dynamic_disable(f) preempt_dynamic_key_disable(f)
7583	# else
7584	# error "Unsupported PREEMPT_DYNAMIC mechanism"
7585	# endif
7586
7587	static DEFINE_MUTEX(sched_dynamic_mutex);
7588
7589	static void __sched_dynamic_update(int mode)
7590	{
7591	/*
7592	* Avoid {NONE,VOLUNTARY} -> FULL transitions from ever ending up in
7593	* the ZERO state, which is invalid.
7594	*/
7595	preempt_dynamic_enable(cond_resched);
7596	preempt_dynamic_enable(might_resched);
7597	preempt_dynamic_enable(preempt_schedule);
7598	preempt_dynamic_enable(preempt_schedule_notrace);
7599	preempt_dynamic_enable(irqentry_exit_cond_resched);
7600	preempt_dynamic_key_disable(preempt_lazy);
7601
7602	switch (mode) {
7603	case preempt_dynamic_none:
7604	preempt_dynamic_enable(cond_resched);
7605	preempt_dynamic_disable(might_resched);
7606	preempt_dynamic_disable(preempt_schedule);
7607	preempt_dynamic_disable(preempt_schedule_notrace);
7608	preempt_dynamic_disable(irqentry_exit_cond_resched);
7609	preempt_dynamic_key_disable(preempt_lazy);
7610	if (mode != preempt_dynamic_mode)
7611	pr_info("Dynamic Preempt: none\n");
7612	break;
7613
7614	case preempt_dynamic_voluntary:
7615	preempt_dynamic_enable(cond_resched);
7616	preempt_dynamic_enable(might_resched);
7617	preempt_dynamic_disable(preempt_schedule);
7618	preempt_dynamic_disable(preempt_schedule_notrace);
7619	preempt_dynamic_disable(irqentry_exit_cond_resched);
7620	preempt_dynamic_key_disable(preempt_lazy);
7621	if (mode != preempt_dynamic_mode)
7622	pr_info("Dynamic Preempt: voluntary\n");
7623	break;
7624
7625	case preempt_dynamic_full:
7626	preempt_dynamic_disable(cond_resched);
7627	preempt_dynamic_disable(might_resched);
7628	preempt_dynamic_enable(preempt_schedule);
7629	preempt_dynamic_enable(preempt_schedule_notrace);
7630	preempt_dynamic_enable(irqentry_exit_cond_resched);
7631	preempt_dynamic_key_disable(preempt_lazy);
7632	if (mode != preempt_dynamic_mode)
7633	pr_info("Dynamic Preempt: full\n");
7634	break;
7635
7636	case preempt_dynamic_lazy:
7637	preempt_dynamic_disable(cond_resched);
7638	preempt_dynamic_disable(might_resched);
7639	preempt_dynamic_enable(preempt_schedule);
7640	preempt_dynamic_enable(preempt_schedule_notrace);
7641	preempt_dynamic_enable(irqentry_exit_cond_resched);
7642	preempt_dynamic_key_enable(preempt_lazy);
7643	if (mode != preempt_dynamic_mode)
7644	pr_info("Dynamic Preempt: lazy\n");
7645	break;
7646	}
7647
7648	preempt_dynamic_mode = mode;
7649	}
7650
7651	void sched_dynamic_update(int mode)
7652	{
7653	mutex_lock(&sched_dynamic_mutex);
7654	__sched_dynamic_update(mode);
7655	mutex_unlock(lock: &sched_dynamic_mutex);
7656	}
7657
7658	static int __init setup_preempt_mode(char *str)
7659	{
7660	int mode = sched_dynamic_mode(str);
7661	if (mode < 0) {
7662	pr_warn("Dynamic Preempt: unsupported mode: %s\n", str);
7663	return 0;
7664	}
7665
7666	sched_dynamic_update(mode);
7667	return 1;
7668	}
7669	__setup("preempt=", setup_preempt_mode);
7670
7671	static void __init preempt_dynamic_init(void)
7672	{
7673	if (preempt_dynamic_mode == preempt_dynamic_undefined) {
7674	if (IS_ENABLED(CONFIG_PREEMPT_NONE)) {
7675	sched_dynamic_update(mode: preempt_dynamic_none);
7676	} else if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY)) {
7677	sched_dynamic_update(mode: preempt_dynamic_voluntary);
7678	} else if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
7679	sched_dynamic_update(mode: preempt_dynamic_lazy);
7680	} else {
7681	/* Default static call setting, nothing to do */
7682	WARN_ON_ONCE(!IS_ENABLED(CONFIG_PREEMPT));
7683	preempt_dynamic_mode = preempt_dynamic_full;
7684	pr_info("Dynamic Preempt: full\n");
7685	}
7686	}
7687	}
7688
7689	# define PREEMPT_MODEL_ACCESSOR(mode) \
7690	bool preempt_model_##mode(void) \
7691	{ \
7692	WARN_ON_ONCE(preempt_dynamic_mode == preempt_dynamic_undefined); \
7693	return preempt_dynamic_mode == preempt_dynamic_##mode; \
7694	} \
7695	EXPORT_SYMBOL_GPL(preempt_model_##mode)
7696
7697	PREEMPT_MODEL_ACCESSOR(none);
7698	PREEMPT_MODEL_ACCESSOR(voluntary);
7699	PREEMPT_MODEL_ACCESSOR(full);
7700	PREEMPT_MODEL_ACCESSOR(lazy);
7701
7702	#else /* !CONFIG_PREEMPT_DYNAMIC: */
7703
7704	#define preempt_dynamic_mode -1
7705
7706	static inline void preempt_dynamic_init(void) { }
7707
7708	#endif /* CONFIG_PREEMPT_DYNAMIC */
7709
7710	const char *preempt_modes[] = {
7711	"none", "voluntary", "full", "lazy", NULL,
7712	};
7713
7714	const char *preempt_model_str(void)
7715	{
7716	bool brace = IS_ENABLED(CONFIG_PREEMPT_RT) &&
7717	(IS_ENABLED(CONFIG_PREEMPT_DYNAMIC) ||
7718	IS_ENABLED(CONFIG_PREEMPT_LAZY));
7719	static char buf[128];
7720
7721	if (IS_ENABLED(CONFIG_PREEMPT_BUILD)) {
7722	struct seq_buf s;
7723
7724	seq_buf_init(s: &s, buf, size: sizeof(buf));
7725	seq_buf_puts(s: &s, str: "PREEMPT");
7726
7727	if (IS_ENABLED(CONFIG_PREEMPT_RT))
7728	seq_buf_printf(s: &s, fmt: "%sRT%s",
7729	brace ? "_{" : "_",
7730	brace ? "," : "");
7731
7732	if (IS_ENABLED(CONFIG_PREEMPT_DYNAMIC)) {
7733	seq_buf_printf(s: &s, fmt: "(%s)%s",
7734	preempt_dynamic_mode >= 0 ?
7735	preempt_modes[preempt_dynamic_mode] : "undef",
7736	brace ? "}" : "");
7737	return seq_buf_str(s: &s);
7738	}
7739
7740	if (IS_ENABLED(CONFIG_PREEMPT_LAZY)) {
7741	seq_buf_printf(s: &s, fmt: "LAZY%s",
7742	brace ? "}" : "");
7743	return seq_buf_str(s: &s);
7744	}
7745
7746	return seq_buf_str(s: &s);
7747	}
7748
7749	if (IS_ENABLED(CONFIG_PREEMPT_VOLUNTARY_BUILD))
7750	return "VOLUNTARY";
7751
7752	return "NONE";
7753	}
7754
7755	int io_schedule_prepare(void)
7756	{
7757	int old_iowait = current->in_iowait;
7758
7759	current->in_iowait = 1;
7760	blk_flush_plug(current->plug, async: true);
7761	return old_iowait;
7762	}
7763
7764	void io_schedule_finish(int token)
7765	{
7766	current->in_iowait = token;
7767	}
7768
7769	/*
7770	* This task is about to go to sleep on IO. Increment rq->nr_iowait so
7771	* that process accounting knows that this is a task in IO wait state.
7772	*/
7773	long __sched io_schedule_timeout(long timeout)
7774	{
7775	int token;
7776	long ret;
7777
7778	token = io_schedule_prepare();
7779	ret = schedule_timeout(timeout);
7780	io_schedule_finish(token);
7781
7782	return ret;
7783	}
7784	EXPORT_SYMBOL(io_schedule_timeout);
7785
7786	void __sched io_schedule(void)
7787	{
7788	int token;
7789
7790	token = io_schedule_prepare();
7791	schedule();
7792	io_schedule_finish(token);
7793	}
7794	EXPORT_SYMBOL(io_schedule);
7795
7796	void sched_show_task(struct task_struct *p)
7797	{
7798	unsigned long free;
7799	int ppid;
7800
7801	if (!try_get_task_stack(tsk: p))
7802	return;
7803
7804	pr_info("task:%-15.15s state:%c", p->comm, task_state_to_char(p));
7805
7806	if (task_is_running(p))
7807	pr_cont(" running task ");
7808	free = stack_not_used(p);
7809	ppid = 0;
7810	rcu_read_lock();
7811	if (pid_alive(p))
7812	ppid = task_pid_nr(rcu_dereference(p->real_parent));
7813	rcu_read_unlock();
7814	pr_cont(" stack:%-5lu pid:%-5d tgid:%-5d ppid:%-6d task_flags:0x%04x flags:0x%08lx\n",
7815	free, task_pid_nr(p), task_tgid_nr(p),
7816	ppid, p->flags, read_task_thread_flags(p));
7817
7818	print_worker_info(KERN_INFO, task: p);
7819	print_stop_info(KERN_INFO, task: p);
7820	print_scx_info(KERN_INFO, p);
7821	show_stack(task: p, NULL, KERN_INFO);
7822	put_task_stack(tsk: p);
7823	}
7824	EXPORT_SYMBOL_GPL(sched_show_task);
7825
7826	static inline bool
7827	state_filter_match(unsigned long state_filter, struct task_struct *p)
7828	{
7829	unsigned int state = READ_ONCE(p->__state);
7830
7831	/* no filter, everything matches */
7832	if (!state_filter)
7833	return true;
7834
7835	/* filter, but doesn't match */
7836	if (!(state & state_filter))
7837	return false;
7838
7839	/*
7840	* When looking for TASK_UNINTERRUPTIBLE skip TASK_IDLE (allows
7841	* TASK_KILLABLE).
7842	*/
7843	if (state_filter == TASK_UNINTERRUPTIBLE && (state & TASK_NOLOAD))
7844	return false;
7845
7846	return true;
7847	}
7848
7849
7850	void show_state_filter(unsigned int state_filter)
7851	{
7852	struct task_struct *g, *p;
7853
7854	rcu_read_lock();
7855	for_each_process_thread(g, p) {
7856	/*
7857	* reset the NMI-timeout, listing all files on a slow
7858	* console might take a lot of time:
7859	* Also, reset softlockup watchdogs on all CPUs, because
7860	* another CPU might be blocked waiting for us to process
7861	* an IPI.
7862	*/
7863	touch_nmi_watchdog();
7864	touch_all_softlockup_watchdogs();
7865	if (state_filter_match(state_filter, p))
7866	sched_show_task(p);
7867	}
7868
7869	if (!state_filter)
7870	sysrq_sched_debug_show();
7871
7872	rcu_read_unlock();
7873	/*
7874	* Only show locks if all tasks are dumped:
7875	*/
7876	if (!state_filter)
7877	debug_show_all_locks();
7878	}
7879
7880	/**
7881	* init_idle - set up an idle thread for a given CPU
7882	* @idle: task in question
7883	* @cpu: CPU the idle task belongs to
7884	*
7885	* NOTE: this function does not set the idle thread's NEED_RESCHED
7886	* flag, to make booting more robust.
7887	*/
7888	void __init init_idle(struct task_struct *idle, int cpu)
7889	{
7890	struct affinity_context ac = (struct affinity_context) {
7891	.new_mask = cpumask_of(cpu),
7892	.flags = 0,
7893	};
7894	struct rq *rq = cpu_rq(cpu);
7895	unsigned long flags;
7896
7897	raw_spin_lock_irqsave(&idle->pi_lock, flags);
7898	raw_spin_rq_lock(rq);
7899
7900	idle->__state = TASK_RUNNING;
7901	idle->se.exec_start = sched_clock();
7902	/*
7903	* PF_KTHREAD should already be set at this point; regardless, make it
7904	* look like a proper per-CPU kthread.
7905	*/
7906	idle->flags |= PF_KTHREAD | PF_NO_SETAFFINITY;
7907	kthread_set_per_cpu(k: idle, cpu);
7908
7909	/*
7910	* No validation and serialization required at boot time and for
7911	* setting up the idle tasks of not yet online CPUs.
7912	*/
7913	set_cpus_allowed_common(p: idle, ctx: &ac);
7914	/*
7915	* We're having a chicken and egg problem, even though we are
7916	* holding rq->lock, the CPU isn't yet set to this CPU so the
7917	* lockdep check in task_group() will fail.
7918	*
7919	* Similar case to sched_fork(). / Alternatively we could
7920	* use task_rq_lock() here and obtain the other rq->lock.
7921	*
7922	* Silence PROVE_RCU
7923	*/
7924	rcu_read_lock();
7925	__set_task_cpu(p: idle, cpu);
7926	rcu_read_unlock();
7927
7928	rq->idle = idle;
7929	rq_set_donor(rq, t: idle);
7930	rcu_assign_pointer(rq->curr, idle);
7931	idle->on_rq = TASK_ON_RQ_QUEUED;
7932	idle->on_cpu = 1;
7933	raw_spin_rq_unlock(rq);
7934	raw_spin_unlock_irqrestore(&idle->pi_lock, flags);
7935
7936	/* Set the preempt count _outside_ the spinlocks! */
7937	init_idle_preempt_count(idle, cpu);
7938
7939	/*
7940	* The idle tasks have their own, simple scheduling class:
7941	*/
7942	idle->sched_class = &idle_sched_class;
7943	ftrace_graph_init_idle_task(t: idle, cpu);
7944	vtime_init_idle(tsk: idle, cpu);
7945	sprintf(buf: idle->comm, fmt: "%s/%d", INIT_TASK_COMM, cpu);
7946	}
7947
7948	int cpuset_cpumask_can_shrink(const struct cpumask *cur,
7949	const struct cpumask *trial)
7950	{
7951	int ret = 1;
7952
7953	if (cpumask_empty(srcp: cur))
7954	return ret;
7955
7956	ret = dl_cpuset_cpumask_can_shrink(cur, trial);
7957
7958	return ret;
7959	}
7960
7961	int task_can_attach(struct task_struct *p)
7962	{
7963	int ret = 0;
7964
7965	/*
7966	* Kthreads which disallow setaffinity shouldn't be moved
7967	* to a new cpuset; we don't want to change their CPU
7968	* affinity and isolating such threads by their set of
7969	* allowed nodes is unnecessary. Thus, cpusets are not
7970	* applicable for such threads. This prevents checking for
7971	* success of set_cpus_allowed_ptr() on all attached tasks
7972	* before cpus_mask may be changed.
7973	*/
7974	if (p->flags & PF_NO_SETAFFINITY)
7975	ret = -EINVAL;
7976
7977	return ret;
7978	}
7979
7980	bool sched_smp_initialized __read_mostly;
7981
7982	#ifdef CONFIG_NUMA_BALANCING
7983	/* Migrate current task p to target_cpu */
7984	int migrate_task_to(struct task_struct *p, int target_cpu)
7985	{
7986	struct migration_arg arg = { p, target_cpu };
7987	int curr_cpu = task_cpu(p);
7988
7989	if (curr_cpu == target_cpu)
7990	return 0;
7991
7992	if (!cpumask_test_cpu(cpu: target_cpu, cpumask: p->cpus_ptr))
7993	return -EINVAL;
7994
7995	/* TODO: This is not properly updating schedstats */
7996
7997	trace_sched_move_numa(tsk: p, src_cpu: curr_cpu, dst_cpu: target_cpu);
7998	return stop_one_cpu(cpu: curr_cpu, fn: migration_cpu_stop, arg: &arg);
7999	}
8000
8001	/*
8002	* Requeue a task on a given node and accurately track the number of NUMA
8003	* tasks on the runqueues
8004	*/
8005	void sched_setnuma(struct task_struct *p, int nid)
8006	{
8007	guard(task_rq_lock)(l: p);
8008	scoped_guard (sched_change, p, DEQUEUE_SAVE)
8009	p->numa_preferred_nid = nid;
8010	}
8011	#endif /* CONFIG_NUMA_BALANCING */
8012
8013	#ifdef CONFIG_HOTPLUG_CPU
8014	/*
8015	* Invoked on the outgoing CPU in context of the CPU hotplug thread
8016	* after ensuring that there are no user space tasks left on the CPU.
8017	*
8018	* If there is a lazy mm in use on the hotplug thread, drop it and
8019	* switch to init_mm.
8020	*
8021	* The reference count on init_mm is dropped in finish_cpu().
8022	*/
8023	static void sched_force_init_mm(void)
8024	{
8025	struct mm_struct *mm = current->active_mm;
8026
8027	if (mm != &init_mm) {
8028	mmgrab_lazy_tlb(mm: &init_mm);
8029	local_irq_disable();
8030	current->active_mm = &init_mm;
8031	switch_mm_irqs_off(prev: mm, next: &init_mm, current);
8032	local_irq_enable();
8033	finish_arch_post_lock_switch();
8034	mmdrop_lazy_tlb(mm);
8035	}
8036
8037	/* finish_cpu(), as ran on the BP, will clean up the active_mm state */
8038	}
8039
8040	static int __balance_push_cpu_stop(void *arg)
8041	{
8042	struct task_struct *p = arg;
8043	struct rq *rq = this_rq();
8044	struct rq_flags rf;
8045	int cpu;
8046
8047	scoped_guard (raw_spinlock_irq, &p->pi_lock) {
8048	cpu = select_fallback_rq(cpu: rq->cpu, p);
8049
8050	rq_lock(rq, rf: &rf);
8051	update_rq_clock(rq);
8052	if (task_rq(p) == rq && task_on_rq_queued(p))
8053	rq = __migrate_task(rq, rf: &rf, p, dest_cpu: cpu);
8054	rq_unlock(rq, rf: &rf);
8055	}
8056
8057	put_task_struct(t: p);
8058
8059	return 0;
8060	}
8061
8062	static DEFINE_PER_CPU(struct cpu_stop_work, push_work);
8063
8064	/*
8065	* Ensure we only run per-cpu kthreads once the CPU goes !active.
8066	*
8067	* This is enabled below SCHED_AP_ACTIVE; when !cpu_active(), but only
8068	* effective when the hotplug motion is down.
8069	*/
8070	static void balance_push(struct rq *rq)
8071	{
8072	struct task_struct *push_task = rq->curr;
8073
8074	lockdep_assert_rq_held(rq);
8075
8076	/*
8077	* Ensure the thing is persistent until balance_push_set(.on = false);
8078	*/
8079	rq->balance_callback = &balance_push_callback;
8080
8081	/*
8082	* Only active while going offline and when invoked on the outgoing
8083	* CPU.
8084	*/
8085	if (!cpu_dying(cpu: rq->cpu) || rq != this_rq())
8086	return;
8087
8088	/*
8089	* Both the cpu-hotplug and stop task are in this case and are
8090	* required to complete the hotplug process.
8091	*/
8092	if (kthread_is_per_cpu(k: push_task) ||
8093	is_migration_disabled(p: push_task)) {
8094
8095	/*
8096	* If this is the idle task on the outgoing CPU try to wake
8097	* up the hotplug control thread which might wait for the
8098	* last task to vanish. The rcuwait_active() check is
8099	* accurate here because the waiter is pinned on this CPU
8100	* and can't obviously be running in parallel.
8101	*
8102	* On RT kernels this also has to check whether there are
8103	* pinned and scheduled out tasks on the runqueue. They
8104	* need to leave the migrate disabled section first.
8105	*/
8106	if (!rq->nr_running && !rq_has_pinned_tasks(rq) &&
8107	rcuwait_active(w: &rq->hotplug_wait)) {
8108	raw_spin_rq_unlock(rq);
8109	rcuwait_wake_up(w: &rq->hotplug_wait);
8110	raw_spin_rq_lock(rq);
8111	}
8112	return;
8113	}
8114
8115	get_task_struct(t: push_task);
8116	/*
8117	* Temporarily drop rq->lock such that we can wake-up the stop task.
8118	* Both preemption and IRQs are still disabled.
8119	*/
8120	preempt_disable();
8121	raw_spin_rq_unlock(rq);
8122	stop_one_cpu_nowait(cpu: rq->cpu, fn: __balance_push_cpu_stop, arg: push_task,
8123	this_cpu_ptr(&push_work));
8124	preempt_enable();
8125	/*
8126	* At this point need_resched() is true and we'll take the loop in
8127	* schedule(). The next pick is obviously going to be the stop task
8128	* which kthread_is_per_cpu() and will push this task away.
8129	*/
8130	raw_spin_rq_lock(rq);
8131	}
8132
8133	static void balance_push_set(int cpu, bool on)
8134	{
8135	struct rq *rq = cpu_rq(cpu);
8136	struct rq_flags rf;
8137
8138	rq_lock_irqsave(rq, rf: &rf);
8139	if (on) {
8140	WARN_ON_ONCE(rq->balance_callback);
8141	rq->balance_callback = &balance_push_callback;
8142	} else if (rq->balance_callback == &balance_push_callback) {
8143	rq->balance_callback = NULL;
8144	}
8145	rq_unlock_irqrestore(rq, rf: &rf);
8146	}
8147
8148	/*
8149	* Invoked from a CPUs hotplug control thread after the CPU has been marked
8150	* inactive. All tasks which are not per CPU kernel threads are either
8151	* pushed off this CPU now via balance_push() or placed on a different CPU
8152	* during wakeup. Wait until the CPU is quiescent.
8153	*/
8154	static void balance_hotplug_wait(void)
8155	{
8156	struct rq *rq = this_rq();
8157
8158	rcuwait_wait_event(&rq->hotplug_wait,
8159	rq->nr_running == 1 && !rq_has_pinned_tasks(rq),
8160	TASK_UNINTERRUPTIBLE);
8161	}
8162
8163	#else /* !CONFIG_HOTPLUG_CPU: */
8164
8165	static inline void balance_push(struct rq *rq)
8166	{
8167	}
8168
8169	static inline void balance_push_set(int cpu, bool on)
8170	{
8171	}
8172
8173	static inline void balance_hotplug_wait(void)
8174	{
8175	}
8176
8177	#endif /* !CONFIG_HOTPLUG_CPU */
8178
8179	void set_rq_online(struct rq *rq)
8180	{
8181	if (!rq->online) {
8182	const struct sched_class *class;
8183
8184	cpumask_set_cpu(cpu: rq->cpu, dstp: rq->rd->online);
8185	rq->online = 1;
8186
8187	for_each_class(class) {
8188	if (class->rq_online)
8189	class->rq_online(rq);
8190	}
8191	}
8192	}
8193
8194	void set_rq_offline(struct rq *rq)
8195	{
8196	if (rq->online) {
8197	const struct sched_class *class;
8198
8199	update_rq_clock(rq);
8200	for_each_class(class) {
8201	if (class->rq_offline)
8202	class->rq_offline(rq);
8203	}
8204
8205	cpumask_clear_cpu(cpu: rq->cpu, dstp: rq->rd->online);
8206	rq->online = 0;
8207	}
8208	}
8209
8210	static inline void sched_set_rq_online(struct rq *rq, int cpu)
8211	{
8212	struct rq_flags rf;
8213
8214	rq_lock_irqsave(rq, rf: &rf);
8215	if (rq->rd) {
8216	BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
8217	set_rq_online(rq);
8218	}
8219	rq_unlock_irqrestore(rq, rf: &rf);
8220	}
8221
8222	static inline void sched_set_rq_offline(struct rq *rq, int cpu)
8223	{
8224	struct rq_flags rf;
8225
8226	rq_lock_irqsave(rq, rf: &rf);
8227	if (rq->rd) {
8228	BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
8229	set_rq_offline(rq);
8230	}
8231	rq_unlock_irqrestore(rq, rf: &rf);
8232	}
8233
8234	/*
8235	* used to mark begin/end of suspend/resume:
8236	*/
8237	static int num_cpus_frozen;
8238
8239	/*
8240	* Update cpusets according to cpu_active mask. If cpusets are
8241	* disabled, cpuset_update_active_cpus() becomes a simple wrapper
8242	* around partition_sched_domains().
8243	*
8244	* If we come here as part of a suspend/resume, don't touch cpusets because we
8245	* want to restore it back to its original state upon resume anyway.
8246	*/
8247	static void cpuset_cpu_active(void)
8248	{
8249	if (cpuhp_tasks_frozen) {
8250	/*
8251	* num_cpus_frozen tracks how many CPUs are involved in suspend
8252	* resume sequence. As long as this is not the last online
8253	* operation in the resume sequence, just build a single sched
8254	* domain, ignoring cpusets.
8255	*/
8256	cpuset_reset_sched_domains();
8257	if (--num_cpus_frozen)
8258	return;
8259	/*
8260	* This is the last CPU online operation. So fall through and
8261	* restore the original sched domains by considering the
8262	* cpuset configurations.
8263	*/
8264	cpuset_force_rebuild();
8265	}
8266	cpuset_update_active_cpus();
8267	}
8268
8269	static void cpuset_cpu_inactive(unsigned int cpu)
8270	{
8271	if (!cpuhp_tasks_frozen) {
8272	cpuset_update_active_cpus();
8273	} else {
8274	num_cpus_frozen++;
8275	cpuset_reset_sched_domains();
8276	}
8277	}
8278
8279	static inline void sched_smt_present_inc(int cpu)
8280	{
8281	#ifdef CONFIG_SCHED_SMT
8282	if (cpumask_weight(srcp: cpu_smt_mask(cpu)) == 2)
8283	static_branch_inc_cpuslocked(&sched_smt_present);
8284	#endif
8285	}
8286
8287	static inline void sched_smt_present_dec(int cpu)
8288	{
8289	#ifdef CONFIG_SCHED_SMT
8290	if (cpumask_weight(srcp: cpu_smt_mask(cpu)) == 2)
8291	static_branch_dec_cpuslocked(&sched_smt_present);
8292	#endif
8293	}
8294
8295	int sched_cpu_activate(unsigned int cpu)
8296	{
8297	struct rq *rq = cpu_rq(cpu);
8298
8299	/*
8300	* Clear the balance_push callback and prepare to schedule
8301	* regular tasks.
8302	*/
8303	balance_push_set(cpu, on: false);
8304
8305	/*
8306	* When going up, increment the number of cores with SMT present.
8307	*/
8308	sched_smt_present_inc(cpu);
8309	set_cpu_active(cpu, true);
8310
8311	if (sched_smp_initialized) {
8312	sched_update_numa(cpu, online: true);
8313	sched_domains_numa_masks_set(cpu);
8314	cpuset_cpu_active();
8315	}
8316
8317	scx_rq_activate(rq);
8318
8319	/*
8320	* Put the rq online, if not already. This happens:
8321	*
8322	* 1) In the early boot process, because we build the real domains
8323	* after all CPUs have been brought up.
8324	*
8325	* 2) At runtime, if cpuset_cpu_active() fails to rebuild the
8326	* domains.
8327	*/
8328	sched_set_rq_online(rq, cpu);
8329
8330	return 0;
8331	}
8332
8333	int sched_cpu_deactivate(unsigned int cpu)
8334	{
8335	struct rq *rq = cpu_rq(cpu);
8336	int ret;
8337
8338	ret = dl_bw_deactivate(cpu);
8339
8340	if (ret)
8341	return ret;
8342
8343	/*
8344	* Remove CPU from nohz.idle_cpus_mask to prevent participating in
8345	* load balancing when not active
8346	*/
8347	nohz_balance_exit_idle(rq);
8348
8349	set_cpu_active(cpu, false);
8350
8351	/*
8352	* From this point forward, this CPU will refuse to run any task that
8353	* is not: migrate_disable() or KTHREAD_IS_PER_CPU, and will actively
8354	* push those tasks away until this gets cleared, see
8355	* sched_cpu_dying().
8356	*/
8357	balance_push_set(cpu, on: true);
8358
8359	/*
8360	* We've cleared cpu_active_mask / set balance_push, wait for all
8361	* preempt-disabled and RCU users of this state to go away such that
8362	* all new such users will observe it.
8363	*
8364	* Specifically, we rely on ttwu to no longer target this CPU, see
8365	* ttwu_queue_cond() and is_cpu_allowed().
8366	*
8367	* Do sync before park smpboot threads to take care the RCU boost case.
8368	*/
8369	synchronize_rcu();
8370
8371	sched_set_rq_offline(rq, cpu);
8372
8373	scx_rq_deactivate(rq);
8374
8375	/*
8376	* When going down, decrement the number of cores with SMT present.
8377	*/
8378	sched_smt_present_dec(cpu);
8379
8380	#ifdef CONFIG_SCHED_SMT
8381	sched_core_cpu_deactivate(cpu);
8382	#endif
8383
8384	if (!sched_smp_initialized)
8385	return 0;
8386
8387	sched_update_numa(cpu, online: false);
8388	cpuset_cpu_inactive(cpu);
8389	sched_domains_numa_masks_clear(cpu);
8390	return 0;
8391	}
8392
8393	static void sched_rq_cpu_starting(unsigned int cpu)
8394	{
8395	struct rq *rq = cpu_rq(cpu);
8396
8397	rq->calc_load_update = calc_load_update;
8398	update_max_interval();
8399	}
8400
8401	int sched_cpu_starting(unsigned int cpu)
8402	{
8403	sched_core_cpu_starting(cpu);
8404	sched_rq_cpu_starting(cpu);
8405	sched_tick_start(cpu);
8406	return 0;
8407	}
8408
8409	#ifdef CONFIG_HOTPLUG_CPU
8410
8411	/*
8412	* Invoked immediately before the stopper thread is invoked to bring the
8413	* CPU down completely. At this point all per CPU kthreads except the
8414	* hotplug thread (current) and the stopper thread (inactive) have been
8415	* either parked or have been unbound from the outgoing CPU. Ensure that
8416	* any of those which might be on the way out are gone.
8417	*
8418	* If after this point a bound task is being woken on this CPU then the
8419	* responsible hotplug callback has failed to do it's job.
8420	* sched_cpu_dying() will catch it with the appropriate fireworks.
8421	*/
8422	int sched_cpu_wait_empty(unsigned int cpu)
8423	{
8424	balance_hotplug_wait();
8425	sched_force_init_mm();
8426	return 0;
8427	}
8428
8429	/*
8430	* Since this CPU is going 'away' for a while, fold any nr_active delta we
8431	* might have. Called from the CPU stopper task after ensuring that the
8432	* stopper is the last running task on the CPU, so nr_active count is
8433	* stable. We need to take the tear-down thread which is calling this into
8434	* account, so we hand in adjust = 1 to the load calculation.
8435	*
8436	* Also see the comment "Global load-average calculations".
8437	*/
8438	static void calc_load_migrate(struct rq *rq)
8439	{
8440	long delta = calc_load_fold_active(this_rq: rq, adjust: 1);
8441
8442	if (delta)
8443	atomic_long_add(i: delta, v: &calc_load_tasks);
8444	}
8445
8446	static void dump_rq_tasks(struct rq *rq, const char *loglvl)
8447	{
8448	struct task_struct *g, *p;
8449	int cpu = cpu_of(rq);
8450
8451	lockdep_assert_rq_held(rq);
8452
8453	printk("%sCPU%d enqueued tasks (%u total):\n", loglvl, cpu, rq->nr_running);
8454	for_each_process_thread(g, p) {
8455	if (task_cpu(p) != cpu)
8456	continue;
8457
8458	if (!task_on_rq_queued(p))
8459	continue;
8460
8461	printk("%s\tpid: %d, name: %s\n", loglvl, p->pid, p->comm);
8462	}
8463	}
8464
8465	int sched_cpu_dying(unsigned int cpu)
8466	{
8467	struct rq *rq = cpu_rq(cpu);
8468	struct rq_flags rf;
8469
8470	/* Handle pending wakeups and then migrate everything off */
8471	sched_tick_stop(cpu);
8472
8473	rq_lock_irqsave(rq, rf: &rf);
8474	update_rq_clock(rq);
8475	if (rq->nr_running != 1 || rq_has_pinned_tasks(rq)) {
8476	WARN(true, "Dying CPU not properly vacated!");
8477	dump_rq_tasks(rq, KERN_WARNING);
8478	}
8479	dl_server_stop(dl_se: &rq->fair_server);
8480	rq_unlock_irqrestore(rq, rf: &rf);
8481
8482	calc_load_migrate(rq);
8483	update_max_interval();
8484	hrtick_clear(rq);
8485	sched_core_cpu_dying(cpu);
8486	return 0;
8487	}
8488	#endif /* CONFIG_HOTPLUG_CPU */
8489
8490	void __init sched_init_smp(void)
8491	{
8492	sched_init_numa(NUMA_NO_NODE);
8493
8494	prandom_init_once(&sched_rnd_state);
8495
8496	/*
8497	* There's no userspace yet to cause hotplug operations; hence all the
8498	* CPU masks are stable and all blatant races in the below code cannot
8499	* happen.
8500	*/
8501	sched_domains_mutex_lock();
8502	sched_init_domains(cpu_active_mask);
8503	sched_domains_mutex_unlock();
8504
8505	/* Move init over to a non-isolated CPU */
8506	if (set_cpus_allowed_ptr(current, housekeeping_cpumask(type: HK_TYPE_DOMAIN)) < 0)
8507	BUG();
8508	current->flags &= ~PF_NO_SETAFFINITY;
8509	sched_init_granularity();
8510
8511	init_sched_rt_class();
8512	init_sched_dl_class();
8513
8514	sched_init_dl_servers();
8515
8516	sched_smp_initialized = true;
8517	}
8518
8519	static int __init migration_init(void)
8520	{
8521	sched_cpu_starting(smp_processor_id());
8522	return 0;
8523	}
8524	early_initcall(migration_init);
8525
8526	int in_sched_functions(unsigned long addr)
8527	{
8528	return in_lock_functions(addr) ||
8529	(addr >= (unsigned long)__sched_text_start
8530	&& addr < (unsigned long)__sched_text_end);
8531	}
8532
8533	#ifdef CONFIG_CGROUP_SCHED
8534	/*
8535	* Default task group.
8536	* Every task in system belongs to this group at bootup.
8537	*/
8538	struct task_group root_task_group;
8539	LIST_HEAD(task_groups);
8540
8541	/* Cacheline aligned slab cache for task_group */
8542	static struct kmem_cache *task_group_cache __ro_after_init;
8543	#endif
8544
8545	void __init sched_init(void)
8546	{
8547	unsigned long ptr = 0;
8548	int i;
8549
8550	/* Make sure the linker didn't screw up */
8551	BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class));
8552	BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class));
8553	BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class));
8554	BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class));
8555	#ifdef CONFIG_SCHED_CLASS_EXT
8556	BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class));
8557	BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class));
8558	#endif
8559
8560	wait_bit_init();
8561
8562	#ifdef CONFIG_FAIR_GROUP_SCHED
8563	ptr += 2 * nr_cpu_ids * sizeof(void **);
8564	#endif
8565	#ifdef CONFIG_RT_GROUP_SCHED
8566	ptr += 2 * nr_cpu_ids * sizeof(void **);
8567	#endif
8568	if (ptr) {
8569	ptr = (unsigned long)kzalloc(ptr, GFP_NOWAIT);
8570
8571	#ifdef CONFIG_FAIR_GROUP_SCHED
8572	root_task_group.se = (struct sched_entity **)ptr;
8573	ptr += nr_cpu_ids * sizeof(void **);
8574
8575	root_task_group.cfs_rq = (struct cfs_rq **)ptr;
8576	ptr += nr_cpu_ids * sizeof(void **);
8577
8578	root_task_group.shares = ROOT_TASK_GROUP_LOAD;
8579	init_cfs_bandwidth(cfs_b: &root_task_group.cfs_bandwidth, NULL);
8580	#endif /* CONFIG_FAIR_GROUP_SCHED */
8581	#ifdef CONFIG_EXT_GROUP_SCHED
8582	scx_tg_init(&root_task_group);
8583	#endif /* CONFIG_EXT_GROUP_SCHED */
8584	#ifdef CONFIG_RT_GROUP_SCHED
8585	root_task_group.rt_se = (struct sched_rt_entity **)ptr;
8586	ptr += nr_cpu_ids * sizeof(void **);
8587
8588	root_task_group.rt_rq = (struct rt_rq **)ptr;
8589	ptr += nr_cpu_ids * sizeof(void **);
8590
8591	#endif /* CONFIG_RT_GROUP_SCHED */
8592	}
8593
8594	init_defrootdomain();
8595
8596	#ifdef CONFIG_RT_GROUP_SCHED
8597	init_rt_bandwidth(rt_b: &root_task_group.rt_bandwidth,
8598	period: global_rt_period(), runtime: global_rt_runtime());
8599	#endif /* CONFIG_RT_GROUP_SCHED */
8600
8601	#ifdef CONFIG_CGROUP_SCHED
8602	task_group_cache = KMEM_CACHE(task_group, 0);
8603
8604	list_add(new: &root_task_group.list, head: &task_groups);
8605	INIT_LIST_HEAD(list: &root_task_group.children);
8606	INIT_LIST_HEAD(list: &root_task_group.siblings);
8607	autogroup_init(init_task: &init_task);
8608	#endif /* CONFIG_CGROUP_SCHED */
8609
8610	for_each_possible_cpu(i) {
8611	struct rq *rq;
8612
8613	rq = cpu_rq(i);
8614	raw_spin_lock_init(&rq->__lock);
8615	rq->nr_running = 0;
8616	rq->calc_load_active = 0;
8617	rq->calc_load_update = jiffies + LOAD_FREQ;
8618	init_cfs_rq(cfs_rq: &rq->cfs);
8619	init_rt_rq(rt_rq: &rq->rt);
8620	init_dl_rq(dl_rq: &rq->dl);
8621	#ifdef CONFIG_FAIR_GROUP_SCHED
8622	INIT_LIST_HEAD(list: &rq->leaf_cfs_rq_list);
8623	rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
8624	/*
8625	* How much CPU bandwidth does root_task_group get?
8626	*
8627	* In case of task-groups formed through the cgroup filesystem, it
8628	* gets 100% of the CPU resources in the system. This overall
8629	* system CPU resource is divided among the tasks of
8630	* root_task_group and its child task-groups in a fair manner,
8631	* based on each entity's (task or task-group's) weight
8632	* (se->load.weight).
8633	*
8634	* In other words, if root_task_group has 10 tasks of weight
8635	* 1024) and two child groups A0 and A1 (of weight 1024 each),
8636	* then A0's share of the CPU resource is:
8637	*
8638	* A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33%
8639	*
8640	* We achieve this by letting root_task_group's tasks sit
8641	* directly in rq->cfs (i.e root_task_group->se[] = NULL).
8642	*/
8643	init_tg_cfs_entry(tg: &root_task_group, cfs_rq: &rq->cfs, NULL, cpu: i, NULL);
8644	#endif /* CONFIG_FAIR_GROUP_SCHED */
8645
8646	#ifdef CONFIG_RT_GROUP_SCHED
8647	/*
8648	* This is required for init cpu because rt.c:__enable_runtime()
8649	* starts working after scheduler_running, which is not the case
8650	* yet.
8651	*/
8652	rq->rt.rt_runtime = global_rt_runtime();
8653	init_tg_rt_entry(tg: &root_task_group, rt_rq: &rq->rt, NULL, cpu: i, NULL);
8654	#endif
8655	rq->sd = NULL;
8656	rq->rd = NULL;
8657	rq->cpu_capacity = SCHED_CAPACITY_SCALE;
8658	rq->balance_callback = &balance_push_callback;
8659	rq->active_balance = 0;
8660	rq->next_balance = jiffies;
8661	rq->push_cpu = 0;
8662	rq->cpu = i;
8663	rq->online = 0;
8664	rq->idle_stamp = 0;
8665	rq->avg_idle = 2*sysctl_sched_migration_cost;
8666	rq->max_idle_balance_cost = sysctl_sched_migration_cost;
8667
8668	INIT_LIST_HEAD(list: &rq->cfs_tasks);
8669
8670	rq_attach_root(rq, rd: &def_root_domain);
8671	#ifdef CONFIG_NO_HZ_COMMON
8672	rq->last_blocked_load_update_tick = jiffies;
8673	atomic_set(v: &rq->nohz_flags, i: 0);
8674
8675	INIT_CSD(&rq->nohz_csd, nohz_csd_func, rq);
8676	#endif
8677	#ifdef CONFIG_HOTPLUG_CPU
8678	rcuwait_init(w: &rq->hotplug_wait);
8679	#endif
8680	hrtick_rq_init(rq);
8681	atomic_set(v: &rq->nr_iowait, i: 0);
8682	fair_server_init(rq);
8683
8684	#ifdef CONFIG_SCHED_CORE
8685	rq->core = rq;
8686	rq->core_pick = NULL;
8687	rq->core_dl_server = NULL;
8688	rq->core_enabled = 0;
8689	rq->core_tree = RB_ROOT;
8690	rq->core_forceidle_count = 0;
8691	rq->core_forceidle_occupation = 0;
8692	rq->core_forceidle_start = 0;
8693
8694	rq->core_cookie = 0UL;
8695	#endif
8696	zalloc_cpumask_var_node(mask: &rq->scratch_mask, GFP_KERNEL, cpu_to_node(cpu: i));
8697	}
8698
8699	set_load_weight(p: &init_task, update_load: false);
8700	init_task.se.slice = sysctl_sched_base_slice,
8701
8702	/*
8703	* The boot idle thread does lazy MMU switching as well:
8704	*/
8705	mmgrab_lazy_tlb(mm: &init_mm);
8706	enter_lazy_tlb(mm: &init_mm, current);
8707
8708	/*
8709	* The idle task doesn't need the kthread struct to function, but it
8710	* is dressed up as a per-CPU kthread and thus needs to play the part
8711	* if we want to avoid special-casing it in code that deals with per-CPU
8712	* kthreads.
8713	*/
8714	WARN_ON(!set_kthread_struct(current));
8715
8716	/*
8717	* Make us the idle thread. Technically, schedule() should not be
8718	* called from this thread, however somewhere below it might be,
8719	* but because we are the idle thread, we just pick up running again
8720	* when this runqueue becomes "idle".
8721	*/
8722	__sched_fork(clone_flags: 0, current);
8723	init_idle(current, smp_processor_id());
8724
8725	calc_load_update = jiffies + LOAD_FREQ;
8726
8727	idle_thread_set_boot_cpu();
8728
8729	balance_push_set(smp_processor_id(), on: false);
8730	init_sched_fair_class();
8731	init_sched_ext_class();
8732
8733	psi_init();
8734
8735	init_uclamp();
8736
8737	preempt_dynamic_init();
8738
8739	scheduler_running = 1;
8740	}
8741
8742	#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
8743
8744	void __might_sleep(const char *file, int line)
8745	{
8746	unsigned int state = get_current_state();
8747	/*
8748	* Blocking primitives will set (and therefore destroy) current->state,
8749	* since we will exit with TASK_RUNNING make sure we enter with it,
8750	* otherwise we will destroy state.
8751	*/
8752	WARN_ONCE(state != TASK_RUNNING && current->task_state_change,
8753	"do not call blocking ops when !TASK_RUNNING; "
8754	"state=%x set at [<%p>] %pS\n", state,
8755	(void *)current->task_state_change,
8756	(void *)current->task_state_change);
8757
8758	__might_resched(file, line, offsets: 0);
8759	}
8760	EXPORT_SYMBOL(__might_sleep);
8761
8762	static void print_preempt_disable_ip(int preempt_offset, unsigned long ip)
8763	{
8764	if (!IS_ENABLED(CONFIG_DEBUG_PREEMPT))
8765	return;
8766
8767	if (preempt_count() == preempt_offset)
8768	return;
8769
8770	pr_err("Preemption disabled at:");
8771	print_ip_sym(KERN_ERR, ip);
8772	}
8773
8774	static inline bool resched_offsets_ok(unsigned int offsets)
8775	{
8776	unsigned int nested = preempt_count();
8777
8778	nested += rcu_preempt_depth() << MIGHT_RESCHED_RCU_SHIFT;
8779
8780	return nested == offsets;
8781	}
8782
8783	void __might_resched(const char *file, int line, unsigned int offsets)
8784	{
8785	/* Ratelimiting timestamp: */
8786	static unsigned long prev_jiffy;
8787
8788	unsigned long preempt_disable_ip;
8789
8790	/* WARN_ON_ONCE() by default, no rate limit required: */
8791	rcu_sleep_check();
8792
8793	if ((resched_offsets_ok(offsets) && !irqs_disabled() &&
8794	!is_idle_task(current) && !current->non_block_count) ||
8795	system_state == SYSTEM_BOOTING || system_state > SYSTEM_RUNNING ||
8796	oops_in_progress)
8797	return;
8798
8799	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
8800	return;
8801	prev_jiffy = jiffies;
8802
8803	/* Save this before calling printk(), since that will clobber it: */
8804	preempt_disable_ip = get_preempt_disable_ip(current);
8805
8806	pr_err("BUG: sleeping function called from invalid context at %s:%d\n",
8807	file, line);
8808	pr_err("in_atomic(): %d, irqs_disabled(): %d, non_block: %d, pid: %d, name: %s\n",
8809	in_atomic(), irqs_disabled(), current->non_block_count,
8810	current->pid, current->comm);
8811	pr_err("preempt_count: %x, expected: %x\n", preempt_count(),
8812	offsets & MIGHT_RESCHED_PREEMPT_MASK);
8813
8814	if (IS_ENABLED(CONFIG_PREEMPT_RCU)) {
8815	pr_err("RCU nest depth: %d, expected: %u\n",
8816	rcu_preempt_depth(), offsets >> MIGHT_RESCHED_RCU_SHIFT);
8817	}
8818
8819	if (task_stack_end_corrupted(current))
8820	pr_emerg("Thread overran stack, or stack corrupted\n");
8821
8822	debug_show_held_locks(current);
8823	if (irqs_disabled())
8824	print_irqtrace_events(current);
8825
8826	print_preempt_disable_ip(preempt_offset: offsets & MIGHT_RESCHED_PREEMPT_MASK,
8827	ip: preempt_disable_ip);
8828
8829	dump_stack();
8830	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
8831	}
8832	EXPORT_SYMBOL(__might_resched);
8833
8834	void __cant_sleep(const char *file, int line, int preempt_offset)
8835	{
8836	static unsigned long prev_jiffy;
8837
8838	if (irqs_disabled())
8839	return;
8840
8841	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
8842	return;
8843
8844	if (preempt_count() > preempt_offset)
8845	return;
8846
8847	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
8848	return;
8849	prev_jiffy = jiffies;
8850
8851	printk(KERN_ERR "BUG: assuming atomic context at %s:%d\n", file, line);
8852	printk(KERN_ERR "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n",
8853	in_atomic(), irqs_disabled(),
8854	current->pid, current->comm);
8855
8856	debug_show_held_locks(current);
8857	dump_stack();
8858	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
8859	}
8860	EXPORT_SYMBOL_GPL(__cant_sleep);
8861
8862	# ifdef CONFIG_SMP
8863	void __cant_migrate(const char *file, int line)
8864	{
8865	static unsigned long prev_jiffy;
8866
8867	if (irqs_disabled())
8868	return;
8869
8870	if (is_migration_disabled(current))
8871	return;
8872
8873	if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
8874	return;
8875
8876	if (preempt_count() > 0)
8877	return;
8878
8879	if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
8880	return;
8881	prev_jiffy = jiffies;
8882
8883	pr_err("BUG: assuming non migratable context at %s:%d\n", file, line);
8884	pr_err("in_atomic(): %d, irqs_disabled(): %d, migration_disabled() %u pid: %d, name: %s\n",
8885	in_atomic(), irqs_disabled(), is_migration_disabled(current),
8886	current->pid, current->comm);
8887
8888	debug_show_held_locks(current);
8889	dump_stack();
8890	add_taint(TAINT_WARN, LOCKDEP_STILL_OK);
8891	}
8892	EXPORT_SYMBOL_GPL(__cant_migrate);
8893	# endif /* CONFIG_SMP */
8894	#endif /* CONFIG_DEBUG_ATOMIC_SLEEP */
8895
8896	#ifdef CONFIG_MAGIC_SYSRQ
8897	void normalize_rt_tasks(void)
8898	{
8899	struct task_struct *g, *p;
8900	struct sched_attr attr = {
8901	.sched_policy = SCHED_NORMAL,
8902	};
8903
8904	read_lock(&tasklist_lock);
8905	for_each_process_thread(g, p) {
8906	/*
8907	* Only normalize user tasks:
8908	*/
8909	if (p->flags & PF_KTHREAD)
8910	continue;
8911
8912	p->se.exec_start = 0;
8913	schedstat_set(p->stats.wait_start, 0);
8914	schedstat_set(p->stats.sleep_start, 0);
8915	schedstat_set(p->stats.block_start, 0);
8916
8917	if (!rt_or_dl_task(p)) {
8918	/*
8919	* Renice negative nice level userspace
8920	* tasks back to 0:
8921	*/
8922	if (task_nice(p) < 0)
8923	set_user_nice(p, nice: 0);
8924	continue;
8925	}
8926
8927	__sched_setscheduler(p, attr: &attr, user: false, pi: false);
8928	}
8929	read_unlock(&tasklist_lock);
8930	}
8931
8932	#endif /* CONFIG_MAGIC_SYSRQ */
8933
8934	#ifdef CONFIG_KGDB_KDB
8935	/*
8936	* These functions are only useful for KDB.
8937	*
8938	* They can only be called when the whole system has been
8939	* stopped - every CPU needs to be quiescent, and no scheduling
8940	* activity can take place. Using them for anything else would
8941	* be a serious bug, and as a result, they aren't even visible
8942	* under any other configuration.
8943	*/
8944
8945	/**
8946	* curr_task - return the current task for a given CPU.
8947	* @cpu: the processor in question.
8948	*
8949	* ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED!
8950	*
8951	* Return: The current task for @cpu.
8952	*/
8953	struct task_struct *curr_task(int cpu)
8954	{
8955	return cpu_curr(cpu);
8956	}
8957
8958	#endif /* CONFIG_KGDB_KDB */
8959
8960	#ifdef CONFIG_CGROUP_SCHED
8961	/* task_group_lock serializes the addition/removal of task groups */
8962	static DEFINE_SPINLOCK(task_group_lock);
8963
8964	static inline void alloc_uclamp_sched_group(struct task_group *tg,
8965	struct task_group *parent)
8966	{
8967	#ifdef CONFIG_UCLAMP_TASK_GROUP
8968	enum uclamp_id clamp_id;
8969
8970	for_each_clamp_id(clamp_id) {
8971	uclamp_se_set(uc_se: &tg->uclamp_req[clamp_id],
8972	value: uclamp_none(clamp_id), user_defined: false);
8973	tg->uclamp[clamp_id] = parent->uclamp[clamp_id];
8974	}
8975	#endif
8976	}
8977
8978	static void sched_free_group(struct task_group *tg)
8979	{
8980	free_fair_sched_group(tg);
8981	free_rt_sched_group(tg);
8982	autogroup_free(tg);
8983	kmem_cache_free(s: task_group_cache, objp: tg);
8984	}
8985
8986	static void sched_free_group_rcu(struct rcu_head *rcu)
8987	{
8988	sched_free_group(container_of(rcu, struct task_group, rcu));
8989	}
8990
8991	static void sched_unregister_group(struct task_group *tg)
8992	{
8993	unregister_fair_sched_group(tg);
8994	unregister_rt_sched_group(tg);
8995	/*
8996	* We have to wait for yet another RCU grace period to expire, as
8997	* print_cfs_stats() might run concurrently.
8998	*/
8999	call_rcu(head: &tg->rcu, func: sched_free_group_rcu);
9000	}
9001
9002	/* allocate runqueue etc for a new task group */
9003	struct task_group *sched_create_group(struct task_group *parent)
9004	{
9005	struct task_group *tg;
9006
9007	tg = kmem_cache_alloc(task_group_cache, GFP_KERNEL | __GFP_ZERO);
9008	if (!tg)
9009	return ERR_PTR(error: -ENOMEM);
9010
9011	if (!alloc_fair_sched_group(tg, parent))
9012	goto err;
9013
9014	if (!alloc_rt_sched_group(tg, parent))
9015	goto err;
9016
9017	scx_tg_init(tg);
9018	alloc_uclamp_sched_group(tg, parent);
9019
9020	return tg;
9021
9022	err:
9023	sched_free_group(tg);
9024	return ERR_PTR(error: -ENOMEM);
9025	}
9026
9027	void sched_online_group(struct task_group *tg, struct task_group *parent)
9028	{
9029	unsigned long flags;
9030
9031	spin_lock_irqsave(&task_group_lock, flags);
9032	list_add_tail_rcu(new: &tg->list, head: &task_groups);
9033
9034	/* Root should already exist: */
9035	WARN_ON(!parent);
9036
9037	tg->parent = parent;
9038	INIT_LIST_HEAD(list: &tg->children);
9039	list_add_rcu(new: &tg->siblings, head: &parent->children);
9040	spin_unlock_irqrestore(lock: &task_group_lock, flags);
9041
9042	online_fair_sched_group(tg);
9043	}
9044
9045	/* RCU callback to free various structures associated with a task group */
9046	static void sched_unregister_group_rcu(struct rcu_head *rhp)
9047	{
9048	/* Now it should be safe to free those cfs_rqs: */
9049	sched_unregister_group(container_of(rhp, struct task_group, rcu));
9050	}
9051
9052	void sched_destroy_group(struct task_group *tg)
9053	{
9054	/* Wait for possible concurrent references to cfs_rqs complete: */
9055	call_rcu(head: &tg->rcu, func: sched_unregister_group_rcu);
9056	}
9057
9058	void sched_release_group(struct task_group *tg)
9059	{
9060	unsigned long flags;
9061
9062	/*
9063	* Unlink first, to avoid walk_tg_tree_from() from finding us (via
9064	* sched_cfs_period_timer()).
9065	*
9066	* For this to be effective, we have to wait for all pending users of
9067	* this task group to leave their RCU critical section to ensure no new
9068	* user will see our dying task group any more. Specifically ensure
9069	* that tg_unthrottle_up() won't add decayed cfs_rq's to it.
9070	*
9071	* We therefore defer calling unregister_fair_sched_group() to
9072	* sched_unregister_group() which is guarantied to get called only after the
9073	* current RCU grace period has expired.
9074	*/
9075	spin_lock_irqsave(&task_group_lock, flags);
9076	list_del_rcu(entry: &tg->list);
9077	list_del_rcu(entry: &tg->siblings);
9078	spin_unlock_irqrestore(lock: &task_group_lock, flags);
9079	}
9080
9081	static void sched_change_group(struct task_struct *tsk)
9082	{
9083	struct task_group *tg;
9084
9085	/*
9086	* All callers are synchronized by task_rq_lock(); we do not use RCU
9087	* which is pointless here. Thus, we pass "true" to task_css_check()
9088	* to prevent lockdep warnings.
9089	*/
9090	tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
9091	struct task_group, css);
9092	tg = autogroup_task_group(p: tsk, tg);
9093	tsk->sched_task_group = tg;
9094
9095	#ifdef CONFIG_FAIR_GROUP_SCHED
9096	if (tsk->sched_class->task_change_group)
9097	tsk->sched_class->task_change_group(tsk);
9098	else
9099	#endif
9100	set_task_rq(p: tsk, cpu: task_cpu(p: tsk));
9101	}
9102
9103	/*
9104	* Change task's runqueue when it moves between groups.
9105	*
9106	* The caller of this function should have put the task in its new group by
9107	* now. This function just updates tsk->se.cfs_rq and tsk->se.parent to reflect
9108	* its new group.
9109	*/
9110	void sched_move_task(struct task_struct *tsk, bool for_autogroup)
9111	{
9112	unsigned int queue_flags = DEQUEUE_SAVE | DEQUEUE_MOVE;
9113	bool resched = false;
9114	struct rq *rq;
9115
9116	CLASS(task_rq_lock, rq_guard)(l: tsk);
9117	rq = rq_guard.rq;
9118
9119	scoped_guard (sched_change, tsk, queue_flags) {
9120	sched_change_group(tsk);
9121	if (!for_autogroup)
9122	scx_cgroup_move_task(p: tsk);
9123	if (scope->running)
9124	resched = true;
9125	}
9126
9127	if (resched)
9128	resched_curr(rq);
9129
9130	__balance_callbacks(rq, rf: &rq_guard.rf);
9131	}
9132
9133	static struct cgroup_subsys_state *
9134	cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
9135	{
9136	struct task_group *parent = css_tg(css: parent_css);
9137	struct task_group *tg;
9138
9139	if (!parent) {
9140	/* This is early initialization for the top cgroup */
9141	return &root_task_group.css;
9142	}
9143
9144	tg = sched_create_group(parent);
9145	if (IS_ERR(ptr: tg))
9146	return ERR_PTR(error: -ENOMEM);
9147
9148	return &tg->css;
9149	}
9150
9151	/* Expose task group only after completing cgroup initialization */
9152	static int cpu_cgroup_css_online(struct cgroup_subsys_state *css)
9153	{
9154	struct task_group *tg = css_tg(css);
9155	struct task_group *parent = css_tg(css: css->parent);
9156	int ret;
9157
9158	ret = scx_tg_online(tg);
9159	if (ret)
9160	return ret;
9161
9162	if (parent)
9163	sched_online_group(tg, parent);
9164
9165	#ifdef CONFIG_UCLAMP_TASK_GROUP
9166	/* Propagate the effective uclamp value for the new group */
9167	guard(mutex)(T: &uclamp_mutex);
9168	guard(rcu)();
9169	cpu_util_update_eff(css);
9170	#endif
9171
9172	return 0;
9173	}
9174
9175	static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
9176	{
9177	struct task_group *tg = css_tg(css);
9178
9179	scx_tg_offline(tg);
9180	}
9181
9182	static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
9183	{
9184	struct task_group *tg = css_tg(css);
9185
9186	sched_release_group(tg);
9187	}
9188
9189	static void cpu_cgroup_css_free(struct cgroup_subsys_state *css)
9190	{
9191	struct task_group *tg = css_tg(css);
9192
9193	/*
9194	* Relies on the RCU grace period between css_released() and this.
9195	*/
9196	sched_unregister_group(tg);
9197	}
9198
9199	static int cpu_cgroup_can_attach(struct cgroup_taskset *tset)
9200	{
9201	#ifdef CONFIG_RT_GROUP_SCHED
9202	struct task_struct *task;
9203	struct cgroup_subsys_state *css;
9204
9205	if (!rt_group_sched_enabled())
9206	goto scx_check;
9207
9208	cgroup_taskset_for_each(task, css, tset) {
9209	if (!sched_rt_can_attach(tg: css_tg(css), tsk: task))
9210	return -EINVAL;
9211	}
9212	scx_check:
9213	#endif /* CONFIG_RT_GROUP_SCHED */
9214	return scx_cgroup_can_attach(tset);
9215	}
9216
9217	static void cpu_cgroup_attach(struct cgroup_taskset *tset)
9218	{
9219	struct task_struct *task;
9220	struct cgroup_subsys_state *css;
9221
9222	cgroup_taskset_for_each(task, css, tset)
9223	sched_move_task(tsk: task, for_autogroup: false);
9224	}
9225
9226	static void cpu_cgroup_cancel_attach(struct cgroup_taskset *tset)
9227	{
9228	scx_cgroup_cancel_attach(tset);
9229	}
9230
9231	#ifdef CONFIG_UCLAMP_TASK_GROUP
9232	static void cpu_util_update_eff(struct cgroup_subsys_state *css)
9233	{
9234	struct cgroup_subsys_state *top_css = css;
9235	struct uclamp_se *uc_parent = NULL;
9236	struct uclamp_se *uc_se = NULL;
9237	unsigned int eff[UCLAMP_CNT];
9238	enum uclamp_id clamp_id;
9239	unsigned int clamps;
9240
9241	lockdep_assert_held(&uclamp_mutex);
9242	WARN_ON_ONCE(!rcu_read_lock_held());
9243
9244	css_for_each_descendant_pre(css, top_css) {
9245	uc_parent = css_tg(css)->parent
9246	? css_tg(css)->parent->uclamp : NULL;
9247
9248	for_each_clamp_id(clamp_id) {
9249	/* Assume effective clamps matches requested clamps */
9250	eff[clamp_id] = css_tg(css)->uclamp_req[clamp_id].value;
9251	/* Cap effective clamps with parent's effective clamps */
9252	if (uc_parent &&
9253	eff[clamp_id] > uc_parent[clamp_id].value) {
9254	eff[clamp_id] = uc_parent[clamp_id].value;
9255	}
9256	}
9257	/* Ensure protection is always capped by limit */
9258	eff[UCLAMP_MIN] = min(eff[UCLAMP_MIN], eff[UCLAMP_MAX]);
9259
9260	/* Propagate most restrictive effective clamps */
9261	clamps = 0x0;
9262	uc_se = css_tg(css)->uclamp;
9263	for_each_clamp_id(clamp_id) {
9264	if (eff[clamp_id] == uc_se[clamp_id].value)
9265	continue;
9266	uc_se[clamp_id].value = eff[clamp_id];
9267	uc_se[clamp_id].bucket_id = uclamp_bucket_id(clamp_value: eff[clamp_id]);
9268	clamps |= (0x1 << clamp_id);
9269	}
9270	if (!clamps) {
9271	css = css_rightmost_descendant(pos: css);
9272	continue;
9273	}
9274
9275	/* Immediately update descendants RUNNABLE tasks */
9276	uclamp_update_active_tasks(css);
9277	}
9278	}
9279
9280	/*
9281	* Integer 10^N with a given N exponent by casting to integer the literal "1eN"
9282	* C expression. Since there is no way to convert a macro argument (N) into a
9283	* character constant, use two levels of macros.
9284	*/
9285	#define _POW10(exp) ((unsigned int)1e##exp)
9286	#define POW10(exp) _POW10(exp)
9287
9288	struct uclamp_request {
9289	#define UCLAMP_PERCENT_SHIFT 2
9290	#define UCLAMP_PERCENT_SCALE (100 * POW10(UCLAMP_PERCENT_SHIFT))
9291	s64 percent;
9292	u64 util;
9293	int ret;
9294	};
9295
9296	static inline struct uclamp_request
9297	capacity_from_percent(char *buf)
9298	{
9299	struct uclamp_request req = {
9300	.percent = UCLAMP_PERCENT_SCALE,
9301	.util = SCHED_CAPACITY_SCALE,
9302	.ret = 0,
9303	};
9304
9305	buf = strim(buf);
9306	if (strcmp(buf, "max")) {
9307	req.ret = cgroup_parse_float(input: buf, UCLAMP_PERCENT_SHIFT,
9308	v: &req.percent);
9309	if (req.ret)
9310	return req;
9311	if ((u64)req.percent > UCLAMP_PERCENT_SCALE) {
9312	req.ret = -ERANGE;
9313	return req;
9314	}
9315
9316	req.util = req.percent << SCHED_CAPACITY_SHIFT;
9317	req.util = DIV_ROUND_CLOSEST_ULL(req.util, UCLAMP_PERCENT_SCALE);
9318	}
9319
9320	return req;
9321	}
9322
9323	static ssize_t cpu_uclamp_write(struct kernfs_open_file *of, char *buf,
9324	size_t nbytes, loff_t off,
9325	enum uclamp_id clamp_id)
9326	{
9327	struct uclamp_request req;
9328	struct task_group *tg;
9329
9330	req = capacity_from_percent(buf);
9331	if (req.ret)
9332	return req.ret;
9333
9334	sched_uclamp_enable();
9335
9336	guard(mutex)(T: &uclamp_mutex);
9337	guard(rcu)();
9338
9339	tg = css_tg(css: of_css(of));
9340	if (tg->uclamp_req[clamp_id].value != req.util)
9341	uclamp_se_set(uc_se: &tg->uclamp_req[clamp_id], value: req.util, user_defined: false);
9342
9343	/*
9344	* Because of not recoverable conversion rounding we keep track of the
9345	* exact requested value
9346	*/
9347	tg->uclamp_pct[clamp_id] = req.percent;
9348
9349	/* Update effective clamps to track the most restrictive value */
9350	cpu_util_update_eff(css: of_css(of));
9351
9352	return nbytes;
9353	}
9354
9355	static ssize_t cpu_uclamp_min_write(struct kernfs_open_file *of,
9356	char *buf, size_t nbytes,
9357	loff_t off)
9358	{
9359	return cpu_uclamp_write(of, buf, nbytes, off, clamp_id: UCLAMP_MIN);
9360	}
9361
9362	static ssize_t cpu_uclamp_max_write(struct kernfs_open_file *of,
9363	char *buf, size_t nbytes,
9364	loff_t off)
9365	{
9366	return cpu_uclamp_write(of, buf, nbytes, off, clamp_id: UCLAMP_MAX);
9367	}
9368
9369	static inline void cpu_uclamp_print(struct seq_file *sf,
9370	enum uclamp_id clamp_id)
9371	{
9372	struct task_group *tg;
9373	u64 util_clamp;
9374	u64 percent;
9375	u32 rem;
9376
9377	scoped_guard (rcu) {
9378	tg = css_tg(css: seq_css(seq: sf));
9379	util_clamp = tg->uclamp_req[clamp_id].value;
9380	}
9381
9382	if (util_clamp == SCHED_CAPACITY_SCALE) {
9383	seq_puts(m: sf, s: "max\n");
9384	return;
9385	}
9386
9387	percent = tg->uclamp_pct[clamp_id];
9388	percent = div_u64_rem(dividend: percent, POW10(UCLAMP_PERCENT_SHIFT), remainder: &rem);
9389	seq_printf(m: sf, fmt: "%llu.%0*u\n", percent, UCLAMP_PERCENT_SHIFT, rem);
9390	}
9391
9392	static int cpu_uclamp_min_show(struct seq_file *sf, void *v)
9393	{
9394	cpu_uclamp_print(sf, clamp_id: UCLAMP_MIN);
9395	return 0;
9396	}
9397
9398	static int cpu_uclamp_max_show(struct seq_file *sf, void *v)
9399	{
9400	cpu_uclamp_print(sf, clamp_id: UCLAMP_MAX);
9401	return 0;
9402	}
9403	#endif /* CONFIG_UCLAMP_TASK_GROUP */
9404
9405	#ifdef CONFIG_GROUP_SCHED_WEIGHT
9406	static unsigned long tg_weight(struct task_group *tg)
9407	{
9408	#ifdef CONFIG_FAIR_GROUP_SCHED
9409	return scale_load_down(tg->shares);
9410	#else
9411	return sched_weight_from_cgroup(tg->scx.weight);
9412	#endif
9413	}
9414
9415	static int cpu_shares_write_u64(struct cgroup_subsys_state *css,
9416	struct cftype *cftype, u64 shareval)
9417	{
9418	int ret;
9419
9420	if (shareval > scale_load_down(ULONG_MAX))
9421	shareval = MAX_SHARES;
9422	ret = sched_group_set_shares(tg: css_tg(css), scale_load(shareval));
9423	if (!ret)
9424	scx_group_set_weight(tg: css_tg(css),
9425	cgrp_weight: sched_weight_to_cgroup(weight: shareval));
9426	return ret;
9427	}
9428
9429	static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css,
9430	struct cftype *cft)
9431	{
9432	return tg_weight(tg: css_tg(css));
9433	}
9434	#endif /* CONFIG_GROUP_SCHED_WEIGHT */
9435
9436	#ifdef CONFIG_CFS_BANDWIDTH
9437	static DEFINE_MUTEX(cfs_constraints_mutex);
9438
9439	static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime);
9440
9441	static int tg_set_cfs_bandwidth(struct task_group *tg,
9442	u64 period_us, u64 quota_us, u64 burst_us)
9443	{
9444	int i, ret = 0, runtime_enabled, runtime_was_enabled;
9445	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
9446	u64 period, quota, burst;
9447
9448	period = (u64)period_us * NSEC_PER_USEC;
9449
9450	if (quota_us == RUNTIME_INF)
9451	quota = RUNTIME_INF;
9452	else
9453	quota = (u64)quota_us * NSEC_PER_USEC;
9454
9455	burst = (u64)burst_us * NSEC_PER_USEC;
9456
9457	/*
9458	* Prevent race between setting of cfs_rq->runtime_enabled and
9459	* unthrottle_offline_cfs_rqs().
9460	*/
9461	guard(cpus_read_lock)();
9462	guard(mutex)(T: &cfs_constraints_mutex);
9463
9464	ret = __cfs_schedulable(tg, period, runtime: quota);
9465	if (ret)
9466	return ret;
9467
9468	runtime_enabled = quota != RUNTIME_INF;
9469	runtime_was_enabled = cfs_b->quota != RUNTIME_INF;
9470	/*
9471	* If we need to toggle cfs_bandwidth_used, off->on must occur
9472	* before making related changes, and on->off must occur afterwards
9473	*/
9474	if (runtime_enabled && !runtime_was_enabled)
9475	cfs_bandwidth_usage_inc();
9476
9477	scoped_guard (raw_spinlock_irq, &cfs_b->lock) {
9478	cfs_b->period = ns_to_ktime(ns: period);
9479	cfs_b->quota = quota;
9480	cfs_b->burst = burst;
9481
9482	__refill_cfs_bandwidth_runtime(cfs_b);
9483
9484	/*
9485	* Restart the period timer (if active) to handle new
9486	* period expiry:
9487	*/
9488	if (runtime_enabled)
9489	start_cfs_bandwidth(cfs_b);
9490	}
9491
9492	for_each_online_cpu(i) {
9493	struct cfs_rq *cfs_rq = tg->cfs_rq[i];
9494	struct rq *rq = cfs_rq->rq;
9495
9496	guard(rq_lock_irq)(l: rq);
9497	cfs_rq->runtime_enabled = runtime_enabled;
9498	cfs_rq->runtime_remaining = 1;
9499
9500	if (cfs_rq->throttled)
9501	unthrottle_cfs_rq(cfs_rq);
9502	}
9503
9504	if (runtime_was_enabled && !runtime_enabled)
9505	cfs_bandwidth_usage_dec();
9506
9507	return 0;
9508	}
9509
9510	static u64 tg_get_cfs_period(struct task_group *tg)
9511	{
9512	u64 cfs_period_us;
9513
9514	cfs_period_us = ktime_to_ns(kt: tg->cfs_bandwidth.period);
9515	do_div(cfs_period_us, NSEC_PER_USEC);
9516
9517	return cfs_period_us;
9518	}
9519
9520	static u64 tg_get_cfs_quota(struct task_group *tg)
9521	{
9522	u64 quota_us;
9523
9524	if (tg->cfs_bandwidth.quota == RUNTIME_INF)
9525	return RUNTIME_INF;
9526
9527	quota_us = tg->cfs_bandwidth.quota;
9528	do_div(quota_us, NSEC_PER_USEC);
9529
9530	return quota_us;
9531	}
9532
9533	static u64 tg_get_cfs_burst(struct task_group *tg)
9534	{
9535	u64 burst_us;
9536
9537	burst_us = tg->cfs_bandwidth.burst;
9538	do_div(burst_us, NSEC_PER_USEC);
9539
9540	return burst_us;
9541	}
9542
9543	struct cfs_schedulable_data {
9544	struct task_group *tg;
9545	u64 period, quota;
9546	};
9547
9548	/*
9549	* normalize group quota/period to be quota/max_period
9550	* note: units are usecs
9551	*/
9552	static u64 normalize_cfs_quota(struct task_group *tg,
9553	struct cfs_schedulable_data *d)
9554	{
9555	u64 quota, period;
9556
9557	if (tg == d->tg) {
9558	period = d->period;
9559	quota = d->quota;
9560	} else {
9561	period = tg_get_cfs_period(tg);
9562	quota = tg_get_cfs_quota(tg);
9563	}
9564
9565	/* note: these should typically be equivalent */
9566	if (quota == RUNTIME_INF || quota == -1)
9567	return RUNTIME_INF;
9568
9569	return to_ratio(period, runtime: quota);
9570	}
9571
9572	static int tg_cfs_schedulable_down(struct task_group *tg, void *data)
9573	{
9574	struct cfs_schedulable_data *d = data;
9575	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
9576	s64 quota = 0, parent_quota = -1;
9577
9578	if (!tg->parent) {
9579	quota = RUNTIME_INF;
9580	} else {
9581	struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth;
9582
9583	quota = normalize_cfs_quota(tg, d);
9584	parent_quota = parent_b->hierarchical_quota;
9585
9586	/*
9587	* Ensure max(child_quota) <= parent_quota. On cgroup2,
9588	* always take the non-RUNTIME_INF min. On cgroup1, only
9589	* inherit when no limit is set. In both cases this is used
9590	* by the scheduler to determine if a given CFS task has a
9591	* bandwidth constraint at some higher level.
9592	*/
9593	if (cgroup_subsys_on_dfl(cpu_cgrp_subsys)) {
9594	if (quota == RUNTIME_INF)
9595	quota = parent_quota;
9596	else if (parent_quota != RUNTIME_INF)
9597	quota = min(quota, parent_quota);
9598	} else {
9599	if (quota == RUNTIME_INF)
9600	quota = parent_quota;
9601	else if (parent_quota != RUNTIME_INF && quota > parent_quota)
9602	return -EINVAL;
9603	}
9604	}
9605	cfs_b->hierarchical_quota = quota;
9606
9607	return 0;
9608	}
9609
9610	static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota)
9611	{
9612	struct cfs_schedulable_data data = {
9613	.tg = tg,
9614	.period = period,
9615	.quota = quota,
9616	};
9617
9618	if (quota != RUNTIME_INF) {
9619	do_div(data.period, NSEC_PER_USEC);
9620	do_div(data.quota, NSEC_PER_USEC);
9621	}
9622
9623	guard(rcu)();
9624	return walk_tg_tree(down: tg_cfs_schedulable_down, up: tg_nop, data: &data);
9625	}
9626
9627	static int cpu_cfs_stat_show(struct seq_file *sf, void *v)
9628	{
9629	struct task_group *tg = css_tg(css: seq_css(seq: sf));
9630	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
9631
9632	seq_printf(m: sf, fmt: "nr_periods %d\n", cfs_b->nr_periods);
9633	seq_printf(m: sf, fmt: "nr_throttled %d\n", cfs_b->nr_throttled);
9634	seq_printf(m: sf, fmt: "throttled_time %llu\n", cfs_b->throttled_time);
9635
9636	if (schedstat_enabled() && tg != &root_task_group) {
9637	struct sched_statistics *stats;
9638	u64 ws = 0;
9639	int i;
9640
9641	for_each_possible_cpu(i) {
9642	stats = __schedstats_from_se(se: tg->se[i]);
9643	ws += schedstat_val(stats->wait_sum);
9644	}
9645
9646	seq_printf(m: sf, fmt: "wait_sum %llu\n", ws);
9647	}
9648
9649	seq_printf(m: sf, fmt: "nr_bursts %d\n", cfs_b->nr_burst);
9650	seq_printf(m: sf, fmt: "burst_time %llu\n", cfs_b->burst_time);
9651
9652	return 0;
9653	}
9654
9655	static u64 throttled_time_self(struct task_group *tg)
9656	{
9657	int i;
9658	u64 total = 0;
9659
9660	for_each_possible_cpu(i) {
9661	total += READ_ONCE(tg->cfs_rq[i]->throttled_clock_self_time);
9662	}
9663
9664	return total;
9665	}
9666
9667	static int cpu_cfs_local_stat_show(struct seq_file *sf, void *v)
9668	{
9669	struct task_group *tg = css_tg(css: seq_css(seq: sf));
9670
9671	seq_printf(m: sf, fmt: "throttled_time %llu\n", throttled_time_self(tg));
9672
9673	return 0;
9674	}
9675	#endif /* CONFIG_CFS_BANDWIDTH */
9676
9677	#ifdef CONFIG_GROUP_SCHED_BANDWIDTH
9678	const u64 max_bw_quota_period_us = 1 * USEC_PER_SEC; /* 1s */
9679	static const u64 min_bw_quota_period_us = 1 * USEC_PER_MSEC; /* 1ms */
9680	/* More than 203 days if BW_SHIFT equals 20. */
9681	static const u64 max_bw_runtime_us = MAX_BW;
9682
9683	static void tg_bandwidth(struct task_group *tg,
9684	u64 *period_us_p, u64 *quota_us_p, u64 *burst_us_p)
9685	{
9686	#ifdef CONFIG_CFS_BANDWIDTH
9687	if (period_us_p)
9688	*period_us_p = tg_get_cfs_period(tg);
9689	if (quota_us_p)
9690	*quota_us_p = tg_get_cfs_quota(tg);
9691	if (burst_us_p)
9692	*burst_us_p = tg_get_cfs_burst(tg);
9693	#else /* !CONFIG_CFS_BANDWIDTH */
9694	if (period_us_p)
9695	*period_us_p = tg->scx.bw_period_us;
9696	if (quota_us_p)
9697	*quota_us_p = tg->scx.bw_quota_us;
9698	if (burst_us_p)
9699	*burst_us_p = tg->scx.bw_burst_us;
9700	#endif /* CONFIG_CFS_BANDWIDTH */
9701	}
9702
9703	static u64 cpu_period_read_u64(struct cgroup_subsys_state *css,
9704	struct cftype *cft)
9705	{
9706	u64 period_us;
9707
9708	tg_bandwidth(tg: css_tg(css), period_us_p: &period_us, NULL, NULL);
9709	return period_us;
9710	}
9711
9712	static int tg_set_bandwidth(struct task_group *tg,
9713	u64 period_us, u64 quota_us, u64 burst_us)
9714	{
9715	const u64 max_usec = U64_MAX / NSEC_PER_USEC;
9716	int ret = 0;
9717
9718	if (tg == &root_task_group)
9719	return -EINVAL;
9720
9721	/* Values should survive translation to nsec */
9722	if (period_us > max_usec ||
9723	(quota_us != RUNTIME_INF && quota_us > max_usec) ||
9724	burst_us > max_usec)
9725	return -EINVAL;
9726
9727	/*
9728	* Ensure we have some amount of bandwidth every period. This is to
9729	* prevent reaching a state of large arrears when throttled via
9730	* entity_tick() resulting in prolonged exit starvation.
9731	*/
9732	if (quota_us < min_bw_quota_period_us ||
9733	period_us < min_bw_quota_period_us)
9734	return -EINVAL;
9735
9736	/*
9737	* Likewise, bound things on the other side by preventing insane quota
9738	* periods. This also allows us to normalize in computing quota
9739	* feasibility.
9740	*/
9741	if (period_us > max_bw_quota_period_us)
9742	return -EINVAL;
9743
9744	/*
9745	* Bound quota to defend quota against overflow during bandwidth shift.
9746	*/
9747	if (quota_us != RUNTIME_INF && quota_us > max_bw_runtime_us)
9748	return -EINVAL;
9749
9750	if (quota_us != RUNTIME_INF && (burst_us > quota_us ||
9751	burst_us + quota_us > max_bw_runtime_us))
9752	return -EINVAL;
9753
9754	#ifdef CONFIG_CFS_BANDWIDTH
9755	ret = tg_set_cfs_bandwidth(tg, period_us, quota_us, burst_us);
9756	#endif /* CONFIG_CFS_BANDWIDTH */
9757	if (!ret)
9758	scx_group_set_bandwidth(tg, period_us, quota_us, burst_us);
9759	return ret;
9760	}
9761
9762	static s64 cpu_quota_read_s64(struct cgroup_subsys_state *css,
9763	struct cftype *cft)
9764	{
9765	u64 quota_us;
9766
9767	tg_bandwidth(tg: css_tg(css), NULL, quota_us_p: &quota_us, NULL);
9768	return quota_us; /* (s64)RUNTIME_INF becomes -1 */
9769	}
9770
9771	static u64 cpu_burst_read_u64(struct cgroup_subsys_state *css,
9772	struct cftype *cft)
9773	{
9774	u64 burst_us;
9775
9776	tg_bandwidth(tg: css_tg(css), NULL, NULL, burst_us_p: &burst_us);
9777	return burst_us;
9778	}
9779
9780	static int cpu_period_write_u64(struct cgroup_subsys_state *css,
9781	struct cftype *cftype, u64 period_us)
9782	{
9783	struct task_group *tg = css_tg(css);
9784	u64 quota_us, burst_us;
9785
9786	tg_bandwidth(tg, NULL, quota_us_p: &quota_us, burst_us_p: &burst_us);
9787	return tg_set_bandwidth(tg, period_us, quota_us, burst_us);
9788	}
9789
9790	static int cpu_quota_write_s64(struct cgroup_subsys_state *css,
9791	struct cftype *cftype, s64 quota_us)
9792	{
9793	struct task_group *tg = css_tg(css);
9794	u64 period_us, burst_us;
9795
9796	if (quota_us < 0)
9797	quota_us = RUNTIME_INF;
9798
9799	tg_bandwidth(tg, period_us_p: &period_us, NULL, burst_us_p: &burst_us);
9800	return tg_set_bandwidth(tg, period_us, quota_us, burst_us);
9801	}
9802
9803	static int cpu_burst_write_u64(struct cgroup_subsys_state *css,
9804	struct cftype *cftype, u64 burst_us)
9805	{
9806	struct task_group *tg = css_tg(css);
9807	u64 period_us, quota_us;
9808
9809	tg_bandwidth(tg, period_us_p: &period_us, quota_us_p: &quota_us, NULL);
9810	return tg_set_bandwidth(tg, period_us, quota_us, burst_us);
9811	}
9812	#endif /* CONFIG_GROUP_SCHED_BANDWIDTH */
9813
9814	#ifdef CONFIG_RT_GROUP_SCHED
9815	static int cpu_rt_runtime_write(struct cgroup_subsys_state *css,
9816	struct cftype *cft, s64 val)
9817	{
9818	return sched_group_set_rt_runtime(tg: css_tg(css), rt_runtime_us: val);
9819	}
9820
9821	static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css,
9822	struct cftype *cft)
9823	{
9824	return sched_group_rt_runtime(tg: css_tg(css));
9825	}
9826
9827	static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css,
9828	struct cftype *cftype, u64 rt_period_us)
9829	{
9830	return sched_group_set_rt_period(tg: css_tg(css), rt_period_us);
9831	}
9832
9833	static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css,
9834	struct cftype *cft)
9835	{
9836	return sched_group_rt_period(tg: css_tg(css));
9837	}
9838	#endif /* CONFIG_RT_GROUP_SCHED */
9839
9840	#ifdef CONFIG_GROUP_SCHED_WEIGHT
9841	static s64 cpu_idle_read_s64(struct cgroup_subsys_state *css,
9842	struct cftype *cft)
9843	{
9844	return css_tg(css)->idle;
9845	}
9846
9847	static int cpu_idle_write_s64(struct cgroup_subsys_state *css,
9848	struct cftype *cft, s64 idle)
9849	{
9850	int ret;
9851
9852	ret = sched_group_set_idle(tg: css_tg(css), idle);
9853	if (!ret)
9854	scx_group_set_idle(tg: css_tg(css), idle);
9855	return ret;
9856	}
9857	#endif /* CONFIG_GROUP_SCHED_WEIGHT */
9858
9859	static struct cftype cpu_legacy_files[] = {
9860	#ifdef CONFIG_GROUP_SCHED_WEIGHT
9861	{
9862	.name = "shares",
9863	.read_u64 = cpu_shares_read_u64,
9864	.write_u64 = cpu_shares_write_u64,
9865	},
9866	{
9867	.name = "idle",
9868	.read_s64 = cpu_idle_read_s64,
9869	.write_s64 = cpu_idle_write_s64,
9870	},
9871	#endif
9872	#ifdef CONFIG_GROUP_SCHED_BANDWIDTH
9873	{
9874	.name = "cfs_period_us",
9875	.read_u64 = cpu_period_read_u64,
9876	.write_u64 = cpu_period_write_u64,
9877	},
9878	{
9879	.name = "cfs_quota_us",
9880	.read_s64 = cpu_quota_read_s64,
9881	.write_s64 = cpu_quota_write_s64,
9882	},
9883	{
9884	.name = "cfs_burst_us",
9885	.read_u64 = cpu_burst_read_u64,
9886	.write_u64 = cpu_burst_write_u64,
9887	},
9888	#endif
9889	#ifdef CONFIG_CFS_BANDWIDTH
9890	{
9891	.name = "stat",
9892	.seq_show = cpu_cfs_stat_show,
9893	},
9894	{
9895	.name = "stat.local",
9896	.seq_show = cpu_cfs_local_stat_show,
9897	},
9898	#endif
9899	#ifdef CONFIG_UCLAMP_TASK_GROUP
9900	{
9901	.name = "uclamp.min",
9902	.flags = CFTYPE_NOT_ON_ROOT,
9903	.seq_show = cpu_uclamp_min_show,
9904	.write = cpu_uclamp_min_write,
9905	},
9906	{
9907	.name = "uclamp.max",
9908	.flags = CFTYPE_NOT_ON_ROOT,
9909	.seq_show = cpu_uclamp_max_show,
9910	.write = cpu_uclamp_max_write,
9911	},
9912	#endif
9913	{ } /* Terminate */
9914	};
9915
9916	#ifdef CONFIG_RT_GROUP_SCHED
9917	static struct cftype rt_group_files[] = {
9918	{
9919	.name = "rt_runtime_us",
9920	.read_s64 = cpu_rt_runtime_read,
9921	.write_s64 = cpu_rt_runtime_write,
9922	},
9923	{
9924	.name = "rt_period_us",
9925	.read_u64 = cpu_rt_period_read_uint,
9926	.write_u64 = cpu_rt_period_write_uint,
9927	},
9928	{ } /* Terminate */
9929	};
9930
9931	# ifdef CONFIG_RT_GROUP_SCHED_DEFAULT_DISABLED
9932	DEFINE_STATIC_KEY_FALSE(rt_group_sched);
9933	# else
9934	DEFINE_STATIC_KEY_TRUE(rt_group_sched);
9935	# endif
9936
9937	static int __init setup_rt_group_sched(char *str)
9938	{
9939	long val;
9940
9941	if (kstrtol(s: str, base: 0, res: &val) || val < 0 || val > 1) {
9942	pr_warn("Unable to set rt_group_sched\n");
9943	return 1;
9944	}
9945	if (val)
9946	static_branch_enable(&rt_group_sched);
9947	else
9948	static_branch_disable(&rt_group_sched);
9949
9950	return 1;
9951	}
9952	__setup("rt_group_sched=", setup_rt_group_sched);
9953
9954	static int __init cpu_rt_group_init(void)
9955	{
9956	if (!rt_group_sched_enabled())
9957	return 0;
9958
9959	WARN_ON(cgroup_add_legacy_cftypes(&cpu_cgrp_subsys, rt_group_files));
9960	return 0;
9961	}
9962	subsys_initcall(cpu_rt_group_init);
9963	#endif /* CONFIG_RT_GROUP_SCHED */
9964
9965	static int cpu_extra_stat_show(struct seq_file *sf,
9966	struct cgroup_subsys_state *css)
9967	{
9968	#ifdef CONFIG_CFS_BANDWIDTH
9969	{
9970	struct task_group *tg = css_tg(css);
9971	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
9972	u64 throttled_usec, burst_usec;
9973
9974	throttled_usec = cfs_b->throttled_time;
9975	do_div(throttled_usec, NSEC_PER_USEC);
9976	burst_usec = cfs_b->burst_time;
9977	do_div(burst_usec, NSEC_PER_USEC);
9978
9979	seq_printf(m: sf, fmt: "nr_periods %d\n"
9980	"nr_throttled %d\n"
9981	"throttled_usec %llu\n"
9982	"nr_bursts %d\n"
9983	"burst_usec %llu\n",
9984	cfs_b->nr_periods, cfs_b->nr_throttled,
9985	throttled_usec, cfs_b->nr_burst, burst_usec);
9986	}
9987	#endif /* CONFIG_CFS_BANDWIDTH */
9988	return 0;
9989	}
9990
9991	static int cpu_local_stat_show(struct seq_file *sf,
9992	struct cgroup_subsys_state *css)
9993	{
9994	#ifdef CONFIG_CFS_BANDWIDTH
9995	{
9996	struct task_group *tg = css_tg(css);
9997	u64 throttled_self_usec;
9998
9999	throttled_self_usec = throttled_time_self(tg);
10000	do_div(throttled_self_usec, NSEC_PER_USEC);
10001
10002	seq_printf(m: sf, fmt: "throttled_usec %llu\n",
10003	throttled_self_usec);
10004	}
10005	#endif
10006	return 0;
10007	}
10008
10009	#ifdef CONFIG_GROUP_SCHED_WEIGHT
10010
10011	static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css,
10012	struct cftype *cft)
10013	{
10014	return sched_weight_to_cgroup(weight: tg_weight(tg: css_tg(css)));
10015	}
10016
10017	static int cpu_weight_write_u64(struct cgroup_subsys_state *css,
10018	struct cftype *cft, u64 cgrp_weight)
10019	{
10020	unsigned long weight;
10021	int ret;
10022
10023	if (cgrp_weight < CGROUP_WEIGHT_MIN || cgrp_weight > CGROUP_WEIGHT_MAX)
10024	return -ERANGE;
10025
10026	weight = sched_weight_from_cgroup(cgrp_weight);
10027
10028	ret = sched_group_set_shares(tg: css_tg(css), scale_load(weight));
10029	if (!ret)
10030	scx_group_set_weight(tg: css_tg(css), cgrp_weight);
10031	return ret;
10032	}
10033
10034	static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css,
10035	struct cftype *cft)
10036	{
10037	unsigned long weight = tg_weight(tg: css_tg(css));
10038	int last_delta = INT_MAX;
10039	int prio, delta;
10040
10041	/* find the closest nice value to the current weight */
10042	for (prio = 0; prio < ARRAY_SIZE(sched_prio_to_weight); prio++) {
10043	delta = abs(sched_prio_to_weight[prio] - weight);
10044	if (delta >= last_delta)
10045	break;
10046	last_delta = delta;
10047	}
10048
10049	return PRIO_TO_NICE(prio - 1 + MAX_RT_PRIO);
10050	}
10051
10052	static int cpu_weight_nice_write_s64(struct cgroup_subsys_state *css,
10053	struct cftype *cft, s64 nice)
10054	{
10055	unsigned long weight;
10056	int idx, ret;
10057
10058	if (nice < MIN_NICE || nice > MAX_NICE)
10059	return -ERANGE;
10060
10061	idx = NICE_TO_PRIO(nice) - MAX_RT_PRIO;
10062	idx = array_index_nospec(idx, 40);
10063	weight = sched_prio_to_weight[idx];
10064
10065	ret = sched_group_set_shares(tg: css_tg(css), scale_load(weight));
10066	if (!ret)
10067	scx_group_set_weight(tg: css_tg(css),
10068	cgrp_weight: sched_weight_to_cgroup(weight));
10069	return ret;
10070	}
10071	#endif /* CONFIG_GROUP_SCHED_WEIGHT */
10072
10073	static void __maybe_unused cpu_period_quota_print(struct seq_file *sf,
10074	long period, long quota)
10075	{
10076	if (quota < 0)
10077	seq_puts(m: sf, s: "max");
10078	else
10079	seq_printf(m: sf, fmt: "%ld", quota);
10080
10081	seq_printf(m: sf, fmt: " %ld\n", period);
10082	}
10083
10084	/* caller should put the current value in *@periodp before calling */
10085	static int __maybe_unused cpu_period_quota_parse(char *buf, u64 *period_us_p,
10086	u64 *quota_us_p)
10087	{
10088	char tok[21]; /* U64_MAX */
10089
10090	if (sscanf(buf, "%20s %llu", tok, period_us_p) < 1)
10091	return -EINVAL;
10092
10093	if (sscanf(tok, "%llu", quota_us_p) < 1) {
10094	if (!strcmp(tok, "max"))
10095	*quota_us_p = RUNTIME_INF;
10096	else
10097	return -EINVAL;
10098	}
10099
10100	return 0;
10101	}
10102
10103	#ifdef CONFIG_GROUP_SCHED_BANDWIDTH
10104	static int cpu_max_show(struct seq_file *sf, void *v)
10105	{
10106	struct task_group *tg = css_tg(css: seq_css(seq: sf));
10107	u64 period_us, quota_us;
10108
10109	tg_bandwidth(tg, period_us_p: &period_us, quota_us_p: &quota_us, NULL);
10110	cpu_period_quota_print(sf, period: period_us, quota: quota_us);
10111	return 0;
10112	}
10113
10114	static ssize_t cpu_max_write(struct kernfs_open_file *of,
10115	char *buf, size_t nbytes, loff_t off)
10116	{
10117	struct task_group *tg = css_tg(css: of_css(of));
10118	u64 period_us, quota_us, burst_us;
10119	int ret;
10120
10121	tg_bandwidth(tg, period_us_p: &period_us, NULL, burst_us_p: &burst_us);
10122	ret = cpu_period_quota_parse(buf, period_us_p: &period_us, quota_us_p: &quota_us);
10123	if (!ret)
10124	ret = tg_set_bandwidth(tg, period_us, quota_us, burst_us);
10125	return ret ?: nbytes;
10126	}
10127	#endif /* CONFIG_CFS_BANDWIDTH */
10128
10129	static struct cftype cpu_files[] = {
10130	#ifdef CONFIG_GROUP_SCHED_WEIGHT
10131	{
10132	.name = "weight",
10133	.flags = CFTYPE_NOT_ON_ROOT,
10134	.read_u64 = cpu_weight_read_u64,
10135	.write_u64 = cpu_weight_write_u64,
10136	},
10137	{
10138	.name = "weight.nice",
10139	.flags = CFTYPE_NOT_ON_ROOT,
10140	.read_s64 = cpu_weight_nice_read_s64,
10141	.write_s64 = cpu_weight_nice_write_s64,
10142	},
10143	{
10144	.name = "idle",
10145	.flags = CFTYPE_NOT_ON_ROOT,
10146	.read_s64 = cpu_idle_read_s64,
10147	.write_s64 = cpu_idle_write_s64,
10148	},
10149	#endif
10150	#ifdef CONFIG_GROUP_SCHED_BANDWIDTH
10151	{
10152	.name = "max",
10153	.flags = CFTYPE_NOT_ON_ROOT,
10154	.seq_show = cpu_max_show,
10155	.write = cpu_max_write,
10156	},
10157	{
10158	.name = "max.burst",
10159	.flags = CFTYPE_NOT_ON_ROOT,
10160	.read_u64 = cpu_burst_read_u64,
10161	.write_u64 = cpu_burst_write_u64,
10162	},
10163	#endif /* CONFIG_CFS_BANDWIDTH */
10164	#ifdef CONFIG_UCLAMP_TASK_GROUP
10165	{
10166	.name = "uclamp.min",
10167	.flags = CFTYPE_NOT_ON_ROOT,
10168	.seq_show = cpu_uclamp_min_show,
10169	.write = cpu_uclamp_min_write,
10170	},
10171	{
10172	.name = "uclamp.max",
10173	.flags = CFTYPE_NOT_ON_ROOT,
10174	.seq_show = cpu_uclamp_max_show,
10175	.write = cpu_uclamp_max_write,
10176	},
10177	#endif /* CONFIG_UCLAMP_TASK_GROUP */
10178	{ } /* terminate */
10179	};
10180
10181	struct cgroup_subsys cpu_cgrp_subsys = {
10182	.css_alloc = cpu_cgroup_css_alloc,
10183	.css_online = cpu_cgroup_css_online,
10184	.css_offline = cpu_cgroup_css_offline,
10185	.css_released = cpu_cgroup_css_released,
10186	.css_free = cpu_cgroup_css_free,
10187	.css_extra_stat_show = cpu_extra_stat_show,
10188	.css_local_stat_show = cpu_local_stat_show,
10189	.can_attach = cpu_cgroup_can_attach,
10190	.attach = cpu_cgroup_attach,
10191	.cancel_attach = cpu_cgroup_cancel_attach,
10192	.legacy_cftypes = cpu_legacy_files,
10193	.dfl_cftypes = cpu_files,
10194	.early_init = true,
10195	.threaded = true,
10196	};
10197
10198	#endif /* CONFIG_CGROUP_SCHED */
10199
10200	void dump_cpu_task(int cpu)
10201	{
10202	if (in_hardirq() && cpu == smp_processor_id()) {
10203	struct pt_regs *regs;
10204
10205	regs = get_irq_regs();
10206	if (regs) {
10207	show_regs(regs);
10208	return;
10209	}
10210	}
10211
10212	if (trigger_single_cpu_backtrace(cpu))
10213	return;
10214
10215	pr_info("Task dump for CPU %d:\n", cpu);
10216	sched_show_task(cpu_curr(cpu));
10217	}
10218
10219	/*
10220	* Nice levels are multiplicative, with a gentle 10% change for every
10221	* nice level changed. I.e. when a CPU-bound task goes from nice 0 to
10222	* nice 1, it will get ~10% less CPU time than another CPU-bound task
10223	* that remained on nice 0.
10224	*
10225	* The "10% effect" is relative and cumulative: from _any_ nice level,
10226	* if you go up 1 level, it's -10% CPU usage, if you go down 1 level
10227	* it's +10% CPU usage. (to achieve that we use a multiplier of 1.25.
10228	* If a task goes up by ~10% and another task goes down by ~10% then
10229	* the relative distance between them is ~25%.)
10230	*/
10231	const int sched_prio_to_weight[40] = {
10232	/* -20 */ 88761, 71755, 56483, 46273, 36291,
10233	/* -15 */ 29154, 23254, 18705, 14949, 11916,
10234	/* -10 */ 9548, 7620, 6100, 4904, 3906,
10235	/* -5 */ 3121, 2501, 1991, 1586, 1277,
10236	/* 0 */ 1024, 820, 655, 526, 423,
10237	/* 5 */ 335, 272, 215, 172, 137,
10238	/* 10 */ 110, 87, 70, 56, 45,
10239	/* 15 */ 36, 29, 23, 18, 15,
10240	};
10241
10242	/*
10243	* Inverse (2^32/x) values of the sched_prio_to_weight[] array, pre-calculated.
10244	*
10245	* In cases where the weight does not change often, we can use the
10246	* pre-calculated inverse to speed up arithmetics by turning divisions
10247	* into multiplications:
10248	*/
10249	const u32 sched_prio_to_wmult[40] = {
10250	/* -20 */ 48388, 59856, 76040, 92818, 118348,
10251	/* -15 */ 147320, 184698, 229616, 287308, 360437,
10252	/* -10 */ 449829, 563644, 704093, 875809, 1099582,
10253	/* -5 */ 1376151, 1717300, 2157191, 2708050, 3363326,
10254	/* 0 */ 4194304, 5237765, 6557202, 8165337, 10153587,
10255	/* 5 */ 12820798, 15790321, 19976592, 24970740, 31350126,
10256	/* 10 */ 39045157, 49367440, 61356676, 76695844, 95443717,
10257	/* 15 */ 119304647, 148102320, 186737708, 238609294, 286331153,
10258	};
10259
10260	void call_trace_sched_update_nr_running(struct rq *rq, int count)
10261	{
10262	trace_sched_update_nr_running_tp(rq, change: count);
10263	}
10264
10265	#ifdef CONFIG_SCHED_MM_CID
10266	/*
10267	* Concurrency IDentifier management
10268	*
10269	* Serialization rules:
10270	*
10271	* mm::mm_cid::mutex: Serializes fork() and exit() and therefore
10272	* protects mm::mm_cid::users.
10273	*
10274	* mm::mm_cid::lock: Serializes mm_update_max_cids() and
10275	* mm_update_cpus_allowed(). Nests in mm_cid::mutex
10276	* and runqueue lock.
10277	*
10278	* The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks
10279	* and can only be modified with atomic operations.
10280	*
10281	* The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
10282	* lock.
10283	*
10284	* CID ownership:
10285	*
10286	* A CID is either owned by a task (stored in task_struct::mm_cid.cid) or
10287	* by a CPU (stored in mm::mm_cid.pcpu::cid). CIDs owned by CPUs have the
10288	* MM_CID_ONCPU bit set. During transition from CPU to task ownership mode,
10289	* MM_CID_TRANSIT is set on the per task CIDs. When this bit is set the
10290	* task needs to drop the CID into the pool when scheduling out. Both bits
10291	* (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
10292	* actually handed over to user space in the RSEQ memory.
10293	*
10294	* Mode switching:
10295	*
10296	* Switching to per CPU mode happens when the user count becomes greater
10297	* than the maximum number of CIDs, which is calculated by:
10298	*
10299	* opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
10300	* max_cids = min(1.25 * opt_cids, num_possible_cpus());
10301	*
10302	* The +25% allowance is useful for tight CPU masks in scenarios where only
10303	* a few threads are created and destroyed to avoid frequent mode
10304	* switches. Though this allowance shrinks, the closer opt_cids becomes to
10305	* num_possible_cpus(), which is the (unfortunate) hard ABI limit.
10306	*
10307	* At the point of switching to per CPU mode the new user is not yet
10308	* visible in the system, so the task which initiated the fork() runs the
10309	* fixup function: mm_cid_fixup_tasks_to_cpu() walks the thread list and
10310	* either transfers each tasks owned CID to the CPU the task runs on or
10311	* drops it into the CID pool if a task is not on a CPU at that point in
10312	* time. Tasks which schedule in before the task walk reaches them do the
10313	* handover in mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes
10314	* it's guaranteed that no task related to that MM owns a CID anymore.
10315	*
10316	* Switching back to task mode happens when the user count goes below the
10317	* threshold which was recorded on the per CPU mode switch:
10318	*
10319	* pcpu_thrs = min(opt_cids - (opt_cids / 4), num_possible_cpus() / 2);
10320	*
10321	* This threshold is updated when a affinity change increases the number of
10322	* allowed CPUs for the MM, which might cause a switch back to per task
10323	* mode.
10324	*
10325	* If the switch back was initiated by a exiting task, then that task runs
10326	* the fixup function. If it was initiated by a affinity change, then it's
10327	* run either in the deferred update function in context of a workqueue or
10328	* by a task which forks a new one or by a task which exits. Whatever
10329	* happens first. mm_cid_fixup_cpus_to_task() walks through the possible
10330	* CPUs and either transfers the CPU owned CIDs to a related task which
10331	* runs on the CPU or drops it into the pool. Tasks which schedule in on a
10332	* CPU which the walk did not cover yet do the handover themself.
10333	*
10334	* This transition from CPU to per task ownership happens in two phases:
10335	*
10336	* 1) mm:mm_cid.transit contains MM_CID_TRANSIT This is OR'ed on the task
10337	* CID and denotes that the CID is only temporarily owned by the
10338	* task. When it schedules out the task drops the CID back into the
10339	* pool if this bit is set.
10340	*
10341	* 2) The initiating context walks the per CPU space and after completion
10342	* clears mm:mm_cid.transit. So after that point the CIDs are strictly
10343	* task owned again.
10344	*
10345	* This two phase transition is required to prevent CID space exhaustion
10346	* during the transition as a direct transfer of ownership would fail if
10347	* two tasks are scheduled in on the same CPU before the fixup freed per
10348	* CPU CIDs.
10349	*
10350	* When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
10351	* related to that MM is owned by a CPU anymore.
10352	*/
10353
10354	/*
10355	* Update the CID range properties when the constraints change. Invoked via
10356	* fork(), exit() and affinity changes
10357	*/
10358	static void __mm_update_max_cids(struct mm_mm_cid *mc)
10359	{
10360	unsigned int opt_cids, max_cids;
10361
10362	/* Calculate the new optimal constraint */
10363	opt_cids = min(mc->nr_cpus_allowed, mc->users);
10364
10365	/* Adjust the maximum CIDs to +25% limited by the number of possible CPUs */
10366	max_cids = min(opt_cids + (opt_cids / 4), num_possible_cpus());
10367	WRITE_ONCE(mc->max_cids, max_cids);
10368	}
10369
10370	static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
10371	{
10372	unsigned int opt_cids;
10373
10374	opt_cids = min(mc->nr_cpus_allowed, mc->users);
10375	/* Has to be at least 1 because 0 indicates PCPU mode off */
10376	return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1);
10377	}
10378
10379	static bool mm_update_max_cids(struct mm_struct *mm)
10380	{
10381	struct mm_mm_cid *mc = &mm->mm_cid;
10382
10383	lockdep_assert_held(&mm->mm_cid.lock);
10384
10385	/* Clear deferred mode switch flag. A change is handled by the caller */
10386	mc->update_deferred = false;
10387	__mm_update_max_cids(mc);
10388
10389	/* Check whether owner mode must be changed */
10390	if (!mc->percpu) {
10391	/* Enable per CPU mode when the number of users is above max_cids */
10392	if (mc->users > mc->max_cids)
10393	mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
10394	} else {
10395	/* Switch back to per task if user count under threshold */
10396	if (mc->users < mc->pcpu_thrs)
10397	mc->pcpu_thrs = 0;
10398	}
10399
10400	/* Mode change required? */
10401	if (!!mc->percpu == !!mc->pcpu_thrs)
10402	return false;
10403	/* When switching back to per TASK mode, set the transition flag */
10404	if (!mc->pcpu_thrs)
10405	WRITE_ONCE(mc->transit, MM_CID_TRANSIT);
10406	WRITE_ONCE(mc->percpu, !!mc->pcpu_thrs);
10407	return true;
10408	}
10409
10410	static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
10411	{
10412	struct cpumask *mm_allowed;
10413	struct mm_mm_cid *mc;
10414	unsigned int weight;
10415
10416	if (!mm || !READ_ONCE(mm->mm_cid.users))
10417	return;
10418	/*
10419	* mm::mm_cid::mm_cpus_allowed is the superset of each threads
10420	* allowed CPUs mask which means it can only grow.
10421	*/
10422	mc = &mm->mm_cid;
10423	guard(raw_spinlock)(l: &mc->lock);
10424	mm_allowed = mm_cpus_allowed(mm);
10425	weight = cpumask_weighted_or(dstp: mm_allowed, src1p: mm_allowed, src2p: affmsk);
10426	if (weight == mc->nr_cpus_allowed)
10427	return;
10428
10429	WRITE_ONCE(mc->nr_cpus_allowed, weight);
10430	__mm_update_max_cids(mc);
10431	if (!mc->percpu)
10432	return;
10433
10434	/* Adjust the threshold to the wider set */
10435	mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
10436	/* Switch back to per task mode? */
10437	if (mc->users >= mc->pcpu_thrs)
10438	return;
10439
10440	/* Don't queue twice */
10441	if (mc->update_deferred)
10442	return;
10443
10444	/* Queue the irq work, which schedules the real work */
10445	mc->update_deferred = true;
10446	irq_work_queue(work: &mc->irq_work);
10447	}
10448
10449	static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
10450	{
10451	if (cid_on_cpu(cid: t->mm_cid.cid)) {
10452	unsigned int cid = cpu_cid_to_cid(cid: t->mm_cid.cid);
10453
10454	t->mm_cid.cid = cid_to_transit_cid(cid);
10455	pcp->cid = t->mm_cid.cid;
10456	}
10457	}
10458
10459	static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
10460	{
10461	unsigned int cpu;
10462
10463	/* Walk the CPUs and fixup all stale CIDs */
10464	for_each_possible_cpu(cpu) {
10465	struct mm_cid_pcpu *pcp = per_cpu_ptr(mm->mm_cid.pcpu, cpu);
10466	struct rq *rq = cpu_rq(cpu);
10467
10468	/* Remote access to mm::mm_cid::pcpu requires rq_lock */
10469	guard(rq_lock_irq)(l: rq);
10470	/* Is the CID still owned by the CPU? */
10471	if (cid_on_cpu(cid: pcp->cid)) {
10472	/*
10473	* If rq->curr has @mm, transfer it with the
10474	* transition bit set. Otherwise drop it.
10475	*/
10476	if (rq->curr->mm == mm && rq->curr->mm_cid.active)
10477	mm_cid_transit_to_task(t: rq->curr, pcp);
10478	else
10479	mm_drop_cid_on_cpu(mm, pcp);
10480
10481	} else if (rq->curr->mm == mm && rq->curr->mm_cid.active) {
10482	unsigned int cid = rq->curr->mm_cid.cid;
10483
10484	/* Ensure it has the transition bit set */
10485	if (!cid_in_transit(cid)) {
10486	cid = cid_to_transit_cid(cid);
10487	rq->curr->mm_cid.cid = cid;
10488	pcp->cid = cid;
10489	}
10490	}
10491	}
10492	/* Clear the transition bit */
10493	WRITE_ONCE(mm->mm_cid.transit, 0);
10494	}
10495
10496	static inline void mm_cid_transfer_to_cpu(struct task_struct *t, struct mm_cid_pcpu *pcp)
10497	{
10498	if (cid_on_task(cid: t->mm_cid.cid)) {
10499	t->mm_cid.cid = cid_to_cpu_cid(cid: t->mm_cid.cid);
10500	pcp->cid = t->mm_cid.cid;
10501	}
10502	}
10503
10504	static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm)
10505	{
10506	/* Remote access to mm::mm_cid::pcpu requires rq_lock */
10507	guard(task_rq_lock)(l: t);
10508	/* If the task is not active it is not in the users count */
10509	if (!t->mm_cid.active)
10510	return false;
10511	if (cid_on_task(cid: t->mm_cid.cid)) {
10512	/* If running on the CPU, transfer the CID, otherwise drop it */
10513	if (task_rq(t)->curr == t)
10514	mm_cid_transfer_to_cpu(t, per_cpu_ptr(mm->mm_cid.pcpu, task_cpu(t)));
10515	else
10516	mm_unset_cid_on_task(t);
10517	}
10518	return true;
10519	}
10520
10521	static void mm_cid_fixup_tasks_to_cpus(void)
10522	{
10523	struct mm_struct *mm = current->mm;
10524	struct task_struct *p, *t;
10525	unsigned int users;
10526
10527	/*
10528	* This can obviously race with a concurrent affinity change, which
10529	* increases the number of allowed CPUs for this mm, but that does
10530	* not affect the mode and only changes the CID constraints. A
10531	* possible switch back to per task mode happens either in the
10532	* deferred handler function or in the next fork()/exit().
10533	*
10534	* The caller has already transferred. The newly incoming task is
10535	* already accounted for, but not yet visible.
10536	*/
10537	users = mm->mm_cid.users - 2;
10538	if (!users)
10539	return;
10540
10541	guard(rcu)();
10542	for_other_threads(current, t) {
10543	if (mm_cid_fixup_task_to_cpu(t, mm))
10544	users--;
10545	}
10546
10547	if (!users)
10548	return;
10549
10550	/* Happens only for VM_CLONE processes. */
10551	for_each_process_thread(p, t) {
10552	if (t == current || t->mm != mm)
10553	continue;
10554	if (mm_cid_fixup_task_to_cpu(t, mm)) {
10555	if (--users == 0)
10556	return;
10557	}
10558	}
10559	}
10560
10561	static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
10562	{
10563	t->mm_cid.active = 1;
10564	mm->mm_cid.users++;
10565	return mm_update_max_cids(mm);
10566	}
10567
10568	void sched_mm_cid_fork(struct task_struct *t)
10569	{
10570	struct mm_struct *mm = t->mm;
10571	bool percpu;
10572
10573	WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
10574
10575	guard(mutex)(T: &mm->mm_cid.mutex);
10576	scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
10577	struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu);
10578
10579	/* First user ? */
10580	if (!mm->mm_cid.users) {
10581	sched_mm_cid_add_user(t, mm);
10582	t->mm_cid.cid = mm_get_cid(mm);
10583	/* Required for execve() */
10584	pcp->cid = t->mm_cid.cid;
10585	return;
10586	}
10587
10588	if (!sched_mm_cid_add_user(t, mm)) {
10589	if (!mm->mm_cid.percpu)
10590	t->mm_cid.cid = mm_get_cid(mm);
10591	return;
10592	}
10593
10594	/* Handle the mode change and transfer current's CID */
10595	percpu = !!mm->mm_cid.percpu;
10596	if (!percpu)
10597	mm_cid_transit_to_task(current, pcp);
10598	else
10599	mm_cid_transfer_to_cpu(current, pcp);
10600	}
10601
10602	if (percpu) {
10603	mm_cid_fixup_tasks_to_cpus();
10604	} else {
10605	mm_cid_fixup_cpus_to_tasks(mm);
10606	t->mm_cid.cid = mm_get_cid(mm);
10607	}
10608	}
10609
10610	static bool sched_mm_cid_remove_user(struct task_struct *t)
10611	{
10612	t->mm_cid.active = 0;
10613	scoped_guard(preempt) {
10614	/* Clear the transition bit */
10615	t->mm_cid.cid = cid_from_transit_cid(cid: t->mm_cid.cid);
10616	mm_unset_cid_on_task(t);
10617	}
10618	t->mm->mm_cid.users--;
10619	return mm_update_max_cids(mm: t->mm);
10620	}
10621
10622	static bool __sched_mm_cid_exit(struct task_struct *t)
10623	{
10624	struct mm_struct *mm = t->mm;
10625
10626	if (!sched_mm_cid_remove_user(t))
10627	return false;
10628	/*
10629	* Contrary to fork() this only deals with a switch back to per
10630	* task mode either because the above decreased users or an
10631	* affinity change increased the number of allowed CPUs and the
10632	* deferred fixup did not run yet.
10633	*/
10634	if (WARN_ON_ONCE(mm->mm_cid.percpu))
10635	return false;
10636	/*
10637	* A failed fork(2) cleanup never gets here, so @current must have
10638	* the same MM as @t. That's true for exit() and the failed
10639	* pthread_create() cleanup case.
10640	*/
10641	if (WARN_ON_ONCE(current->mm != mm))
10642	return false;
10643	return true;
10644	}
10645
10646	/*
10647	* When a task exits, the MM CID held by the task is not longer required as
10648	* the task cannot return to user space.
10649	*/
10650	void sched_mm_cid_exit(struct task_struct *t)
10651	{
10652	struct mm_struct *mm = t->mm;
10653
10654	if (!mm || !t->mm_cid.active)
10655	return;
10656	/*
10657	* Ensure that only one instance is doing MM CID operations within
10658	* a MM. The common case is uncontended. The rare fixup case adds
10659	* some overhead.
10660	*/
10661	scoped_guard(mutex, &mm->mm_cid.mutex) {
10662	/* mm_cid::mutex is sufficient to protect mm_cid::users */
10663	if (likely(mm->mm_cid.users > 1)) {
10664	scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
10665	if (!__sched_mm_cid_exit(t))
10666	return;
10667	/* Mode change required. Transfer currents CID */
10668	mm_cid_transit_to_task(current, this_cpu_ptr(mm->mm_cid.pcpu));
10669	}
10670	mm_cid_fixup_cpus_to_tasks(mm);
10671	return;
10672	}
10673	/* Last user */
10674	scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
10675	/* Required across execve() */
10676	if (t == current)
10677	mm_cid_transit_to_task(t, this_cpu_ptr(mm->mm_cid.pcpu));
10678	/* Ignore mode change. There is nothing to do. */
10679	sched_mm_cid_remove_user(t);
10680	}
10681	}
10682
10683	/*
10684	* As this is the last user (execve(), process exit or failed
10685	* fork(2)) there is no concurrency anymore.
10686	*
10687	* Synchronize eventually pending work to ensure that there are no
10688	* dangling references left. @t->mm_cid.users is zero so nothing
10689	* can queue this work anymore.
10690	*/
10691	irq_work_sync(work: &mm->mm_cid.irq_work);
10692	cancel_work_sync(work: &mm->mm_cid.work);
10693	}
10694
10695	/* Deactivate MM CID allocation across execve() */
10696	void sched_mm_cid_before_execve(struct task_struct *t)
10697	{
10698	sched_mm_cid_exit(t);
10699	}
10700
10701	/* Reactivate MM CID after execve() */
10702	void sched_mm_cid_after_execve(struct task_struct *t)
10703	{
10704	if (t->mm)
10705	sched_mm_cid_fork(t);
10706	}
10707
10708	static void mm_cid_work_fn(struct work_struct *work)
10709	{
10710	struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
10711
10712	guard(mutex)(T: &mm->mm_cid.mutex);
10713	/* Did the last user task exit already? */
10714	if (!mm->mm_cid.users)
10715	return;
10716
10717	scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
10718	/* Have fork() or exit() handled it already? */
10719	if (!mm->mm_cid.update_deferred)
10720	return;
10721	/* This clears mm_cid::update_deferred */
10722	if (!mm_update_max_cids(mm))
10723	return;
10724	/* Affinity changes can only switch back to task mode */
10725	if (WARN_ON_ONCE(mm->mm_cid.percpu))
10726	return;
10727	}
10728	mm_cid_fixup_cpus_to_tasks(mm);
10729	}
10730
10731	static void mm_cid_irq_work(struct irq_work *work)
10732	{
10733	struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work);
10734
10735	/*
10736	* Needs to be unconditional because mm_cid::lock cannot be held
10737	* when scheduling work as mm_update_cpus_allowed() nests inside
10738	* rq::lock and schedule_work() might end up in wakeup...
10739	*/
10740	schedule_work(work: &mm->mm_cid.work);
10741	}
10742
10743	void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
10744	{
10745	mm->mm_cid.max_cids = 0;
10746	mm->mm_cid.percpu = 0;
10747	mm->mm_cid.transit = 0;
10748	mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
10749	mm->mm_cid.users = 0;
10750	mm->mm_cid.pcpu_thrs = 0;
10751	mm->mm_cid.update_deferred = 0;
10752	raw_spin_lock_init(&mm->mm_cid.lock);
10753	mutex_init(&mm->mm_cid.mutex);
10754	mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work);
10755	INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn);
10756	cpumask_copy(dstp: mm_cpus_allowed(mm), srcp: &p->cpus_mask);
10757	bitmap_zero(dst: mm_cidmask(mm), nbits: num_possible_cpus());
10758	}
10759	#else /* CONFIG_SCHED_MM_CID */
10760	static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
10761	#endif /* !CONFIG_SCHED_MM_CID */
10762
10763	static DEFINE_PER_CPU(struct sched_change_ctx, sched_change_ctx);
10764
10765	struct sched_change_ctx *sched_change_begin(struct task_struct *p, unsigned int flags)
10766	{
10767	struct sched_change_ctx *ctx = this_cpu_ptr(&sched_change_ctx);
10768	struct rq *rq = task_rq(p);
10769
10770	/*
10771	* Must exclusively use matched flags since this is both dequeue and
10772	* enqueue.
10773	*/
10774	WARN_ON_ONCE(flags & 0xFFFF0000);
10775
10776	lockdep_assert_rq_held(rq);
10777
10778	if (!(flags & DEQUEUE_NOCLOCK)) {
10779	update_rq_clock(rq);
10780	flags |= DEQUEUE_NOCLOCK;
10781	}
10782
10783	if (flags & DEQUEUE_CLASS) {
10784	if (p->sched_class->switching_from)
10785	p->sched_class->switching_from(rq, p);
10786	}
10787
10788	*ctx = (struct sched_change_ctx){
10789	.p = p,
10790	.flags = flags,
10791	.queued = task_on_rq_queued(p),
10792	.running = task_current_donor(rq, p),
10793	};
10794
10795	if (!(flags & DEQUEUE_CLASS)) {
10796	if (p->sched_class->get_prio)
10797	ctx->prio = p->sched_class->get_prio(rq, p);
10798	else
10799	ctx->prio = p->prio;
10800	}
10801
10802	if (ctx->queued)
10803	dequeue_task(rq, p, flags);
10804	if (ctx->running)
10805	put_prev_task(rq, prev: p);
10806
10807	if ((flags & DEQUEUE_CLASS) && p->sched_class->switched_from)
10808	p->sched_class->switched_from(rq, p);
10809
10810	return ctx;
10811	}
10812
10813	void sched_change_end(struct sched_change_ctx *ctx)
10814	{
10815	struct task_struct *p = ctx->p;
10816	struct rq *rq = task_rq(p);
10817
10818	lockdep_assert_rq_held(rq);
10819
10820	if ((ctx->flags & ENQUEUE_CLASS) && p->sched_class->switching_to)
10821	p->sched_class->switching_to(rq, p);
10822
10823	if (ctx->queued)
10824	enqueue_task(rq, p, flags: ctx->flags);
10825	if (ctx->running)
10826	set_next_task(rq, next: p);
10827
10828	if (ctx->flags & ENQUEUE_CLASS) {
10829	if (p->sched_class->switched_to)
10830	p->sched_class->switched_to(rq, p);
10831	} else {
10832	p->sched_class->prio_changed(rq, p, ctx->prio);
10833	}
10834	}
10835