1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Completely Fair Scheduling (CFS) Class (SCHED_NORMAL/SCHED_BATCH)
4	*
5	* Copyright (C) 2007 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
6	*
7	* Interactivity improvements by Mike Galbraith
8	* (C) 2007 Mike Galbraith <efault@gmx.de>
9	*
10	* Various enhancements by Dmitry Adamushko.
11	* (C) 2007 Dmitry Adamushko <dmitry.adamushko@gmail.com>
12	*
13	* Group scheduling enhancements by Srivatsa Vaddagiri
14	* Copyright IBM Corporation, 2007
15	* Author: Srivatsa Vaddagiri <vatsa@linux.vnet.ibm.com>
16	*
17	* Scaled math optimizations by Thomas Gleixner
18	* Copyright (C) 2007, Thomas Gleixner <tglx@linutronix.de>
19	*
20	* Adaptive scheduling granularity, math enhancements by Peter Zijlstra
21	* Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra
22	*/
23	#include <linux/energy_model.h>
24	#include <linux/mmap_lock.h>
25	#include <linux/hugetlb_inline.h>
26	#include <linux/jiffies.h>
27	#include <linux/mm_api.h>
28	#include <linux/highmem.h>
29	#include <linux/spinlock_api.h>
30	#include <linux/cpumask_api.h>
31	#include <linux/lockdep_api.h>
32	#include <linux/softirq.h>
33	#include <linux/refcount_api.h>
34	#include <linux/topology.h>
35	#include <linux/sched/clock.h>
36	#include <linux/sched/cond_resched.h>
37	#include <linux/sched/cputime.h>
38	#include <linux/sched/isolation.h>
39	#include <linux/sched/nohz.h>
40
41	#include <linux/cpuidle.h>
42	#include <linux/interrupt.h>
43	#include <linux/memory-tiers.h>
44	#include <linux/mempolicy.h>
45	#include <linux/mutex_api.h>
46	#include <linux/profile.h>
47	#include <linux/psi.h>
48	#include <linux/ratelimit.h>
49	#include <linux/task_work.h>
50	#include <linux/rbtree_augmented.h>
51
52	#include <asm/switch_to.h>
53
54	#include "sched.h"
55	#include "stats.h"
56	#include "autogroup.h"
57
58	/*
59	* The initial- and re-scaling of tunables is configurable
60	*
61	* Options are:
62	*
63	* SCHED_TUNABLESCALING_NONE - unscaled, always *1
64	* SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus)
65	* SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus
66	*
67	* (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus))
68	*/
69	unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;
70
71	/*
72	* Minimal preemption granularity for CPU-bound tasks:
73	*
74	* (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
75	*/
76	unsigned int sysctl_sched_base_slice = 750000ULL;
77	static unsigned int normalized_sysctl_sched_base_slice = 750000ULL;
78
79	const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
80
81	int sched_thermal_decay_shift;
82	static int __init setup_sched_thermal_decay_shift(char *str)
83	{
84	int _shift = 0;
85
86	if (kstrtoint(s: str, base: 0, res: &_shift))
87	pr_warn("Unable to set scheduler thermal pressure decay shift parameter\n");
88
89	sched_thermal_decay_shift = clamp(_shift, 0, 10);
90	return 1;
91	}
92	__setup("sched_thermal_decay_shift=", setup_sched_thermal_decay_shift);
93
94	#ifdef CONFIG_SMP
95	/*
96	* For asym packing, by default the lower numbered CPU has higher priority.
97	*/
98	int __weak arch_asym_cpu_priority(int cpu)
99	{
100	return -cpu;
101	}
102
103	/*
104	* The margin used when comparing utilization with CPU capacity.
105	*
106	* (default: ~20%)
107	*/
108	#define fits_capacity(cap, max) ((cap) * 1280 < (max) * 1024)
109
110	/*
111	* The margin used when comparing CPU capacities.
112	* is 'cap1' noticeably greater than 'cap2'
113	*
114	* (default: ~5%)
115	*/
116	#define capacity_greater(cap1, cap2) ((cap1) * 1024 > (cap2) * 1078)
117	#endif
118
119	#ifdef CONFIG_CFS_BANDWIDTH
120	/*
121	* Amount of runtime to allocate from global (tg) to local (per-cfs_rq) pool
122	* each time a cfs_rq requests quota.
123	*
124	* Note: in the case that the slice exceeds the runtime remaining (either due
125	* to consumption or the quota being specified to be smaller than the slice)
126	* we will always only issue the remaining available time.
127	*
128	* (default: 5 msec, units: microseconds)
129	*/
130	static unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
131	#endif
132
133	#ifdef CONFIG_NUMA_BALANCING
134	/* Restrict the NUMA promotion throughput (MB/s) for each target node. */
135	static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536;
136	#endif
137
138	#ifdef CONFIG_SYSCTL
139	static struct ctl_table sched_fair_sysctls[] = {
140	#ifdef CONFIG_CFS_BANDWIDTH
141	{
142	.procname = "sched_cfs_bandwidth_slice_us",
143	.data = &sysctl_sched_cfs_bandwidth_slice,
144	.maxlen = sizeof(unsigned int),
145	.mode = 0644,
146	.proc_handler = proc_dointvec_minmax,
147	.extra1 = SYSCTL_ONE,
148	},
149	#endif
150	#ifdef CONFIG_NUMA_BALANCING
151	{
152	.procname = "numa_balancing_promote_rate_limit_MBps",
153	.data = &sysctl_numa_balancing_promote_rate_limit,
154	.maxlen = sizeof(unsigned int),
155	.mode = 0644,
156	.proc_handler = proc_dointvec_minmax,
157	.extra1 = SYSCTL_ZERO,
158	},
159	#endif /* CONFIG_NUMA_BALANCING */
160	{}
161	};
162
163	static int __init sched_fair_sysctl_init(void)
164	{
165	register_sysctl_init("kernel", sched_fair_sysctls);
166	return 0;
167	}
168	late_initcall(sched_fair_sysctl_init);
169	#endif
170
171	static inline void update_load_add(struct load_weight *lw, unsigned long inc)
172	{
173	lw->weight += inc;
174	lw->inv_weight = 0;
175	}
176
177	static inline void update_load_sub(struct load_weight *lw, unsigned long dec)
178	{
179	lw->weight -= dec;
180	lw->inv_weight = 0;
181	}
182
183	static inline void update_load_set(struct load_weight *lw, unsigned long w)
184	{
185	lw->weight = w;
186	lw->inv_weight = 0;
187	}
188
189	/*
190	* Increase the granularity value when there are more CPUs,
191	* because with more CPUs the 'effective latency' as visible
192	* to users decreases. But the relationship is not linear,
193	* so pick a second-best guess by going with the log2 of the
194	* number of CPUs.
195	*
196	* This idea comes from the SD scheduler of Con Kolivas:
197	*/
198	static unsigned int get_update_sysctl_factor(void)
199	{
200	unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8);
201	unsigned int factor;
202
203	switch (sysctl_sched_tunable_scaling) {
204	case SCHED_TUNABLESCALING_NONE:
205	factor = 1;
206	break;
207	case SCHED_TUNABLESCALING_LINEAR:
208	factor = cpus;
209	break;
210	case SCHED_TUNABLESCALING_LOG:
211	default:
212	factor = 1 + ilog2(cpus);
213	break;
214	}
215
216	return factor;
217	}
218
219	static void update_sysctl(void)
220	{
221	unsigned int factor = get_update_sysctl_factor();
222
223	#define SET_SYSCTL(name) \
224	(sysctl_##name = (factor) * normalized_sysctl_##name)
225	SET_SYSCTL(sched_base_slice);
226	#undef SET_SYSCTL
227	}
228
229	void __init sched_init_granularity(void)
230	{
231	update_sysctl();
232	}
233
234	#define WMULT_CONST (~0U)
235	#define WMULT_SHIFT 32
236
237	static void __update_inv_weight(struct load_weight *lw)
238	{
239	unsigned long w;
240
241	if (likely(lw->inv_weight))
242	return;
243
244	w = scale_load_down(lw->weight);
245
246	if (BITS_PER_LONG > 32 && unlikely(w >= WMULT_CONST))
247	lw->inv_weight = 1;
248	else if (unlikely(!w))
249	lw->inv_weight = WMULT_CONST;
250	else
251	lw->inv_weight = WMULT_CONST / w;
252	}
253
254	/*
255	* delta_exec * weight / lw.weight
256	* OR
257	* (delta_exec * (weight * lw->inv_weight)) >> WMULT_SHIFT
258	*
259	* Either weight := NICE_0_LOAD and lw \e sched_prio_to_wmult[], in which case
260	* we're guaranteed shift stays positive because inv_weight is guaranteed to
261	* fit 32 bits, and NICE_0_LOAD gives another 10 bits; therefore shift >= 22.
262	*
263	* Or, weight =< lw.weight (because lw.weight is the runqueue weight), thus
264	* weight/lw.weight <= 1, and therefore our shift will also be positive.
265	*/
266	static u64 __calc_delta(u64 delta_exec, unsigned long weight, struct load_weight *lw)
267	{
268	u64 fact = scale_load_down(weight);
269	u32 fact_hi = (u32)(fact >> 32);
270	int shift = WMULT_SHIFT;
271	int fs;
272
273	__update_inv_weight(lw);
274
275	if (unlikely(fact_hi)) {
276	fs = fls(x: fact_hi);
277	shift -= fs;
278	fact >>= fs;
279	}
280
281	fact = mul_u32_u32(a: fact, b: lw->inv_weight);
282
283	fact_hi = (u32)(fact >> 32);
284	if (fact_hi) {
285	fs = fls(x: fact_hi);
286	shift -= fs;
287	fact >>= fs;
288	}
289
290	return mul_u64_u32_shr(a: delta_exec, mul: fact, shift);
291	}
292
293	/*
294	* delta /= w
295	*/
296	static inline u64 calc_delta_fair(u64 delta, struct sched_entity *se)
297	{
298	if (unlikely(se->load.weight != NICE_0_LOAD))
299	delta = __calc_delta(delta_exec: delta, NICE_0_LOAD, lw: &se->load);
300
301	return delta;
302	}
303
304	const struct sched_class fair_sched_class;
305
306	/**************************************************************
307	* CFS operations on generic schedulable entities:
308	*/
309
310	#ifdef CONFIG_FAIR_GROUP_SCHED
311
312	/* Walk up scheduling entities hierarchy */
313	#define for_each_sched_entity(se) \
314	for (; se; se = se->parent)
315
316	static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
317	{
318	struct rq *rq = rq_of(cfs_rq);
319	int cpu = cpu_of(rq);
320
321	if (cfs_rq->on_list)
322	return rq->tmp_alone_branch == &rq->leaf_cfs_rq_list;
323
324	cfs_rq->on_list = 1;
325
326	/*
327	* Ensure we either appear before our parent (if already
328	* enqueued) or force our parent to appear after us when it is
329	* enqueued. The fact that we always enqueue bottom-up
330	* reduces this to two cases and a special case for the root
331	* cfs_rq. Furthermore, it also means that we will always reset
332	* tmp_alone_branch either when the branch is connected
333	* to a tree or when we reach the top of the tree
334	*/
335	if (cfs_rq->tg->parent &&
336	cfs_rq->tg->parent->cfs_rq[cpu]->on_list) {
337	/*
338	* If parent is already on the list, we add the child
339	* just before. Thanks to circular linked property of
340	* the list, this means to put the child at the tail
341	* of the list that starts by parent.
342	*/
343	list_add_tail_rcu(new: &cfs_rq->leaf_cfs_rq_list,
344	head: &(cfs_rq->tg->parent->cfs_rq[cpu]->leaf_cfs_rq_list));
345	/*
346	* The branch is now connected to its tree so we can
347	* reset tmp_alone_branch to the beginning of the
348	* list.
349	*/
350	rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
351	return true;
352	}
353
354	if (!cfs_rq->tg->parent) {
355	/*
356	* cfs rq without parent should be put
357	* at the tail of the list.
358	*/
359	list_add_tail_rcu(new: &cfs_rq->leaf_cfs_rq_list,
360	head: &rq->leaf_cfs_rq_list);
361	/*
362	* We have reach the top of a tree so we can reset
363	* tmp_alone_branch to the beginning of the list.
364	*/
365	rq->tmp_alone_branch = &rq->leaf_cfs_rq_list;
366	return true;
367	}
368
369	/*
370	* The parent has not already been added so we want to
371	* make sure that it will be put after us.
372	* tmp_alone_branch points to the begin of the branch
373	* where we will add parent.
374	*/
375	list_add_rcu(new: &cfs_rq->leaf_cfs_rq_list, head: rq->tmp_alone_branch);
376	/*
377	* update tmp_alone_branch to points to the new begin
378	* of the branch
379	*/
380	rq->tmp_alone_branch = &cfs_rq->leaf_cfs_rq_list;
381	return false;
382	}
383
384	static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
385	{
386	if (cfs_rq->on_list) {
387	struct rq *rq = rq_of(cfs_rq);
388
389	/*
390	* With cfs_rq being unthrottled/throttled during an enqueue,
391	* it can happen the tmp_alone_branch points the a leaf that
392	* we finally want to del. In this case, tmp_alone_branch moves
393	* to the prev element but it will point to rq->leaf_cfs_rq_list
394	* at the end of the enqueue.
395	*/
396	if (rq->tmp_alone_branch == &cfs_rq->leaf_cfs_rq_list)
397	rq->tmp_alone_branch = cfs_rq->leaf_cfs_rq_list.prev;
398
399	list_del_rcu(entry: &cfs_rq->leaf_cfs_rq_list);
400	cfs_rq->on_list = 0;
401	}
402	}
403
404	static inline void assert_list_leaf_cfs_rq(struct rq *rq)
405	{
406	SCHED_WARN_ON(rq->tmp_alone_branch != &rq->leaf_cfs_rq_list);
407	}
408
409	/* Iterate thr' all leaf cfs_rq's on a runqueue */
410	#define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \
411	list_for_each_entry_safe(cfs_rq, pos, &rq->leaf_cfs_rq_list, \
412	leaf_cfs_rq_list)
413
414	/* Do the two (enqueued) entities belong to the same group ? */
415	static inline struct cfs_rq *
416	is_same_group(struct sched_entity *se, struct sched_entity *pse)
417	{
418	if (se->cfs_rq == pse->cfs_rq)
419	return se->cfs_rq;
420
421	return NULL;
422	}
423
424	static inline struct sched_entity *parent_entity(const struct sched_entity *se)
425	{
426	return se->parent;
427	}
428
429	static void
430	find_matching_se(struct sched_entity **se, struct sched_entity **pse)
431	{
432	int se_depth, pse_depth;
433
434	/*
435	* preemption test can be made between sibling entities who are in the
436	* same cfs_rq i.e who have a common parent. Walk up the hierarchy of
437	* both tasks until we find their ancestors who are siblings of common
438	* parent.
439	*/
440
441	/* First walk up until both entities are at same depth */
442	se_depth = (*se)->depth;
443	pse_depth = (*pse)->depth;
444
445	while (se_depth > pse_depth) {
446	se_depth--;
447	*se = parent_entity(se: *se);
448	}
449
450	while (pse_depth > se_depth) {
451	pse_depth--;
452	*pse = parent_entity(se: *pse);
453	}
454
455	while (!is_same_group(se: *se, pse: *pse)) {
456	*se = parent_entity(se: *se);
457	*pse = parent_entity(se: *pse);
458	}
459	}
460
461	static int tg_is_idle(struct task_group *tg)
462	{
463	return tg->idle > 0;
464	}
465
466	static int cfs_rq_is_idle(struct cfs_rq *cfs_rq)
467	{
468	return cfs_rq->idle > 0;
469	}
470
471	static int se_is_idle(struct sched_entity *se)
472	{
473	if (entity_is_task(se))
474	return task_has_idle_policy(p: task_of(se));
475	return cfs_rq_is_idle(cfs_rq: group_cfs_rq(grp: se));
476	}
477
478	#else /* !CONFIG_FAIR_GROUP_SCHED */
479
480	#define for_each_sched_entity(se) \
481	for (; se; se = NULL)
482
483	static inline bool list_add_leaf_cfs_rq(struct cfs_rq *cfs_rq)
484	{
485	return true;
486	}
487
488	static inline void list_del_leaf_cfs_rq(struct cfs_rq *cfs_rq)
489	{
490	}
491
492	static inline void assert_list_leaf_cfs_rq(struct rq *rq)
493	{
494	}
495
496	#define for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) \
497	for (cfs_rq = &rq->cfs, pos = NULL; cfs_rq; cfs_rq = pos)
498
499	static inline struct sched_entity *parent_entity(struct sched_entity *se)
500	{
501	return NULL;
502	}
503
504	static inline void
505	find_matching_se(struct sched_entity **se, struct sched_entity **pse)
506	{
507	}
508
509	static inline int tg_is_idle(struct task_group *tg)
510	{
511	return 0;
512	}
513
514	static int cfs_rq_is_idle(struct cfs_rq *cfs_rq)
515	{
516	return 0;
517	}
518
519	static int se_is_idle(struct sched_entity *se)
520	{
521	return 0;
522	}
523
524	#endif /* CONFIG_FAIR_GROUP_SCHED */
525
526	static __always_inline
527	void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec);
528
529	/**************************************************************
530	* Scheduling class tree data structure manipulation methods:
531	*/
532
533	static inline u64 max_vruntime(u64 max_vruntime, u64 vruntime)
534	{
535	s64 delta = (s64)(vruntime - max_vruntime);
536	if (delta > 0)
537	max_vruntime = vruntime;
538
539	return max_vruntime;
540	}
541
542	static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime)
543	{
544	s64 delta = (s64)(vruntime - min_vruntime);
545	if (delta < 0)
546	min_vruntime = vruntime;
547
548	return min_vruntime;
549	}
550
551	static inline bool entity_before(const struct sched_entity *a,
552	const struct sched_entity *b)
553	{
554	/*
555	* Tiebreak on vruntime seems unnecessary since it can
556	* hardly happen.
557	*/
558	return (s64)(a->deadline - b->deadline) < 0;
559	}
560
561	static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se)
562	{
563	return (s64)(se->vruntime - cfs_rq->min_vruntime);
564	}
565
566	#define __node_2_se(node) \
567	rb_entry((node), struct sched_entity, run_node)
568
569	/*
570	* Compute virtual time from the per-task service numbers:
571	*
572	* Fair schedulers conserve lag:
573	*
574	* \Sum lag_i = 0
575	*
576	* Where lag_i is given by:
577	*
578	* lag_i = S - s_i = w_i * (V - v_i)
579	*
580	* Where S is the ideal service time and V is it's virtual time counterpart.
581	* Therefore:
582	*
583	* \Sum lag_i = 0
584	* \Sum w_i * (V - v_i) = 0
585	* \Sum w_i * V - w_i * v_i = 0
586	*
587	* From which we can solve an expression for V in v_i (which we have in
588	* se->vruntime):
589	*
590	* \Sum v_i * w_i \Sum v_i * w_i
591	* V = -------------- = --------------
592	* \Sum w_i W
593	*
594	* Specifically, this is the weighted average of all entity virtual runtimes.
595	*
596	* [[ NOTE: this is only equal to the ideal scheduler under the condition
597	* that join/leave operations happen at lag_i = 0, otherwise the
598	* virtual time has non-continguous motion equivalent to:
599	*
600	* V +-= lag_i / W
601	*
602	* Also see the comment in place_entity() that deals with this. ]]
603	*
604	* However, since v_i is u64, and the multiplcation could easily overflow
605	* transform it into a relative form that uses smaller quantities:
606	*
607	* Substitute: v_i == (v_i - v0) + v0
608	*
609	* \Sum ((v_i - v0) + v0) * w_i \Sum (v_i - v0) * w_i
610	* V = ---------------------------- = --------------------- + v0
611	* W W
612	*
613	* Which we track using:
614	*
615	* v0 := cfs_rq->min_vruntime
616	* \Sum (v_i - v0) * w_i := cfs_rq->avg_vruntime
617	* \Sum w_i := cfs_rq->avg_load
618	*
619	* Since min_vruntime is a monotonic increasing variable that closely tracks
620	* the per-task service, these deltas: (v_i - v), will be in the order of the
621	* maximal (virtual) lag induced in the system due to quantisation.
622	*
623	* Also, we use scale_load_down() to reduce the size.
624	*
625	* As measured, the max (key * weight) value was ~44 bits for a kernel build.
626	*/
627	static void
628	avg_vruntime_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
629	{
630	unsigned long weight = scale_load_down(se->load.weight);
631	s64 key = entity_key(cfs_rq, se);
632
633	cfs_rq->avg_vruntime += key * weight;
634	cfs_rq->avg_load += weight;
635	}
636
637	static void
638	avg_vruntime_sub(struct cfs_rq *cfs_rq, struct sched_entity *se)
639	{
640	unsigned long weight = scale_load_down(se->load.weight);
641	s64 key = entity_key(cfs_rq, se);
642
643	cfs_rq->avg_vruntime -= key * weight;
644	cfs_rq->avg_load -= weight;
645	}
646
647	static inline
648	void avg_vruntime_update(struct cfs_rq *cfs_rq, s64 delta)
649	{
650	/*
651	* v' = v + d ==> avg_vruntime' = avg_runtime - d*avg_load
652	*/
653	cfs_rq->avg_vruntime -= cfs_rq->avg_load * delta;
654	}
655
656	/*
657	* Specifically: avg_runtime() + 0 must result in entity_eligible() := true
658	* For this to be so, the result of this function must have a left bias.
659	*/
660	u64 avg_vruntime(struct cfs_rq *cfs_rq)
661	{
662	struct sched_entity *curr = cfs_rq->curr;
663	s64 avg = cfs_rq->avg_vruntime;
664	long load = cfs_rq->avg_load;
665
666	if (curr && curr->on_rq) {
667	unsigned long weight = scale_load_down(curr->load.weight);
668
669	avg += entity_key(cfs_rq, se: curr) * weight;
670	load += weight;
671	}
672
673	if (load) {
674	/* sign flips effective floor / ceil */
675	if (avg < 0)
676	avg -= (load - 1);
677	avg = div_s64(dividend: avg, divisor: load);
678	}
679
680	return cfs_rq->min_vruntime + avg;
681	}
682
683	/*
684	* lag_i = S - s_i = w_i * (V - v_i)
685	*
686	* However, since V is approximated by the weighted average of all entities it
687	* is possible -- by addition/removal/reweight to the tree -- to move V around
688	* and end up with a larger lag than we started with.
689	*
690	* Limit this to either double the slice length with a minimum of TICK_NSEC
691	* since that is the timing granularity.
692	*
693	* EEVDF gives the following limit for a steady state system:
694	*
695	* -r_max < lag < max(r_max, q)
696	*
697	* XXX could add max_slice to the augmented data to track this.
698	*/
699	static void update_entity_lag(struct cfs_rq *cfs_rq, struct sched_entity *se)
700	{
701	s64 lag, limit;
702
703	SCHED_WARN_ON(!se->on_rq);
704	lag = avg_vruntime(cfs_rq) - se->vruntime;
705
706	limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se);
707	se->vlag = clamp(lag, -limit, limit);
708	}
709
710	/*
711	* Entity is eligible once it received less service than it ought to have,
712	* eg. lag >= 0.
713	*
714	* lag_i = S - s_i = w_i*(V - v_i)
715	*
716	* lag_i >= 0 -> V >= v_i
717	*
718	* \Sum (v_i - v)*w_i
719	* V = ------------------ + v
720	* \Sum w_i
721	*
722	* lag_i >= 0 -> \Sum (v_i - v)*w_i >= (v_i - v)*(\Sum w_i)
723	*
724	* Note: using 'avg_vruntime() > se->vruntime' is inacurate due
725	* to the loss in precision caused by the division.
726	*/
727	static int vruntime_eligible(struct cfs_rq *cfs_rq, u64 vruntime)
728	{
729	struct sched_entity *curr = cfs_rq->curr;
730	s64 avg = cfs_rq->avg_vruntime;
731	long load = cfs_rq->avg_load;
732
733	if (curr && curr->on_rq) {
734	unsigned long weight = scale_load_down(curr->load.weight);
735
736	avg += entity_key(cfs_rq, se: curr) * weight;
737	load += weight;
738	}
739
740	return avg >= (s64)(vruntime - cfs_rq->min_vruntime) * load;
741	}
742
743	int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se)
744	{
745	return vruntime_eligible(cfs_rq, vruntime: se->vruntime);
746	}
747
748	static u64 __update_min_vruntime(struct cfs_rq *cfs_rq, u64 vruntime)
749	{
750	u64 min_vruntime = cfs_rq->min_vruntime;
751	/*
752	* open coded max_vruntime() to allow updating avg_vruntime
753	*/
754	s64 delta = (s64)(vruntime - min_vruntime);
755	if (delta > 0) {
756	avg_vruntime_update(cfs_rq, delta);
757	min_vruntime = vruntime;
758	}
759	return min_vruntime;
760	}
761
762	static void update_min_vruntime(struct cfs_rq *cfs_rq)
763	{
764	struct sched_entity *se = __pick_root_entity(cfs_rq);
765	struct sched_entity *curr = cfs_rq->curr;
766	u64 vruntime = cfs_rq->min_vruntime;
767
768	if (curr) {
769	if (curr->on_rq)
770	vruntime = curr->vruntime;
771	else
772	curr = NULL;
773	}
774
775	if (se) {
776	if (!curr)
777	vruntime = se->min_vruntime;
778	else
779	vruntime = min_vruntime(min_vruntime: vruntime, vruntime: se->min_vruntime);
780	}
781
782	/* ensure we never gain time by being placed backwards. */
783	u64_u32_store(cfs_rq->min_vruntime,
784	__update_min_vruntime(cfs_rq, vruntime));
785	}
786
787	static inline bool __entity_less(struct rb_node *a, const struct rb_node *b)
788	{
789	return entity_before(__node_2_se(a), __node_2_se(b));
790	}
791
792	#define vruntime_gt(field, lse, rse) ({ (s64)((lse)->field - (rse)->field) > 0; })
793
794	static inline void __min_vruntime_update(struct sched_entity *se, struct rb_node *node)
795	{
796	if (node) {
797	struct sched_entity *rse = __node_2_se(node);
798	if (vruntime_gt(min_vruntime, se, rse))
799	se->min_vruntime = rse->min_vruntime;
800	}
801	}
802
803	/*
804	* se->min_vruntime = min(se->vruntime, {left,right}->min_vruntime)
805	*/
806	static inline bool min_vruntime_update(struct sched_entity *se, bool exit)
807	{
808	u64 old_min_vruntime = se->min_vruntime;
809	struct rb_node *node = &se->run_node;
810
811	se->min_vruntime = se->vruntime;
812	__min_vruntime_update(se, node: node->rb_right);
813	__min_vruntime_update(se, node: node->rb_left);
814
815	return se->min_vruntime == old_min_vruntime;
816	}
817
818	RB_DECLARE_CALLBACKS(static, min_vruntime_cb, struct sched_entity,
819	run_node, min_vruntime, min_vruntime_update);
820
821	/*
822	* Enqueue an entity into the rb-tree:
823	*/
824	static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
825	{
826	avg_vruntime_add(cfs_rq, se);
827	se->min_vruntime = se->vruntime;
828	rb_add_augmented_cached(node: &se->run_node, tree: &cfs_rq->tasks_timeline,
829	less: __entity_less, augment: &min_vruntime_cb);
830	}
831
832	static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
833	{
834	rb_erase_augmented_cached(node: &se->run_node, root: &cfs_rq->tasks_timeline,
835	augment: &min_vruntime_cb);
836	avg_vruntime_sub(cfs_rq, se);
837	}
838
839	struct sched_entity *__pick_root_entity(struct cfs_rq *cfs_rq)
840	{
841	struct rb_node *root = cfs_rq->tasks_timeline.rb_root.rb_node;
842
843	if (!root)
844	return NULL;
845
846	return __node_2_se(root);
847	}
848
849	struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq)
850	{
851	struct rb_node *left = rb_first_cached(&cfs_rq->tasks_timeline);
852
853	if (!left)
854	return NULL;
855
856	return __node_2_se(left);
857	}
858
859	/*
860	* Earliest Eligible Virtual Deadline First
861	*
862	* In order to provide latency guarantees for different request sizes
863	* EEVDF selects the best runnable task from two criteria:
864	*
865	* 1) the task must be eligible (must be owed service)
866	*
867	* 2) from those tasks that meet 1), we select the one
868	* with the earliest virtual deadline.
869	*
870	* We can do this in O(log n) time due to an augmented RB-tree. The
871	* tree keeps the entries sorted on deadline, but also functions as a
872	* heap based on the vruntime by keeping:
873	*
874	* se->min_vruntime = min(se->vruntime, se->{left,right}->min_vruntime)
875	*
876	* Which allows tree pruning through eligibility.
877	*/
878	static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq)
879	{
880	struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node;
881	struct sched_entity *se = __pick_first_entity(cfs_rq);
882	struct sched_entity *curr = cfs_rq->curr;
883	struct sched_entity *best = NULL;
884
885	/*
886	* We can safely skip eligibility check if there is only one entity
887	* in this cfs_rq, saving some cycles.
888	*/
889	if (cfs_rq->nr_running == 1)
890	return curr && curr->on_rq ? curr : se;
891
892	if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, se: curr)))
893	curr = NULL;
894
895	/*
896	* Once selected, run a task until it either becomes non-eligible or
897	* until it gets a new slice. See the HACK in set_next_entity().
898	*/
899	if (sched_feat(RUN_TO_PARITY) && curr && curr->vlag == curr->deadline)
900	return curr;
901
902	/* Pick the leftmost entity if it's eligible */
903	if (se && entity_eligible(cfs_rq, se)) {
904	best = se;
905	goto found;
906	}
907
908	/* Heap search for the EEVD entity */
909	while (node) {
910	struct rb_node *left = node->rb_left;
911
912	/*
913	* Eligible entities in left subtree are always better
914	* choices, since they have earlier deadlines.
915	*/
916	if (left && vruntime_eligible(cfs_rq,
917	__node_2_se(left)->min_vruntime)) {
918	node = left;
919	continue;
920	}
921
922	se = __node_2_se(node);
923
924	/*
925	* The left subtree either is empty or has no eligible
926	* entity, so check the current node since it is the one
927	* with earliest deadline that might be eligible.
928	*/
929	if (entity_eligible(cfs_rq, se)) {
930	best = se;
931	break;
932	}
933
934	node = node->rb_right;
935	}
936	found:
937	if (!best || (curr && entity_before(a: curr, b: best)))
938	best = curr;
939
940	return best;
941	}
942
943	#ifdef CONFIG_SCHED_DEBUG
944	struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq)
945	{
946	struct rb_node *last = rb_last(&cfs_rq->tasks_timeline.rb_root);
947
948	if (!last)
949	return NULL;
950
951	return __node_2_se(last);
952	}
953
954	/**************************************************************
955	* Scheduling class statistics methods:
956	*/
957	#ifdef CONFIG_SMP
958	int sched_update_scaling(void)
959	{
960	unsigned int factor = get_update_sysctl_factor();
961
962	#define WRT_SYSCTL(name) \
963	(normalized_sysctl_##name = sysctl_##name / (factor))
964	WRT_SYSCTL(sched_base_slice);
965	#undef WRT_SYSCTL
966
967	return 0;
968	}
969	#endif
970	#endif
971
972	static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se);
973
974	/*
975	* XXX: strictly: vd_i += N*r_i/w_i such that: vd_i > ve_i
976	* this is probably good enough.
977	*/
978	static void update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se)
979	{
980	if ((s64)(se->vruntime - se->deadline) < 0)
981	return;
982
983	/*
984	* For EEVDF the virtual time slope is determined by w_i (iow.
985	* nice) while the request time r_i is determined by
986	* sysctl_sched_base_slice.
987	*/
988	se->slice = sysctl_sched_base_slice;
989
990	/*
991	* EEVDF: vd_i = ve_i + r_i / w_i
992	*/
993	se->deadline = se->vruntime + calc_delta_fair(delta: se->slice, se);
994
995	/*
996	* The task has consumed its request, reschedule.
997	*/
998	if (cfs_rq->nr_running > 1) {
999	resched_curr(rq: rq_of(cfs_rq));
1000	clear_buddies(cfs_rq, se);
1001	}
1002	}
1003
1004	#include "pelt.h"
1005	#ifdef CONFIG_SMP
1006
1007	static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu);
1008	static unsigned long task_h_load(struct task_struct *p);
1009	static unsigned long capacity_of(int cpu);
1010
1011	/* Give new sched_entity start runnable values to heavy its load in infant time */
1012	void init_entity_runnable_average(struct sched_entity *se)
1013	{
1014	struct sched_avg *sa = &se->avg;
1015
1016	memset(sa, 0, sizeof(*sa));
1017
1018	/*
1019	* Tasks are initialized with full load to be seen as heavy tasks until
1020	* they get a chance to stabilize to their real load level.
1021	* Group entities are initialized with zero load to reflect the fact that
1022	* nothing has been attached to the task group yet.
1023	*/
1024	if (entity_is_task(se))
1025	sa->load_avg = scale_load_down(se->load.weight);
1026
1027	/* when this task enqueue'ed, it will contribute to its cfs_rq's load_avg */
1028	}
1029
1030	/*
1031	* With new tasks being created, their initial util_avgs are extrapolated
1032	* based on the cfs_rq's current util_avg:
1033	*
1034	* util_avg = cfs_rq->util_avg / (cfs_rq->load_avg + 1) * se.load.weight
1035	*
1036	* However, in many cases, the above util_avg does not give a desired
1037	* value. Moreover, the sum of the util_avgs may be divergent, such
1038	* as when the series is a harmonic series.
1039	*
1040	* To solve this problem, we also cap the util_avg of successive tasks to
1041	* only 1/2 of the left utilization budget:
1042	*
1043	* util_avg_cap = (cpu_scale - cfs_rq->avg.util_avg) / 2^n
1044	*
1045	* where n denotes the nth task and cpu_scale the CPU capacity.
1046	*
1047	* For example, for a CPU with 1024 of capacity, a simplest series from
1048	* the beginning would be like:
1049	*
1050	* task util_avg: 512, 256, 128, 64, 32, 16, 8, ...
1051	* cfs_rq util_avg: 512, 768, 896, 960, 992, 1008, 1016, ...
1052	*
1053	* Finally, that extrapolated util_avg is clamped to the cap (util_avg_cap)
1054	* if util_avg > util_avg_cap.
1055	*/
1056	void post_init_entity_util_avg(struct task_struct *p)
1057	{
1058	struct sched_entity *se = &p->se;
1059	struct cfs_rq *cfs_rq = cfs_rq_of(se);
1060	struct sched_avg *sa = &se->avg;
1061	long cpu_scale = arch_scale_cpu_capacity(cpu: cpu_of(rq: rq_of(cfs_rq)));
1062	long cap = (long)(cpu_scale - cfs_rq->avg.util_avg) / 2;
1063
1064	if (p->sched_class != &fair_sched_class) {
1065	/*
1066	* For !fair tasks do:
1067	*
1068	update_cfs_rq_load_avg(now, cfs_rq);
1069	attach_entity_load_avg(cfs_rq, se);
1070	switched_from_fair(rq, p);
1071	*
1072	* such that the next switched_to_fair() has the
1073	* expected state.
1074	*/
1075	se->avg.last_update_time = cfs_rq_clock_pelt(cfs_rq);
1076	return;
1077	}
1078
1079	if (cap > 0) {
1080	if (cfs_rq->avg.util_avg != 0) {
1081	sa->util_avg = cfs_rq->avg.util_avg * se->load.weight;
1082	sa->util_avg /= (cfs_rq->avg.load_avg + 1);
1083
1084	if (sa->util_avg > cap)
1085	sa->util_avg = cap;
1086	} else {
1087	sa->util_avg = cap;
1088	}
1089	}
1090
1091	sa->runnable_avg = sa->util_avg;
1092	}
1093
1094	#else /* !CONFIG_SMP */
1095	void init_entity_runnable_average(struct sched_entity *se)
1096	{
1097	}
1098	void post_init_entity_util_avg(struct task_struct *p)
1099	{
1100	}
1101	static void update_tg_load_avg(struct cfs_rq *cfs_rq)
1102	{
1103	}
1104	#endif /* CONFIG_SMP */
1105
1106	static s64 update_curr_se(struct rq *rq, struct sched_entity *curr)
1107	{
1108	u64 now = rq_clock_task(rq);
1109	s64 delta_exec;
1110
1111	delta_exec = now - curr->exec_start;
1112	if (unlikely(delta_exec <= 0))
1113	return delta_exec;
1114
1115	curr->exec_start = now;
1116	curr->sum_exec_runtime += delta_exec;
1117
1118	if (schedstat_enabled()) {
1119	struct sched_statistics *stats;
1120
1121	stats = __schedstats_from_se(se: curr);
1122	__schedstat_set(stats->exec_max,
1123	max(delta_exec, stats->exec_max));
1124	}
1125
1126	return delta_exec;
1127	}
1128
1129	static inline void update_curr_task(struct task_struct *p, s64 delta_exec)
1130	{
1131	trace_sched_stat_runtime(tsk: p, runtime: delta_exec);
1132	account_group_exec_runtime(tsk: p, ns: delta_exec);
1133	cgroup_account_cputime(task: p, delta_exec);
1134	if (p->dl_server)
1135	dl_server_update(dl_se: p->dl_server, delta_exec);
1136	}
1137
1138	/*
1139	* Used by other classes to account runtime.
1140	*/
1141	s64 update_curr_common(struct rq *rq)
1142	{
1143	struct task_struct *curr = rq->curr;
1144	s64 delta_exec;
1145
1146	delta_exec = update_curr_se(rq, curr: &curr->se);
1147	if (likely(delta_exec > 0))
1148	update_curr_task(p: curr, delta_exec);
1149
1150	return delta_exec;
1151	}
1152
1153	/*
1154	* Update the current task's runtime statistics.
1155	*/
1156	static void update_curr(struct cfs_rq *cfs_rq)
1157	{
1158	struct sched_entity *curr = cfs_rq->curr;
1159	s64 delta_exec;
1160
1161	if (unlikely(!curr))
1162	return;
1163
1164	delta_exec = update_curr_se(rq: rq_of(cfs_rq), curr);
1165	if (unlikely(delta_exec <= 0))
1166	return;
1167
1168	curr->vruntime += calc_delta_fair(delta: delta_exec, se: curr);
1169	update_deadline(cfs_rq, se: curr);
1170	update_min_vruntime(cfs_rq);
1171
1172	if (entity_is_task(curr))
1173	update_curr_task(p: task_of(se: curr), delta_exec);
1174
1175	account_cfs_rq_runtime(cfs_rq, delta_exec);
1176	}
1177
1178	static void update_curr_fair(struct rq *rq)
1179	{
1180	update_curr(cfs_rq: cfs_rq_of(se: &rq->curr->se));
1181	}
1182
1183	static inline void
1184	update_stats_wait_start_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
1185	{
1186	struct sched_statistics *stats;
1187	struct task_struct *p = NULL;
1188
1189	if (!schedstat_enabled())
1190	return;
1191
1192	stats = __schedstats_from_se(se);
1193
1194	if (entity_is_task(se))
1195	p = task_of(se);
1196
1197	__update_stats_wait_start(rq: rq_of(cfs_rq), p, stats);
1198	}
1199
1200	static inline void
1201	update_stats_wait_end_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
1202	{
1203	struct sched_statistics *stats;
1204	struct task_struct *p = NULL;
1205
1206	if (!schedstat_enabled())
1207	return;
1208
1209	stats = __schedstats_from_se(se);
1210
1211	/*
1212	* When the sched_schedstat changes from 0 to 1, some sched se
1213	* maybe already in the runqueue, the se->statistics.wait_start
1214	* will be 0.So it will let the delta wrong. We need to avoid this
1215	* scenario.
1216	*/
1217	if (unlikely(!schedstat_val(stats->wait_start)))
1218	return;
1219
1220	if (entity_is_task(se))
1221	p = task_of(se);
1222
1223	__update_stats_wait_end(rq: rq_of(cfs_rq), p, stats);
1224	}
1225
1226	static inline void
1227	update_stats_enqueue_sleeper_fair(struct cfs_rq *cfs_rq, struct sched_entity *se)
1228	{
1229	struct sched_statistics *stats;
1230	struct task_struct *tsk = NULL;
1231
1232	if (!schedstat_enabled())
1233	return;
1234
1235	stats = __schedstats_from_se(se);
1236
1237	if (entity_is_task(se))
1238	tsk = task_of(se);
1239
1240	__update_stats_enqueue_sleeper(rq: rq_of(cfs_rq), p: tsk, stats);
1241	}
1242
1243	/*
1244	* Task is being enqueued - update stats:
1245	*/
1246	static inline void
1247	update_stats_enqueue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
1248	{
1249	if (!schedstat_enabled())
1250	return;
1251
1252	/*
1253	* Are we enqueueing a waiting task? (for current tasks
1254	* a dequeue/enqueue event is a NOP)
1255	*/
1256	if (se != cfs_rq->curr)
1257	update_stats_wait_start_fair(cfs_rq, se);
1258
1259	if (flags & ENQUEUE_WAKEUP)
1260	update_stats_enqueue_sleeper_fair(cfs_rq, se);
1261	}
1262
1263	static inline void
1264	update_stats_dequeue_fair(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
1265	{
1266
1267	if (!schedstat_enabled())
1268	return;
1269
1270	/*
1271	* Mark the end of the wait period if dequeueing a
1272	* waiting task:
1273	*/
1274	if (se != cfs_rq->curr)
1275	update_stats_wait_end_fair(cfs_rq, se);
1276
1277	if ((flags & DEQUEUE_SLEEP) && entity_is_task(se)) {
1278	struct task_struct *tsk = task_of(se);
1279	unsigned int state;
1280
1281	/* XXX racy against TTWU */
1282	state = READ_ONCE(tsk->__state);
1283	if (state & TASK_INTERRUPTIBLE)
1284	__schedstat_set(tsk->stats.sleep_start,
1285	rq_clock(rq_of(cfs_rq)));
1286	if (state & TASK_UNINTERRUPTIBLE)
1287	__schedstat_set(tsk->stats.block_start,
1288	rq_clock(rq_of(cfs_rq)));
1289	}
1290	}
1291
1292	/*
1293	* We are picking a new current task - update its stats:
1294	*/
1295	static inline void
1296	update_stats_curr_start(struct cfs_rq *cfs_rq, struct sched_entity *se)
1297	{
1298	/*
1299	* We are starting a new run period:
1300	*/
1301	se->exec_start = rq_clock_task(rq: rq_of(cfs_rq));
1302	}
1303
1304	/**************************************************
1305	* Scheduling class queueing methods:
1306	*/
1307
1308	static inline bool is_core_idle(int cpu)
1309	{
1310	#ifdef CONFIG_SCHED_SMT
1311	int sibling;
1312
1313	for_each_cpu(sibling, cpu_smt_mask(cpu)) {
1314	if (cpu == sibling)
1315	continue;
1316
1317	if (!idle_cpu(cpu: sibling))
1318	return false;
1319	}
1320	#endif
1321
1322	return true;
1323	}
1324
1325	#ifdef CONFIG_NUMA
1326	#define NUMA_IMBALANCE_MIN 2
1327
1328	static inline long
1329	adjust_numa_imbalance(int imbalance, int dst_running, int imb_numa_nr)
1330	{
1331	/*
1332	* Allow a NUMA imbalance if busy CPUs is less than the maximum
1333	* threshold. Above this threshold, individual tasks may be contending
1334	* for both memory bandwidth and any shared HT resources. This is an
1335	* approximation as the number of running tasks may not be related to
1336	* the number of busy CPUs due to sched_setaffinity.
1337	*/
1338	if (dst_running > imb_numa_nr)
1339	return imbalance;
1340
1341	/*
1342	* Allow a small imbalance based on a simple pair of communicating
1343	* tasks that remain local when the destination is lightly loaded.
1344	*/
1345	if (imbalance <= NUMA_IMBALANCE_MIN)
1346	return 0;
1347
1348	return imbalance;
1349	}
1350	#endif /* CONFIG_NUMA */
1351
1352	#ifdef CONFIG_NUMA_BALANCING
1353	/*
1354	* Approximate time to scan a full NUMA task in ms. The task scan period is
1355	* calculated based on the tasks virtual memory size and
1356	* numa_balancing_scan_size.
1357	*/
1358	unsigned int sysctl_numa_balancing_scan_period_min = 1000;
1359	unsigned int sysctl_numa_balancing_scan_period_max = 60000;
1360
1361	/* Portion of address space to scan in MB */
1362	unsigned int sysctl_numa_balancing_scan_size = 256;
1363
1364	/* Scan @scan_size MB every @scan_period after an initial @scan_delay in ms */
1365	unsigned int sysctl_numa_balancing_scan_delay = 1000;
1366
1367	/* The page with hint page fault latency < threshold in ms is considered hot */
1368	unsigned int sysctl_numa_balancing_hot_threshold = MSEC_PER_SEC;
1369
1370	struct numa_group {
1371	refcount_t refcount;
1372
1373	spinlock_t lock; /* nr_tasks, tasks */
1374	int nr_tasks;
1375	pid_t gid;
1376	int active_nodes;
1377
1378	struct rcu_head rcu;
1379	unsigned long total_faults;
1380	unsigned long max_faults_cpu;
1381	/*
1382	* faults[] array is split into two regions: faults_mem and faults_cpu.
1383	*
1384	* Faults_cpu is used to decide whether memory should move
1385	* towards the CPU. As a consequence, these stats are weighted
1386	* more by CPU use than by memory faults.
1387	*/
1388	unsigned long faults[];
1389	};
1390
1391	/*
1392	* For functions that can be called in multiple contexts that permit reading
1393	* ->numa_group (see struct task_struct for locking rules).
1394	*/
1395	static struct numa_group *deref_task_numa_group(struct task_struct *p)
1396	{
1397	return rcu_dereference_check(p->numa_group, p == current ||
1398	(lockdep_is_held(__rq_lockp(task_rq(p))) && !READ_ONCE(p->on_cpu)));
1399	}
1400
1401	static struct numa_group *deref_curr_numa_group(struct task_struct *p)
1402	{
1403	return rcu_dereference_protected(p->numa_group, p == current);
1404	}
1405
1406	static inline unsigned long group_faults_priv(struct numa_group *ng);
1407	static inline unsigned long group_faults_shared(struct numa_group *ng);
1408
1409	static unsigned int task_nr_scan_windows(struct task_struct *p)
1410	{
1411	unsigned long rss = 0;
1412	unsigned long nr_scan_pages;
1413
1414	/*
1415	* Calculations based on RSS as non-present and empty pages are skipped
1416	* by the PTE scanner and NUMA hinting faults should be trapped based
1417	* on resident pages
1418	*/
1419	nr_scan_pages = sysctl_numa_balancing_scan_size << (20 - PAGE_SHIFT);
1420	rss = get_mm_rss(mm: p->mm);
1421	if (!rss)
1422	rss = nr_scan_pages;
1423
1424	rss = round_up(rss, nr_scan_pages);
1425	return rss / nr_scan_pages;
1426	}
1427
1428	/* For sanity's sake, never scan more PTEs than MAX_SCAN_WINDOW MB/sec. */
1429	#define MAX_SCAN_WINDOW 2560
1430
1431	static unsigned int task_scan_min(struct task_struct *p)
1432	{
1433	unsigned int scan_size = READ_ONCE(sysctl_numa_balancing_scan_size);
1434	unsigned int scan, floor;
1435	unsigned int windows = 1;
1436
1437	if (scan_size < MAX_SCAN_WINDOW)
1438	windows = MAX_SCAN_WINDOW / scan_size;
1439	floor = 1000 / windows;
1440
1441	scan = sysctl_numa_balancing_scan_period_min / task_nr_scan_windows(p);
1442	return max_t(unsigned int, floor, scan);
1443	}
1444
1445	static unsigned int task_scan_start(struct task_struct *p)
1446	{
1447	unsigned long smin = task_scan_min(p);
1448	unsigned long period = smin;
1449	struct numa_group *ng;
1450
1451	/* Scale the maximum scan period with the amount of shared memory. */
1452	rcu_read_lock();
1453	ng = rcu_dereference(p->numa_group);
1454	if (ng) {
1455	unsigned long shared = group_faults_shared(ng);
1456	unsigned long private = group_faults_priv(ng);
1457
1458	period *= refcount_read(r: &ng->refcount);
1459	period *= shared + 1;
1460	period /= private + shared + 1;
1461	}
1462	rcu_read_unlock();
1463
1464	return max(smin, period);
1465	}
1466
1467	static unsigned int task_scan_max(struct task_struct *p)
1468	{
1469	unsigned long smin = task_scan_min(p);
1470	unsigned long smax;
1471	struct numa_group *ng;
1472
1473	/* Watch for min being lower than max due to floor calculations */
1474	smax = sysctl_numa_balancing_scan_period_max / task_nr_scan_windows(p);
1475
1476	/* Scale the maximum scan period with the amount of shared memory. */
1477	ng = deref_curr_numa_group(p);
1478	if (ng) {
1479	unsigned long shared = group_faults_shared(ng);
1480	unsigned long private = group_faults_priv(ng);
1481	unsigned long period = smax;
1482
1483	period *= refcount_read(r: &ng->refcount);
1484	period *= shared + 1;
1485	period /= private + shared + 1;
1486
1487	smax = max(smax, period);
1488	}
1489
1490	return max(smin, smax);
1491	}
1492
1493	static void account_numa_enqueue(struct rq *rq, struct task_struct *p)
1494	{
1495	rq->nr_numa_running += (p->numa_preferred_nid != NUMA_NO_NODE);
1496	rq->nr_preferred_running += (p->numa_preferred_nid == task_node(p));
1497	}
1498
1499	static void account_numa_dequeue(struct rq *rq, struct task_struct *p)
1500	{
1501	rq->nr_numa_running -= (p->numa_preferred_nid != NUMA_NO_NODE);
1502	rq->nr_preferred_running -= (p->numa_preferred_nid == task_node(p));
1503	}
1504
1505	/* Shared or private faults. */
1506	#define NR_NUMA_HINT_FAULT_TYPES 2
1507
1508	/* Memory and CPU locality */
1509	#define NR_NUMA_HINT_FAULT_STATS (NR_NUMA_HINT_FAULT_TYPES * 2)
1510
1511	/* Averaged statistics, and temporary buffers. */
1512	#define NR_NUMA_HINT_FAULT_BUCKETS (NR_NUMA_HINT_FAULT_STATS * 2)
1513
1514	pid_t task_numa_group_id(struct task_struct *p)
1515	{
1516	struct numa_group *ng;
1517	pid_t gid = 0;
1518
1519	rcu_read_lock();
1520	ng = rcu_dereference(p->numa_group);
1521	if (ng)
1522	gid = ng->gid;
1523	rcu_read_unlock();
1524
1525	return gid;
1526	}
1527
1528	/*
1529	* The averaged statistics, shared & private, memory & CPU,
1530	* occupy the first half of the array. The second half of the
1531	* array is for current counters, which are averaged into the
1532	* first set by task_numa_placement.
1533	*/
1534	static inline int task_faults_idx(enum numa_faults_stats s, int nid, int priv)
1535	{
1536	return NR_NUMA_HINT_FAULT_TYPES * (s * nr_node_ids + nid) + priv;
1537	}
1538
1539	static inline unsigned long task_faults(struct task_struct *p, int nid)
1540	{
1541	if (!p->numa_faults)
1542	return 0;
1543
1544	return p->numa_faults[task_faults_idx(s: NUMA_MEM, nid, priv: 0)] +
1545	p->numa_faults[task_faults_idx(s: NUMA_MEM, nid, priv: 1)];
1546	}
1547
1548	static inline unsigned long group_faults(struct task_struct *p, int nid)
1549	{
1550	struct numa_group *ng = deref_task_numa_group(p);
1551
1552	if (!ng)
1553	return 0;
1554
1555	return ng->faults[task_faults_idx(s: NUMA_MEM, nid, priv: 0)] +
1556	ng->faults[task_faults_idx(s: NUMA_MEM, nid, priv: 1)];
1557	}
1558
1559	static inline unsigned long group_faults_cpu(struct numa_group *group, int nid)
1560	{
1561	return group->faults[task_faults_idx(s: NUMA_CPU, nid, priv: 0)] +
1562	group->faults[task_faults_idx(s: NUMA_CPU, nid, priv: 1)];
1563	}
1564
1565	static inline unsigned long group_faults_priv(struct numa_group *ng)
1566	{
1567	unsigned long faults = 0;
1568	int node;
1569
1570	for_each_online_node(node) {
1571	faults += ng->faults[task_faults_idx(s: NUMA_MEM, nid: node, priv: 1)];
1572	}
1573
1574	return faults;
1575	}
1576
1577	static inline unsigned long group_faults_shared(struct numa_group *ng)
1578	{
1579	unsigned long faults = 0;
1580	int node;
1581
1582	for_each_online_node(node) {
1583	faults += ng->faults[task_faults_idx(s: NUMA_MEM, nid: node, priv: 0)];
1584	}
1585
1586	return faults;
1587	}
1588
1589	/*
1590	* A node triggering more than 1/3 as many NUMA faults as the maximum is
1591	* considered part of a numa group's pseudo-interleaving set. Migrations
1592	* between these nodes are slowed down, to allow things to settle down.
1593	*/
1594	#define ACTIVE_NODE_FRACTION 3
1595
1596	static bool numa_is_active_node(int nid, struct numa_group *ng)
1597	{
1598	return group_faults_cpu(group: ng, nid) * ACTIVE_NODE_FRACTION > ng->max_faults_cpu;
1599	}
1600
1601	/* Handle placement on systems where not all nodes are directly connected. */
1602	static unsigned long score_nearby_nodes(struct task_struct *p, int nid,
1603	int lim_dist, bool task)
1604	{
1605	unsigned long score = 0;
1606	int node, max_dist;
1607
1608	/*
1609	* All nodes are directly connected, and the same distance
1610	* from each other. No need for fancy placement algorithms.
1611	*/
1612	if (sched_numa_topology_type == NUMA_DIRECT)
1613	return 0;
1614
1615	/* sched_max_numa_distance may be changed in parallel. */
1616	max_dist = READ_ONCE(sched_max_numa_distance);
1617	/*
1618	* This code is called for each node, introducing N^2 complexity,
1619	* which should be ok given the number of nodes rarely exceeds 8.
1620	*/
1621	for_each_online_node(node) {
1622	unsigned long faults;
1623	int dist = node_distance(nid, node);
1624
1625	/*
1626	* The furthest away nodes in the system are not interesting
1627	* for placement; nid was already counted.
1628	*/
1629	if (dist >= max_dist || node == nid)
1630	continue;
1631
1632	/*
1633	* On systems with a backplane NUMA topology, compare groups
1634	* of nodes, and move tasks towards the group with the most
1635	* memory accesses. When comparing two nodes at distance
1636	* "hoplimit", only nodes closer by than "hoplimit" are part
1637	* of each group. Skip other nodes.
1638	*/
1639	if (sched_numa_topology_type == NUMA_BACKPLANE && dist >= lim_dist)
1640	continue;
1641
1642	/* Add up the faults from nearby nodes. */
1643	if (task)
1644	faults = task_faults(p, nid: node);
1645	else
1646	faults = group_faults(p, nid: node);
1647
1648	/*
1649	* On systems with a glueless mesh NUMA topology, there are
1650	* no fixed "groups of nodes". Instead, nodes that are not
1651	* directly connected bounce traffic through intermediate
1652	* nodes; a numa_group can occupy any set of nodes.
1653	* The further away a node is, the less the faults count.
1654	* This seems to result in good task placement.
1655	*/
1656	if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
1657	faults *= (max_dist - dist);
1658	faults /= (max_dist - LOCAL_DISTANCE);
1659	}
1660
1661	score += faults;
1662	}
1663
1664	return score;
1665	}
1666
1667	/*
1668	* These return the fraction of accesses done by a particular task, or
1669	* task group, on a particular numa node. The group weight is given a
1670	* larger multiplier, in order to group tasks together that are almost
1671	* evenly spread out between numa nodes.
1672	*/
1673	static inline unsigned long task_weight(struct task_struct *p, int nid,
1674	int dist)
1675	{
1676	unsigned long faults, total_faults;
1677
1678	if (!p->numa_faults)
1679	return 0;
1680
1681	total_faults = p->total_numa_faults;
1682
1683	if (!total_faults)
1684	return 0;
1685
1686	faults = task_faults(p, nid);
1687	faults += score_nearby_nodes(p, nid, lim_dist: dist, task: true);
1688
1689	return 1000 * faults / total_faults;
1690	}
1691
1692	static inline unsigned long group_weight(struct task_struct *p, int nid,
1693	int dist)
1694	{
1695	struct numa_group *ng = deref_task_numa_group(p);
1696	unsigned long faults, total_faults;
1697
1698	if (!ng)
1699	return 0;
1700
1701	total_faults = ng->total_faults;
1702
1703	if (!total_faults)
1704	return 0;
1705
1706	faults = group_faults(p, nid);
1707	faults += score_nearby_nodes(p, nid, lim_dist: dist, task: false);
1708
1709	return 1000 * faults / total_faults;
1710	}
1711
1712	/*
1713	* If memory tiering mode is enabled, cpupid of slow memory page is
1714	* used to record scan time instead of CPU and PID. When tiering mode
1715	* is disabled at run time, the scan time (in cpupid) will be
1716	* interpreted as CPU and PID. So CPU needs to be checked to avoid to
1717	* access out of array bound.
1718	*/
1719	static inline bool cpupid_valid(int cpupid)
1720	{
1721	return cpupid_to_cpu(cpupid) < nr_cpu_ids;
1722	}
1723
1724	/*
1725	* For memory tiering mode, if there are enough free pages (more than
1726	* enough watermark defined here) in fast memory node, to take full
1727	* advantage of fast memory capacity, all recently accessed slow
1728	* memory pages will be migrated to fast memory node without
1729	* considering hot threshold.
1730	*/
1731	static bool pgdat_free_space_enough(struct pglist_data *pgdat)
1732	{
1733	int z;
1734	unsigned long enough_wmark;
1735
1736	enough_wmark = max(1UL * 1024 * 1024 * 1024 >> PAGE_SHIFT,
1737	pgdat->node_present_pages >> 4);
1738	for (z = pgdat->nr_zones - 1; z >= 0; z--) {
1739	struct zone *zone = pgdat->node_zones + z;
1740
1741	if (!populated_zone(zone))
1742	continue;
1743
1744	if (zone_watermark_ok(z: zone, order: 0,
1745	wmark_pages(zone, WMARK_PROMO) + enough_wmark,
1746	highest_zoneidx: ZONE_MOVABLE, alloc_flags: 0))
1747	return true;
1748	}
1749	return false;
1750	}
1751
1752	/*
1753	* For memory tiering mode, when page tables are scanned, the scan
1754	* time will be recorded in struct page in addition to make page
1755	* PROT_NONE for slow memory page. So when the page is accessed, in
1756	* hint page fault handler, the hint page fault latency is calculated
1757	* via,
1758	*
1759	* hint page fault latency = hint page fault time - scan time
1760	*
1761	* The smaller the hint page fault latency, the higher the possibility
1762	* for the page to be hot.
1763	*/
1764	static int numa_hint_fault_latency(struct folio *folio)
1765	{
1766	int last_time, time;
1767
1768	time = jiffies_to_msecs(j: jiffies);
1769	last_time = folio_xchg_access_time(folio, time);
1770
1771	return (time - last_time) & PAGE_ACCESS_TIME_MASK;
1772	}
1773
1774	/*
1775	* For memory tiering mode, too high promotion/demotion throughput may
1776	* hurt application latency. So we provide a mechanism to rate limit
1777	* the number of pages that are tried to be promoted.
1778	*/
1779	static bool numa_promotion_rate_limit(struct pglist_data *pgdat,
1780	unsigned long rate_limit, int nr)
1781	{
1782	unsigned long nr_cand;
1783	unsigned int now, start;
1784
1785	now = jiffies_to_msecs(j: jiffies);
1786	mod_node_page_state(pgdat, PGPROMOTE_CANDIDATE, nr);
1787	nr_cand = node_page_state(pgdat, item: PGPROMOTE_CANDIDATE);
1788	start = pgdat->nbp_rl_start;
1789	if (now - start > MSEC_PER_SEC &&
1790	cmpxchg(&pgdat->nbp_rl_start, start, now) == start)
1791	pgdat->nbp_rl_nr_cand = nr_cand;
1792	if (nr_cand - pgdat->nbp_rl_nr_cand >= rate_limit)
1793	return true;
1794	return false;
1795	}
1796
1797	#define NUMA_MIGRATION_ADJUST_STEPS 16
1798
1799	static void numa_promotion_adjust_threshold(struct pglist_data *pgdat,
1800	unsigned long rate_limit,
1801	unsigned int ref_th)
1802	{
1803	unsigned int now, start, th_period, unit_th, th;
1804	unsigned long nr_cand, ref_cand, diff_cand;
1805
1806	now = jiffies_to_msecs(j: jiffies);
1807	th_period = sysctl_numa_balancing_scan_period_max;
1808	start = pgdat->nbp_th_start;
1809	if (now - start > th_period &&
1810	cmpxchg(&pgdat->nbp_th_start, start, now) == start) {
1811	ref_cand = rate_limit *
1812	sysctl_numa_balancing_scan_period_max / MSEC_PER_SEC;
1813	nr_cand = node_page_state(pgdat, item: PGPROMOTE_CANDIDATE);
1814	diff_cand = nr_cand - pgdat->nbp_th_nr_cand;
1815	unit_th = ref_th * 2 / NUMA_MIGRATION_ADJUST_STEPS;
1816	th = pgdat->nbp_threshold ? : ref_th;
1817	if (diff_cand > ref_cand * 11 / 10)
1818	th = max(th - unit_th, unit_th);
1819	else if (diff_cand < ref_cand * 9 / 10)
1820	th = min(th + unit_th, ref_th * 2);
1821	pgdat->nbp_th_nr_cand = nr_cand;
1822	pgdat->nbp_threshold = th;
1823	}
1824	}
1825
1826	bool should_numa_migrate_memory(struct task_struct *p, struct folio *folio,
1827	int src_nid, int dst_cpu)
1828	{
1829	struct numa_group *ng = deref_curr_numa_group(p);
1830	int dst_nid = cpu_to_node(cpu: dst_cpu);
1831	int last_cpupid, this_cpupid;
1832
1833	/*
1834	* Cannot migrate to memoryless nodes.
1835	*/
1836	if (!node_state(node: dst_nid, state: N_MEMORY))
1837	return false;
1838
1839	/*
1840	* The pages in slow memory node should be migrated according
1841	* to hot/cold instead of private/shared.
1842	*/
1843	if (sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING &&
1844	!node_is_toptier(node: src_nid)) {
1845	struct pglist_data *pgdat;
1846	unsigned long rate_limit;
1847	unsigned int latency, th, def_th;
1848
1849	pgdat = NODE_DATA(dst_nid);
1850	if (pgdat_free_space_enough(pgdat)) {
1851	/* workload changed, reset hot threshold */
1852	pgdat->nbp_threshold = 0;
1853	return true;
1854	}
1855
1856	def_th = sysctl_numa_balancing_hot_threshold;
1857	rate_limit = sysctl_numa_balancing_promote_rate_limit << \
1858	(20 - PAGE_SHIFT);
1859	numa_promotion_adjust_threshold(pgdat, rate_limit, ref_th: def_th);
1860
1861	th = pgdat->nbp_threshold ? : def_th;
1862	latency = numa_hint_fault_latency(folio);
1863	if (latency >= th)
1864	return false;
1865
1866	return !numa_promotion_rate_limit(pgdat, rate_limit,
1867	nr: folio_nr_pages(folio));
1868	}
1869
1870	this_cpupid = cpu_pid_to_cpupid(cpu: dst_cpu, current->pid);
1871	last_cpupid = folio_xchg_last_cpupid(folio, cpupid: this_cpupid);
1872
1873	if (!(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING) &&
1874	!node_is_toptier(node: src_nid) && !cpupid_valid(cpupid: last_cpupid))
1875	return false;
1876
1877	/*
1878	* Allow first faults or private faults to migrate immediately early in
1879	* the lifetime of a task. The magic number 4 is based on waiting for
1880	* two full passes of the "multi-stage node selection" test that is
1881	* executed below.
1882	*/
1883	if ((p->numa_preferred_nid == NUMA_NO_NODE || p->numa_scan_seq <= 4) &&
1884	(cpupid_pid_unset(cpupid: last_cpupid) || cpupid_match_pid(p, last_cpupid)))
1885	return true;
1886
1887	/*
1888	* Multi-stage node selection is used in conjunction with a periodic
1889	* migration fault to build a temporal task<->page relation. By using
1890	* a two-stage filter we remove short/unlikely relations.
1891	*
1892	* Using P(p) ~ n_p / n_t as per frequentist probability, we can equate
1893	* a task's usage of a particular page (n_p) per total usage of this
1894	* page (n_t) (in a given time-span) to a probability.
1895	*
1896	* Our periodic faults will sample this probability and getting the
1897	* same result twice in a row, given these samples are fully
1898	* independent, is then given by P(n)^2, provided our sample period
1899	* is sufficiently short compared to the usage pattern.
1900	*
1901	* This quadric squishes small probabilities, making it less likely we
1902	* act on an unlikely task<->page relation.
1903	*/
1904	if (!cpupid_pid_unset(cpupid: last_cpupid) &&
1905	cpupid_to_nid(cpupid: last_cpupid) != dst_nid)
1906	return false;
1907
1908	/* Always allow migrate on private faults */
1909	if (cpupid_match_pid(p, last_cpupid))
1910	return true;
1911
1912	/* A shared fault, but p->numa_group has not been set up yet. */
1913	if (!ng)
1914	return true;
1915
1916	/*
1917	* Destination node is much more heavily used than the source
1918	* node? Allow migration.
1919	*/
1920	if (group_faults_cpu(group: ng, nid: dst_nid) > group_faults_cpu(group: ng, nid: src_nid) *
1921	ACTIVE_NODE_FRACTION)
1922	return true;
1923
1924	/*
1925	* Distribute memory according to CPU & memory use on each node,
1926	* with 3/4 hysteresis to avoid unnecessary memory migrations:
1927	*
1928	* faults_cpu(dst) 3 faults_cpu(src)
1929	* --------------- * - > ---------------
1930	* faults_mem(dst) 4 faults_mem(src)
1931	*/
1932	return group_faults_cpu(group: ng, nid: dst_nid) * group_faults(p, nid: src_nid) * 3 >
1933	group_faults_cpu(group: ng, nid: src_nid) * group_faults(p, nid: dst_nid) * 4;
1934	}
1935
1936	/*
1937	* 'numa_type' describes the node at the moment of load balancing.
1938	*/
1939	enum numa_type {
1940	/* The node has spare capacity that can be used to run more tasks. */
1941	node_has_spare = 0,
1942	/*
1943	* The node is fully used and the tasks don't compete for more CPU
1944	* cycles. Nevertheless, some tasks might wait before running.
1945	*/
1946	node_fully_busy,
1947	/*
1948	* The node is overloaded and can't provide expected CPU cycles to all
1949	* tasks.
1950	*/
1951	node_overloaded
1952	};
1953
1954	/* Cached statistics for all CPUs within a node */
1955	struct numa_stats {
1956	unsigned long load;
1957	unsigned long runnable;
1958	unsigned long util;
1959	/* Total compute capacity of CPUs on a node */
1960	unsigned long compute_capacity;
1961	unsigned int nr_running;
1962	unsigned int weight;
1963	enum numa_type node_type;
1964	int idle_cpu;
1965	};
1966
1967	struct task_numa_env {
1968	struct task_struct *p;
1969
1970	int src_cpu, src_nid;
1971	int dst_cpu, dst_nid;
1972	int imb_numa_nr;
1973
1974	struct numa_stats src_stats, dst_stats;
1975
1976	int imbalance_pct;
1977	int dist;
1978
1979	struct task_struct *best_task;
1980	long best_imp;
1981	int best_cpu;
1982	};
1983
1984	static unsigned long cpu_load(struct rq *rq);
1985	static unsigned long cpu_runnable(struct rq *rq);
1986
1987	static inline enum
1988	numa_type numa_classify(unsigned int imbalance_pct,
1989	struct numa_stats *ns)
1990	{
1991	if ((ns->nr_running > ns->weight) &&
1992	(((ns->compute_capacity * 100) < (ns->util * imbalance_pct)) ||
1993	((ns->compute_capacity * imbalance_pct) < (ns->runnable * 100))))
1994	return node_overloaded;
1995
1996	if ((ns->nr_running < ns->weight) ||
1997	(((ns->compute_capacity * 100) > (ns->util * imbalance_pct)) &&
1998	((ns->compute_capacity * imbalance_pct) > (ns->runnable * 100))))
1999	return node_has_spare;
2000
2001	return node_fully_busy;
2002	}
2003
2004	#ifdef CONFIG_SCHED_SMT
2005	/* Forward declarations of select_idle_sibling helpers */
2006	static inline bool test_idle_cores(int cpu);
2007	static inline int numa_idle_core(int idle_core, int cpu)
2008	{
2009	if (!static_branch_likely(&sched_smt_present) ||
2010	idle_core >= 0 || !test_idle_cores(cpu))
2011	return idle_core;
2012
2013	/*
2014	* Prefer cores instead of packing HT siblings
2015	* and triggering future load balancing.
2016	*/
2017	if (is_core_idle(cpu))
2018	idle_core = cpu;
2019
2020	return idle_core;
2021	}
2022	#else
2023	static inline int numa_idle_core(int idle_core, int cpu)
2024	{
2025	return idle_core;
2026	}
2027	#endif
2028
2029	/*
2030	* Gather all necessary information to make NUMA balancing placement
2031	* decisions that are compatible with standard load balancer. This
2032	* borrows code and logic from update_sg_lb_stats but sharing a
2033	* common implementation is impractical.
2034	*/
2035	static void update_numa_stats(struct task_numa_env *env,
2036	struct numa_stats *ns, int nid,
2037	bool find_idle)
2038	{
2039	int cpu, idle_core = -1;
2040
2041	memset(ns, 0, sizeof(*ns));
2042	ns->idle_cpu = -1;
2043
2044	rcu_read_lock();
2045	for_each_cpu(cpu, cpumask_of_node(nid)) {
2046	struct rq *rq = cpu_rq(cpu);
2047
2048	ns->load += cpu_load(rq);
2049	ns->runnable += cpu_runnable(rq);
2050	ns->util += cpu_util_cfs(cpu);
2051	ns->nr_running += rq->cfs.h_nr_running;
2052	ns->compute_capacity += capacity_of(cpu);
2053
2054	if (find_idle && idle_core < 0 && !rq->nr_running && idle_cpu(cpu)) {
2055	if (READ_ONCE(rq->numa_migrate_on) ||
2056	!cpumask_test_cpu(cpu, cpumask: env->p->cpus_ptr))
2057	continue;
2058
2059	if (ns->idle_cpu == -1)
2060	ns->idle_cpu = cpu;
2061
2062	idle_core = numa_idle_core(idle_core, cpu);
2063	}
2064	}
2065	rcu_read_unlock();
2066
2067	ns->weight = cpumask_weight(srcp: cpumask_of_node(node: nid));
2068
2069	ns->node_type = numa_classify(imbalance_pct: env->imbalance_pct, ns);
2070
2071	if (idle_core >= 0)
2072	ns->idle_cpu = idle_core;
2073	}
2074
2075	static void task_numa_assign(struct task_numa_env *env,
2076	struct task_struct *p, long imp)
2077	{
2078	struct rq *rq = cpu_rq(env->dst_cpu);
2079
2080	/* Check if run-queue part of active NUMA balance. */
2081	if (env->best_cpu != env->dst_cpu && xchg(&rq->numa_migrate_on, 1)) {
2082	int cpu;
2083	int start = env->dst_cpu;
2084
2085	/* Find alternative idle CPU. */
2086	for_each_cpu_wrap(cpu, cpumask_of_node(env->dst_nid), start + 1) {
2087	if (cpu == env->best_cpu || !idle_cpu(cpu) ||
2088	!cpumask_test_cpu(cpu, cpumask: env->p->cpus_ptr)) {
2089	continue;
2090	}
2091
2092	env->dst_cpu = cpu;
2093	rq = cpu_rq(env->dst_cpu);
2094	if (!xchg(&rq->numa_migrate_on, 1))
2095	goto assign;
2096	}
2097
2098	/* Failed to find an alternative idle CPU */
2099	return;
2100	}
2101
2102	assign:
2103	/*
2104	* Clear previous best_cpu/rq numa-migrate flag, since task now
2105	* found a better CPU to move/swap.
2106	*/
2107	if (env->best_cpu != -1 && env->best_cpu != env->dst_cpu) {
2108	rq = cpu_rq(env->best_cpu);
2109	WRITE_ONCE(rq->numa_migrate_on, 0);
2110	}
2111
2112	if (env->best_task)
2113	put_task_struct(t: env->best_task);
2114	if (p)
2115	get_task_struct(t: p);
2116
2117	env->best_task = p;
2118	env->best_imp = imp;
2119	env->best_cpu = env->dst_cpu;
2120	}
2121
2122	static bool load_too_imbalanced(long src_load, long dst_load,
2123	struct task_numa_env *env)
2124	{
2125	long imb, old_imb;
2126	long orig_src_load, orig_dst_load;
2127	long src_capacity, dst_capacity;
2128
2129	/*
2130	* The load is corrected for the CPU capacity available on each node.
2131	*
2132	* src_load dst_load
2133	* ------------ vs ---------
2134	* src_capacity dst_capacity
2135	*/
2136	src_capacity = env->src_stats.compute_capacity;
2137	dst_capacity = env->dst_stats.compute_capacity;
2138
2139	imb = abs(dst_load * src_capacity - src_load * dst_capacity);
2140
2141	orig_src_load = env->src_stats.load;
2142	orig_dst_load = env->dst_stats.load;
2143
2144	old_imb = abs(orig_dst_load * src_capacity - orig_src_load * dst_capacity);
2145
2146	/* Would this change make things worse? */
2147	return (imb > old_imb);
2148	}
2149
2150	/*
2151	* Maximum NUMA importance can be 1998 (2*999);
2152	* SMALLIMP @ 30 would be close to 1998/64.
2153	* Used to deter task migration.
2154	*/
2155	#define SMALLIMP 30
2156
2157	/*
2158	* This checks if the overall compute and NUMA accesses of the system would
2159	* be improved if the source tasks was migrated to the target dst_cpu taking
2160	* into account that it might be best if task running on the dst_cpu should
2161	* be exchanged with the source task
2162	*/
2163	static bool task_numa_compare(struct task_numa_env *env,
2164	long taskimp, long groupimp, bool maymove)
2165	{
2166	struct numa_group *cur_ng, *p_ng = deref_curr_numa_group(p: env->p);
2167	struct rq *dst_rq = cpu_rq(env->dst_cpu);
2168	long imp = p_ng ? groupimp : taskimp;
2169	struct task_struct *cur;
2170	long src_load, dst_load;
2171	int dist = env->dist;
2172	long moveimp = imp;
2173	long load;
2174	bool stopsearch = false;
2175
2176	if (READ_ONCE(dst_rq->numa_migrate_on))
2177	return false;
2178
2179	rcu_read_lock();
2180	cur = rcu_dereference(dst_rq->curr);
2181	if (cur && ((cur->flags & PF_EXITING) || is_idle_task(p: cur)))
2182	cur = NULL;
2183
2184	/*
2185	* Because we have preemption enabled we can get migrated around and
2186	* end try selecting ourselves (current == env->p) as a swap candidate.
2187	*/
2188	if (cur == env->p) {
2189	stopsearch = true;
2190	goto unlock;
2191	}
2192
2193	if (!cur) {
2194	if (maymove && moveimp >= env->best_imp)
2195	goto assign;
2196	else
2197	goto unlock;
2198	}
2199
2200	/* Skip this swap candidate if cannot move to the source cpu. */
2201	if (!cpumask_test_cpu(cpu: env->src_cpu, cpumask: cur->cpus_ptr))
2202	goto unlock;
2203
2204	/*
2205	* Skip this swap candidate if it is not moving to its preferred
2206	* node and the best task is.
2207	*/
2208	if (env->best_task &&
2209	env->best_task->numa_preferred_nid == env->src_nid &&
2210	cur->numa_preferred_nid != env->src_nid) {
2211	goto unlock;
2212	}
2213
2214	/*
2215	* "imp" is the fault differential for the source task between the
2216	* source and destination node. Calculate the total differential for
2217	* the source task and potential destination task. The more negative
2218	* the value is, the more remote accesses that would be expected to
2219	* be incurred if the tasks were swapped.
2220	*
2221	* If dst and source tasks are in the same NUMA group, or not
2222	* in any group then look only at task weights.
2223	*/
2224	cur_ng = rcu_dereference(cur->numa_group);
2225	if (cur_ng == p_ng) {
2226	/*
2227	* Do not swap within a group or between tasks that have
2228	* no group if there is spare capacity. Swapping does
2229	* not address the load imbalance and helps one task at
2230	* the cost of punishing another.
2231	*/
2232	if (env->dst_stats.node_type == node_has_spare)
2233	goto unlock;
2234
2235	imp = taskimp + task_weight(p: cur, nid: env->src_nid, dist) -
2236	task_weight(p: cur, nid: env->dst_nid, dist);
2237	/*
2238	* Add some hysteresis to prevent swapping the
2239	* tasks within a group over tiny differences.
2240	*/
2241	if (cur_ng)
2242	imp -= imp / 16;
2243	} else {
2244	/*
2245	* Compare the group weights. If a task is all by itself
2246	* (not part of a group), use the task weight instead.
2247	*/
2248	if (cur_ng && p_ng)
2249	imp += group_weight(p: cur, nid: env->src_nid, dist) -
2250	group_weight(p: cur, nid: env->dst_nid, dist);
2251	else
2252	imp += task_weight(p: cur, nid: env->src_nid, dist) -
2253	task_weight(p: cur, nid: env->dst_nid, dist);
2254	}
2255
2256	/* Discourage picking a task already on its preferred node */
2257	if (cur->numa_preferred_nid == env->dst_nid)
2258	imp -= imp / 16;
2259
2260	/*
2261	* Encourage picking a task that moves to its preferred node.
2262	* This potentially makes imp larger than it's maximum of
2263	* 1998 (see SMALLIMP and task_weight for why) but in this
2264	* case, it does not matter.
2265	*/
2266	if (cur->numa_preferred_nid == env->src_nid)
2267	imp += imp / 8;
2268
2269	if (maymove && moveimp > imp && moveimp > env->best_imp) {
2270	imp = moveimp;
2271	cur = NULL;
2272	goto assign;
2273	}
2274
2275	/*
2276	* Prefer swapping with a task moving to its preferred node over a
2277	* task that is not.
2278	*/
2279	if (env->best_task && cur->numa_preferred_nid == env->src_nid &&
2280	env->best_task->numa_preferred_nid != env->src_nid) {
2281	goto assign;
2282	}
2283
2284	/*
2285	* If the NUMA importance is less than SMALLIMP,
2286	* task migration might only result in ping pong
2287	* of tasks and also hurt performance due to cache
2288	* misses.
2289	*/
2290	if (imp < SMALLIMP || imp <= env->best_imp + SMALLIMP / 2)
2291	goto unlock;
2292
2293	/*
2294	* In the overloaded case, try and keep the load balanced.
2295	*/
2296	load = task_h_load(p: env->p) - task_h_load(p: cur);
2297	if (!load)
2298	goto assign;
2299
2300	dst_load = env->dst_stats.load + load;
2301	src_load = env->src_stats.load - load;
2302
2303	if (load_too_imbalanced(src_load, dst_load, env))
2304	goto unlock;
2305
2306	assign:
2307	/* Evaluate an idle CPU for a task numa move. */
2308	if (!cur) {
2309	int cpu = env->dst_stats.idle_cpu;
2310
2311	/* Nothing cached so current CPU went idle since the search. */
2312	if (cpu < 0)
2313	cpu = env->dst_cpu;
2314
2315	/*
2316	* If the CPU is no longer truly idle and the previous best CPU
2317	* is, keep using it.
2318	*/
2319	if (!idle_cpu(cpu) && env->best_cpu >= 0 &&
2320	idle_cpu(cpu: env->best_cpu)) {
2321	cpu = env->best_cpu;
2322	}
2323
2324	env->dst_cpu = cpu;
2325	}
2326
2327	task_numa_assign(env, p: cur, imp);
2328
2329	/*
2330	* If a move to idle is allowed because there is capacity or load
2331	* balance improves then stop the search. While a better swap
2332	* candidate may exist, a search is not free.
2333	*/
2334	if (maymove && !cur && env->best_cpu >= 0 && idle_cpu(cpu: env->best_cpu))
2335	stopsearch = true;
2336
2337	/*
2338	* If a swap candidate must be identified and the current best task
2339	* moves its preferred node then stop the search.
2340	*/
2341	if (!maymove && env->best_task &&
2342	env->best_task->numa_preferred_nid == env->src_nid) {
2343	stopsearch = true;
2344	}
2345	unlock:
2346	rcu_read_unlock();
2347
2348	return stopsearch;
2349	}
2350
2351	static void task_numa_find_cpu(struct task_numa_env *env,
2352	long taskimp, long groupimp)
2353	{
2354	bool maymove = false;
2355	int cpu;
2356
2357	/*
2358	* If dst node has spare capacity, then check if there is an
2359	* imbalance that would be overruled by the load balancer.
2360	*/
2361	if (env->dst_stats.node_type == node_has_spare) {
2362	unsigned int imbalance;
2363	int src_running, dst_running;
2364
2365	/*
2366	* Would movement cause an imbalance? Note that if src has
2367	* more running tasks that the imbalance is ignored as the
2368	* move improves the imbalance from the perspective of the
2369	* CPU load balancer.
2370	* */
2371	src_running = env->src_stats.nr_running - 1;
2372	dst_running = env->dst_stats.nr_running + 1;
2373	imbalance = max(0, dst_running - src_running);
2374	imbalance = adjust_numa_imbalance(imbalance, dst_running,
2375	imb_numa_nr: env->imb_numa_nr);
2376
2377	/* Use idle CPU if there is no imbalance */
2378	if (!imbalance) {
2379	maymove = true;
2380	if (env->dst_stats.idle_cpu >= 0) {
2381	env->dst_cpu = env->dst_stats.idle_cpu;
2382	task_numa_assign(env, NULL, imp: 0);
2383	return;
2384	}
2385	}
2386	} else {
2387	long src_load, dst_load, load;
2388	/*
2389	* If the improvement from just moving env->p direction is better
2390	* than swapping tasks around, check if a move is possible.
2391	*/
2392	load = task_h_load(p: env->p);
2393	dst_load = env->dst_stats.load + load;
2394	src_load = env->src_stats.load - load;
2395	maymove = !load_too_imbalanced(src_load, dst_load, env);
2396	}
2397
2398	for_each_cpu(cpu, cpumask_of_node(env->dst_nid)) {
2399	/* Skip this CPU if the source task cannot migrate */
2400	if (!cpumask_test_cpu(cpu, cpumask: env->p->cpus_ptr))
2401	continue;
2402
2403	env->dst_cpu = cpu;
2404	if (task_numa_compare(env, taskimp, groupimp, maymove))
2405	break;
2406	}
2407	}
2408
2409	static int task_numa_migrate(struct task_struct *p)
2410	{
2411	struct task_numa_env env = {
2412	.p = p,
2413
2414	.src_cpu = task_cpu(p),
2415	.src_nid = task_node(p),
2416
2417	.imbalance_pct = 112,
2418
2419	.best_task = NULL,
2420	.best_imp = 0,
2421	.best_cpu = -1,
2422	};
2423	unsigned long taskweight, groupweight;
2424	struct sched_domain *sd;
2425	long taskimp, groupimp;
2426	struct numa_group *ng;
2427	struct rq *best_rq;
2428	int nid, ret, dist;
2429
2430	/*
2431	* Pick the lowest SD_NUMA domain, as that would have the smallest
2432	* imbalance and would be the first to start moving tasks about.
2433	*
2434	* And we want to avoid any moving of tasks about, as that would create
2435	* random movement of tasks -- counter the numa conditions we're trying
2436	* to satisfy here.
2437	*/
2438	rcu_read_lock();
2439	sd = rcu_dereference(per_cpu(sd_numa, env.src_cpu));
2440	if (sd) {
2441	env.imbalance_pct = 100 + (sd->imbalance_pct - 100) / 2;
2442	env.imb_numa_nr = sd->imb_numa_nr;
2443	}
2444	rcu_read_unlock();
2445
2446	/*
2447	* Cpusets can break the scheduler domain tree into smaller
2448	* balance domains, some of which do not cross NUMA boundaries.
2449	* Tasks that are "trapped" in such domains cannot be migrated
2450	* elsewhere, so there is no point in (re)trying.
2451	*/
2452	if (unlikely(!sd)) {
2453	sched_setnuma(p, node: task_node(p));
2454	return -EINVAL;
2455	}
2456
2457	env.dst_nid = p->numa_preferred_nid;
2458	dist = env.dist = node_distance(env.src_nid, env.dst_nid);
2459	taskweight = task_weight(p, nid: env.src_nid, dist);
2460	groupweight = group_weight(p, nid: env.src_nid, dist);
2461	update_numa_stats(env: &env, ns: &env.src_stats, nid: env.src_nid, find_idle: false);
2462	taskimp = task_weight(p, nid: env.dst_nid, dist) - taskweight;
2463	groupimp = group_weight(p, nid: env.dst_nid, dist) - groupweight;
2464	update_numa_stats(env: &env, ns: &env.dst_stats, nid: env.dst_nid, find_idle: true);
2465
2466	/* Try to find a spot on the preferred nid. */
2467	task_numa_find_cpu(env: &env, taskimp, groupimp);
2468
2469	/*
2470	* Look at other nodes in these cases:
2471	* - there is no space available on the preferred_nid
2472	* - the task is part of a numa_group that is interleaved across
2473	* multiple NUMA nodes; in order to better consolidate the group,
2474	* we need to check other locations.
2475	*/
2476	ng = deref_curr_numa_group(p);
2477	if (env.best_cpu == -1 || (ng && ng->active_nodes > 1)) {
2478	for_each_node_state(nid, N_CPU) {
2479	if (nid == env.src_nid || nid == p->numa_preferred_nid)
2480	continue;
2481
2482	dist = node_distance(env.src_nid, env.dst_nid);
2483	if (sched_numa_topology_type == NUMA_BACKPLANE &&
2484	dist != env.dist) {
2485	taskweight = task_weight(p, nid: env.src_nid, dist);
2486	groupweight = group_weight(p, nid: env.src_nid, dist);
2487	}
2488
2489	/* Only consider nodes where both task and groups benefit */
2490	taskimp = task_weight(p, nid, dist) - taskweight;
2491	groupimp = group_weight(p, nid, dist) - groupweight;
2492	if (taskimp < 0 && groupimp < 0)
2493	continue;
2494
2495	env.dist = dist;
2496	env.dst_nid = nid;
2497	update_numa_stats(env: &env, ns: &env.dst_stats, nid: env.dst_nid, find_idle: true);
2498	task_numa_find_cpu(env: &env, taskimp, groupimp);
2499	}
2500	}
2501
2502	/*
2503	* If the task is part of a workload that spans multiple NUMA nodes,
2504	* and is migrating into one of the workload's active nodes, remember
2505	* this node as the task's preferred numa node, so the workload can
2506	* settle down.
2507	* A task that migrated to a second choice node will be better off
2508	* trying for a better one later. Do not set the preferred node here.
2509	*/
2510	if (ng) {
2511	if (env.best_cpu == -1)
2512	nid = env.src_nid;
2513	else
2514	nid = cpu_to_node(cpu: env.best_cpu);
2515
2516	if (nid != p->numa_preferred_nid)
2517	sched_setnuma(p, node: nid);
2518	}
2519
2520	/* No better CPU than the current one was found. */
2521	if (env.best_cpu == -1) {
2522	trace_sched_stick_numa(src_tsk: p, src_cpu: env.src_cpu, NULL, dst_cpu: -1);
2523	return -EAGAIN;
2524	}
2525
2526	best_rq = cpu_rq(env.best_cpu);
2527	if (env.best_task == NULL) {
2528	ret = migrate_task_to(p, cpu: env.best_cpu);
2529	WRITE_ONCE(best_rq->numa_migrate_on, 0);
2530	if (ret != 0)
2531	trace_sched_stick_numa(src_tsk: p, src_cpu: env.src_cpu, NULL, dst_cpu: env.best_cpu);
2532	return ret;
2533	}
2534
2535	ret = migrate_swap(p, t: env.best_task, cpu: env.best_cpu, scpu: env.src_cpu);
2536	WRITE_ONCE(best_rq->numa_migrate_on, 0);
2537
2538	if (ret != 0)
2539	trace_sched_stick_numa(src_tsk: p, src_cpu: env.src_cpu, dst_tsk: env.best_task, dst_cpu: env.best_cpu);
2540	put_task_struct(t: env.best_task);
2541	return ret;
2542	}
2543
2544	/* Attempt to migrate a task to a CPU on the preferred node. */
2545	static void numa_migrate_preferred(struct task_struct *p)
2546	{
2547	unsigned long interval = HZ;
2548
2549	/* This task has no NUMA fault statistics yet */
2550	if (unlikely(p->numa_preferred_nid == NUMA_NO_NODE || !p->numa_faults))
2551	return;
2552
2553	/* Periodically retry migrating the task to the preferred node */
2554	interval = min(interval, msecs_to_jiffies(p->numa_scan_period) / 16);
2555	p->numa_migrate_retry = jiffies + interval;
2556
2557	/* Success if task is already running on preferred CPU */
2558	if (task_node(p) == p->numa_preferred_nid)
2559	return;
2560
2561	/* Otherwise, try migrate to a CPU on the preferred node */
2562	task_numa_migrate(p);
2563	}
2564
2565	/*
2566	* Find out how many nodes the workload is actively running on. Do this by
2567	* tracking the nodes from which NUMA hinting faults are triggered. This can
2568	* be different from the set of nodes where the workload's memory is currently
2569	* located.
2570	*/
2571	static void numa_group_count_active_nodes(struct numa_group *numa_group)
2572	{
2573	unsigned long faults, max_faults = 0;
2574	int nid, active_nodes = 0;
2575
2576	for_each_node_state(nid, N_CPU) {
2577	faults = group_faults_cpu(group: numa_group, nid);
2578	if (faults > max_faults)
2579	max_faults = faults;
2580	}
2581
2582	for_each_node_state(nid, N_CPU) {
2583	faults = group_faults_cpu(group: numa_group, nid);
2584	if (faults * ACTIVE_NODE_FRACTION > max_faults)
2585	active_nodes++;
2586	}
2587
2588	numa_group->max_faults_cpu = max_faults;
2589	numa_group->active_nodes = active_nodes;
2590	}
2591
2592	/*
2593	* When adapting the scan rate, the period is divided into NUMA_PERIOD_SLOTS
2594	* increments. The more local the fault statistics are, the higher the scan
2595	* period will be for the next scan window. If local/(local+remote) ratio is
2596	* below NUMA_PERIOD_THRESHOLD (where range of ratio is 1..NUMA_PERIOD_SLOTS)
2597	* the scan period will decrease. Aim for 70% local accesses.
2598	*/
2599	#define NUMA_PERIOD_SLOTS 10
2600	#define NUMA_PERIOD_THRESHOLD 7
2601
2602	/*
2603	* Increase the scan period (slow down scanning) if the majority of
2604	* our memory is already on our local node, or if the majority of
2605	* the page accesses are shared with other processes.
2606	* Otherwise, decrease the scan period.
2607	*/
2608	static void update_task_scan_period(struct task_struct *p,
2609	unsigned long shared, unsigned long private)
2610	{
2611	unsigned int period_slot;
2612	int lr_ratio, ps_ratio;
2613	int diff;
2614
2615	unsigned long remote = p->numa_faults_locality[0];
2616	unsigned long local = p->numa_faults_locality[1];
2617
2618	/*
2619	* If there were no record hinting faults then either the task is
2620	* completely idle or all activity is in areas that are not of interest
2621	* to automatic numa balancing. Related to that, if there were failed
2622	* migration then it implies we are migrating too quickly or the local
2623	* node is overloaded. In either case, scan slower
2624	*/
2625	if (local + shared == 0 || p->numa_faults_locality[2]) {
2626	p->numa_scan_period = min(p->numa_scan_period_max,
2627	p->numa_scan_period << 1);
2628
2629	p->mm->numa_next_scan = jiffies +
2630	msecs_to_jiffies(m: p->numa_scan_period);
2631
2632	return;
2633	}
2634
2635	/*
2636	* Prepare to scale scan period relative to the current period.
2637	* == NUMA_PERIOD_THRESHOLD scan period stays the same
2638	* < NUMA_PERIOD_THRESHOLD scan period decreases (scan faster)
2639	* >= NUMA_PERIOD_THRESHOLD scan period increases (scan slower)
2640	*/
2641	period_slot = DIV_ROUND_UP(p->numa_scan_period, NUMA_PERIOD_SLOTS);
2642	lr_ratio = (local * NUMA_PERIOD_SLOTS) / (local + remote);
2643	ps_ratio = (private * NUMA_PERIOD_SLOTS) / (private + shared);
2644
2645	if (ps_ratio >= NUMA_PERIOD_THRESHOLD) {
2646	/*
2647	* Most memory accesses are local. There is no need to
2648	* do fast NUMA scanning, since memory is already local.
2649	*/
2650	int slot = ps_ratio - NUMA_PERIOD_THRESHOLD;
2651	if (!slot)
2652	slot = 1;
2653	diff = slot * period_slot;
2654	} else if (lr_ratio >= NUMA_PERIOD_THRESHOLD) {
2655	/*
2656	* Most memory accesses are shared with other tasks.
2657	* There is no point in continuing fast NUMA scanning,
2658	* since other tasks may just move the memory elsewhere.
2659	*/
2660	int slot = lr_ratio - NUMA_PERIOD_THRESHOLD;
2661	if (!slot)
2662	slot = 1;
2663	diff = slot * period_slot;
2664	} else {
2665	/*
2666	* Private memory faults exceed (SLOTS-THRESHOLD)/SLOTS,
2667	* yet they are not on the local NUMA node. Speed up
2668	* NUMA scanning to get the memory moved over.
2669	*/
2670	int ratio = max(lr_ratio, ps_ratio);
2671	diff = -(NUMA_PERIOD_THRESHOLD - ratio) * period_slot;
2672	}
2673
2674	p->numa_scan_period = clamp(p->numa_scan_period + diff,
2675	task_scan_min(p), task_scan_max(p));
2676	memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
2677	}
2678
2679	/*
2680	* Get the fraction of time the task has been running since the last
2681	* NUMA placement cycle. The scheduler keeps similar statistics, but
2682	* decays those on a 32ms period, which is orders of magnitude off
2683	* from the dozens-of-seconds NUMA balancing period. Use the scheduler
2684	* stats only if the task is so new there are no NUMA statistics yet.
2685	*/
2686	static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period)
2687	{
2688	u64 runtime, delta, now;
2689	/* Use the start of this time slice to avoid calculations. */
2690	now = p->se.exec_start;
2691	runtime = p->se.sum_exec_runtime;
2692
2693	if (p->last_task_numa_placement) {
2694	delta = runtime - p->last_sum_exec_runtime;
2695	*period = now - p->last_task_numa_placement;
2696
2697	/* Avoid time going backwards, prevent potential divide error: */
2698	if (unlikely((s64)*period < 0))
2699	*period = 0;
2700	} else {
2701	delta = p->se.avg.load_sum;
2702	*period = LOAD_AVG_MAX;
2703	}
2704
2705	p->last_sum_exec_runtime = runtime;
2706	p->last_task_numa_placement = now;
2707
2708	return delta;
2709	}
2710
2711	/*
2712	* Determine the preferred nid for a task in a numa_group. This needs to
2713	* be done in a way that produces consistent results with group_weight,
2714	* otherwise workloads might not converge.
2715	*/
2716	static int preferred_group_nid(struct task_struct *p, int nid)
2717	{
2718	nodemask_t nodes;
2719	int dist;
2720
2721	/* Direct connections between all NUMA nodes. */
2722	if (sched_numa_topology_type == NUMA_DIRECT)
2723	return nid;
2724
2725	/*
2726	* On a system with glueless mesh NUMA topology, group_weight
2727	* scores nodes according to the number of NUMA hinting faults on
2728	* both the node itself, and on nearby nodes.
2729	*/
2730	if (sched_numa_topology_type == NUMA_GLUELESS_MESH) {
2731	unsigned long score, max_score = 0;
2732	int node, max_node = nid;
2733
2734	dist = sched_max_numa_distance;
2735
2736	for_each_node_state(node, N_CPU) {
2737	score = group_weight(p, nid: node, dist);
2738	if (score > max_score) {
2739	max_score = score;
2740	max_node = node;
2741	}
2742	}
2743	return max_node;
2744	}
2745
2746	/*
2747	* Finding the preferred nid in a system with NUMA backplane
2748	* interconnect topology is more involved. The goal is to locate
2749	* tasks from numa_groups near each other in the system, and
2750	* untangle workloads from different sides of the system. This requires
2751	* searching down the hierarchy of node groups, recursively searching
2752	* inside the highest scoring group of nodes. The nodemask tricks
2753	* keep the complexity of the search down.
2754	*/
2755	nodes = node_states[N_CPU];
2756	for (dist = sched_max_numa_distance; dist > LOCAL_DISTANCE; dist--) {
2757	unsigned long max_faults = 0;
2758	nodemask_t max_group = NODE_MASK_NONE;
2759	int a, b;
2760
2761	/* Are there nodes at this distance from each other? */
2762	if (!find_numa_distance(distance: dist))
2763	continue;
2764
2765	for_each_node_mask(a, nodes) {
2766	unsigned long faults = 0;
2767	nodemask_t this_group;
2768	nodes_clear(this_group);
2769
2770	/* Sum group's NUMA faults; includes a==b case. */
2771	for_each_node_mask(b, nodes) {
2772	if (node_distance(a, b) < dist) {
2773	faults += group_faults(p, nid: b);
2774	node_set(b, this_group);
2775	node_clear(b, nodes);
2776	}
2777	}
2778
2779	/* Remember the top group. */
2780	if (faults > max_faults) {
2781	max_faults = faults;
2782	max_group = this_group;
2783	/*
2784	* subtle: at the smallest distance there is
2785	* just one node left in each "group", the
2786	* winner is the preferred nid.
2787	*/
2788	nid = a;
2789	}
2790	}
2791	/* Next round, evaluate the nodes within max_group. */
2792	if (!max_faults)
2793	break;
2794	nodes = max_group;
2795	}
2796	return nid;
2797	}
2798
2799	static void task_numa_placement(struct task_struct *p)
2800	{
2801	int seq, nid, max_nid = NUMA_NO_NODE;
2802	unsigned long max_faults = 0;
2803	unsigned long fault_types[2] = { 0, 0 };
2804	unsigned long total_faults;
2805	u64 runtime, period;
2806	spinlock_t *group_lock = NULL;
2807	struct numa_group *ng;
2808
2809	/*
2810	* The p->mm->numa_scan_seq field gets updated without
2811	* exclusive access. Use READ_ONCE() here to ensure
2812	* that the field is read in a single access:
2813	*/
2814	seq = READ_ONCE(p->mm->numa_scan_seq);
2815	if (p->numa_scan_seq == seq)
2816	return;
2817	p->numa_scan_seq = seq;
2818	p->numa_scan_period_max = task_scan_max(p);
2819
2820	total_faults = p->numa_faults_locality[0] +
2821	p->numa_faults_locality[1];
2822	runtime = numa_get_avg_runtime(p, period: &period);
2823
2824	/* If the task is part of a group prevent parallel updates to group stats */
2825	ng = deref_curr_numa_group(p);
2826	if (ng) {
2827	group_lock = &ng->lock;
2828	spin_lock_irq(lock: group_lock);
2829	}
2830
2831	/* Find the node with the highest number of faults */
2832	for_each_online_node(nid) {
2833	/* Keep track of the offsets in numa_faults array */
2834	int mem_idx, membuf_idx, cpu_idx, cpubuf_idx;
2835	unsigned long faults = 0, group_faults = 0;
2836	int priv;
2837
2838	for (priv = 0; priv < NR_NUMA_HINT_FAULT_TYPES; priv++) {
2839	long diff, f_diff, f_weight;
2840
2841	mem_idx = task_faults_idx(s: NUMA_MEM, nid, priv);
2842	membuf_idx = task_faults_idx(s: NUMA_MEMBUF, nid, priv);
2843	cpu_idx = task_faults_idx(s: NUMA_CPU, nid, priv);
2844	cpubuf_idx = task_faults_idx(s: NUMA_CPUBUF, nid, priv);
2845
2846	/* Decay existing window, copy faults since last scan */
2847	diff = p->numa_faults[membuf_idx] - p->numa_faults[mem_idx] / 2;
2848	fault_types[priv] += p->numa_faults[membuf_idx];
2849	p->numa_faults[membuf_idx] = 0;
2850
2851	/*
2852	* Normalize the faults_from, so all tasks in a group
2853	* count according to CPU use, instead of by the raw
2854	* number of faults. Tasks with little runtime have
2855	* little over-all impact on throughput, and thus their
2856	* faults are less important.
2857	*/
2858	f_weight = div64_u64(dividend: runtime << 16, divisor: period + 1);
2859	f_weight = (f_weight * p->numa_faults[cpubuf_idx]) /
2860	(total_faults + 1);
2861	f_diff = f_weight - p->numa_faults[cpu_idx] / 2;
2862	p->numa_faults[cpubuf_idx] = 0;
2863
2864	p->numa_faults[mem_idx] += diff;
2865	p->numa_faults[cpu_idx] += f_diff;
2866	faults += p->numa_faults[mem_idx];
2867	p->total_numa_faults += diff;
2868	if (ng) {
2869	/*
2870	* safe because we can only change our own group
2871	*
2872	* mem_idx represents the offset for a given
2873	* nid and priv in a specific region because it
2874	* is at the beginning of the numa_faults array.
2875	*/
2876	ng->faults[mem_idx] += diff;
2877	ng->faults[cpu_idx] += f_diff;
2878	ng->total_faults += diff;
2879	group_faults += ng->faults[mem_idx];
2880	}
2881	}
2882
2883	if (!ng) {
2884	if (faults > max_faults) {
2885	max_faults = faults;
2886	max_nid = nid;
2887	}
2888	} else if (group_faults > max_faults) {
2889	max_faults = group_faults;
2890	max_nid = nid;
2891	}
2892	}
2893
2894	/* Cannot migrate task to CPU-less node */
2895	max_nid = numa_nearest_node(node: max_nid, state: N_CPU);
2896
2897	if (ng) {
2898	numa_group_count_active_nodes(numa_group: ng);
2899	spin_unlock_irq(lock: group_lock);
2900	max_nid = preferred_group_nid(p, nid: max_nid);
2901	}
2902
2903	if (max_faults) {
2904	/* Set the new preferred node */
2905	if (max_nid != p->numa_preferred_nid)
2906	sched_setnuma(p, node: max_nid);
2907	}
2908
2909	update_task_scan_period(p, shared: fault_types[0], private: fault_types[1]);
2910	}
2911
2912	static inline int get_numa_group(struct numa_group *grp)
2913	{
2914	return refcount_inc_not_zero(r: &grp->refcount);
2915	}
2916
2917	static inline void put_numa_group(struct numa_group *grp)
2918	{
2919	if (refcount_dec_and_test(r: &grp->refcount))
2920	kfree_rcu(grp, rcu);
2921	}
2922
2923	static void task_numa_group(struct task_struct *p, int cpupid, int flags,
2924	int *priv)
2925	{
2926	struct numa_group *grp, *my_grp;
2927	struct task_struct *tsk;
2928	bool join = false;
2929	int cpu = cpupid_to_cpu(cpupid);
2930	int i;
2931
2932	if (unlikely(!deref_curr_numa_group(p))) {
2933	unsigned int size = sizeof(struct numa_group) +
2934	NR_NUMA_HINT_FAULT_STATS *
2935	nr_node_ids * sizeof(unsigned long);
2936
2937	grp = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
2938	if (!grp)
2939	return;
2940
2941	refcount_set(r: &grp->refcount, n: 1);
2942	grp->active_nodes = 1;
2943	grp->max_faults_cpu = 0;
2944	spin_lock_init(&grp->lock);
2945	grp->gid = p->pid;
2946
2947	for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
2948	grp->faults[i] = p->numa_faults[i];
2949
2950	grp->total_faults = p->total_numa_faults;
2951
2952	grp->nr_tasks++;
2953	rcu_assign_pointer(p->numa_group, grp);
2954	}
2955
2956	rcu_read_lock();
2957	tsk = READ_ONCE(cpu_rq(cpu)->curr);
2958
2959	if (!cpupid_match_pid(tsk, cpupid))
2960	goto no_join;
2961
2962	grp = rcu_dereference(tsk->numa_group);
2963	if (!grp)
2964	goto no_join;
2965
2966	my_grp = deref_curr_numa_group(p);
2967	if (grp == my_grp)
2968	goto no_join;
2969
2970	/*
2971	* Only join the other group if its bigger; if we're the bigger group,
2972	* the other task will join us.
2973	*/
2974	if (my_grp->nr_tasks > grp->nr_tasks)
2975	goto no_join;
2976
2977	/*
2978	* Tie-break on the grp address.
2979	*/
2980	if (my_grp->nr_tasks == grp->nr_tasks && my_grp > grp)
2981	goto no_join;
2982
2983	/* Always join threads in the same process. */
2984	if (tsk->mm == current->mm)
2985	join = true;
2986
2987	/* Simple filter to avoid false positives due to PID collisions */
2988	if (flags & TNF_SHARED)
2989	join = true;
2990
2991	/* Update priv based on whether false sharing was detected */
2992	*priv = !join;
2993
2994	if (join && !get_numa_group(grp))
2995	goto no_join;
2996
2997	rcu_read_unlock();
2998
2999	if (!join)
3000	return;
3001
3002	WARN_ON_ONCE(irqs_disabled());
3003	double_lock_irq(l1: &my_grp->lock, l2: &grp->lock);
3004
3005	for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++) {
3006	my_grp->faults[i] -= p->numa_faults[i];
3007	grp->faults[i] += p->numa_faults[i];
3008	}
3009	my_grp->total_faults -= p->total_numa_faults;
3010	grp->total_faults += p->total_numa_faults;
3011
3012	my_grp->nr_tasks--;
3013	grp->nr_tasks++;
3014
3015	spin_unlock(lock: &my_grp->lock);
3016	spin_unlock_irq(lock: &grp->lock);
3017
3018	rcu_assign_pointer(p->numa_group, grp);
3019
3020	put_numa_group(grp: my_grp);
3021	return;
3022
3023	no_join:
3024	rcu_read_unlock();
3025	return;
3026	}
3027
3028	/*
3029	* Get rid of NUMA statistics associated with a task (either current or dead).
3030	* If @final is set, the task is dead and has reached refcount zero, so we can
3031	* safely free all relevant data structures. Otherwise, there might be
3032	* concurrent reads from places like load balancing and procfs, and we should
3033	* reset the data back to default state without freeing ->numa_faults.
3034	*/
3035	void task_numa_free(struct task_struct *p, bool final)
3036	{
3037	/* safe: p either is current or is being freed by current */
3038	struct numa_group *grp = rcu_dereference_raw(p->numa_group);
3039	unsigned long *numa_faults = p->numa_faults;
3040	unsigned long flags;
3041	int i;
3042
3043	if (!numa_faults)
3044	return;
3045
3046	if (grp) {
3047	spin_lock_irqsave(&grp->lock, flags);
3048	for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
3049	grp->faults[i] -= p->numa_faults[i];
3050	grp->total_faults -= p->total_numa_faults;
3051
3052	grp->nr_tasks--;
3053	spin_unlock_irqrestore(lock: &grp->lock, flags);
3054	RCU_INIT_POINTER(p->numa_group, NULL);
3055	put_numa_group(grp);
3056	}
3057
3058	if (final) {
3059	p->numa_faults = NULL;
3060	kfree(objp: numa_faults);
3061	} else {
3062	p->total_numa_faults = 0;
3063	for (i = 0; i < NR_NUMA_HINT_FAULT_STATS * nr_node_ids; i++)
3064	numa_faults[i] = 0;
3065	}
3066	}
3067
3068	/*
3069	* Got a PROT_NONE fault for a page on @node.
3070	*/
3071	void task_numa_fault(int last_cpupid, int mem_node, int pages, int flags)
3072	{
3073	struct task_struct *p = current;
3074	bool migrated = flags & TNF_MIGRATED;
3075	int cpu_node = task_node(current);
3076	int local = !!(flags & TNF_FAULT_LOCAL);
3077	struct numa_group *ng;
3078	int priv;
3079
3080	if (!static_branch_likely(&sched_numa_balancing))
3081	return;
3082
3083	/* for example, ksmd faulting in a user's mm */
3084	if (!p->mm)
3085	return;
3086
3087	/*
3088	* NUMA faults statistics are unnecessary for the slow memory
3089	* node for memory tiering mode.
3090	*/
3091	if (!node_is_toptier(node: mem_node) &&
3092	(sysctl_numa_balancing_mode & NUMA_BALANCING_MEMORY_TIERING ||
3093	!cpupid_valid(cpupid: last_cpupid)))
3094	return;
3095
3096	/* Allocate buffer to track faults on a per-node basis */
3097	if (unlikely(!p->numa_faults)) {
3098	int size = sizeof(*p->numa_faults) *
3099	NR_NUMA_HINT_FAULT_BUCKETS * nr_node_ids;
3100
3101	p->numa_faults = kzalloc(size, GFP_KERNEL|__GFP_NOWARN);
3102	if (!p->numa_faults)
3103	return;
3104
3105	p->total_numa_faults = 0;
3106	memset(p->numa_faults_locality, 0, sizeof(p->numa_faults_locality));
3107	}
3108
3109	/*
3110	* First accesses are treated as private, otherwise consider accesses
3111	* to be private if the accessing pid has not changed
3112	*/
3113	if (unlikely(last_cpupid == (-1 & LAST_CPUPID_MASK))) {
3114	priv = 1;
3115	} else {
3116	priv = cpupid_match_pid(p, last_cpupid);
3117	if (!priv && !(flags & TNF_NO_GROUP))
3118	task_numa_group(p, cpupid: last_cpupid, flags, priv: &priv);
3119	}
3120
3121	/*
3122	* If a workload spans multiple NUMA nodes, a shared fault that
3123	* occurs wholly within the set of nodes that the workload is
3124	* actively using should be counted as local. This allows the
3125	* scan rate to slow down when a workload has settled down.
3126	*/
3127	ng = deref_curr_numa_group(p);
3128	if (!priv && !local && ng && ng->active_nodes > 1 &&
3129	numa_is_active_node(nid: cpu_node, ng) &&
3130	numa_is_active_node(nid: mem_node, ng))
3131	local = 1;
3132
3133	/*
3134	* Retry to migrate task to preferred node periodically, in case it
3135	* previously failed, or the scheduler moved us.
3136	*/
3137	if (time_after(jiffies, p->numa_migrate_retry)) {
3138	task_numa_placement(p);
3139	numa_migrate_preferred(p);
3140	}
3141
3142	if (migrated)
3143	p->numa_pages_migrated += pages;
3144	if (flags & TNF_MIGRATE_FAIL)
3145	p->numa_faults_locality[2] += pages;
3146
3147	p->numa_faults[task_faults_idx(s: NUMA_MEMBUF, nid: mem_node, priv)] += pages;
3148	p->numa_faults[task_faults_idx(s: NUMA_CPUBUF, nid: cpu_node, priv)] += pages;
3149	p->numa_faults_locality[local] += pages;
3150	}
3151
3152	static void reset_ptenuma_scan(struct task_struct *p)
3153	{
3154	/*
3155	* We only did a read acquisition of the mmap sem, so
3156	* p->mm->numa_scan_seq is written to without exclusive access
3157	* and the update is not guaranteed to be atomic. That's not
3158	* much of an issue though, since this is just used for
3159	* statistical sampling. Use READ_ONCE/WRITE_ONCE, which are not
3160	* expensive, to avoid any form of compiler optimizations:
3161	*/
3162	WRITE_ONCE(p->mm->numa_scan_seq, READ_ONCE(p->mm->numa_scan_seq) + 1);
3163	p->mm->numa_scan_offset = 0;
3164	}
3165
3166	static bool vma_is_accessed(struct mm_struct *mm, struct vm_area_struct *vma)
3167	{
3168	unsigned long pids;
3169	/*
3170	* Allow unconditional access first two times, so that all the (pages)
3171	* of VMAs get prot_none fault introduced irrespective of accesses.
3172	* This is also done to avoid any side effect of task scanning
3173	* amplifying the unfairness of disjoint set of VMAs' access.
3174	*/
3175	if ((READ_ONCE(current->mm->numa_scan_seq) - vma->numab_state->start_scan_seq) < 2)
3176	return true;
3177
3178	pids = vma->numab_state->pids_active[0] | vma->numab_state->pids_active[1];
3179	if (test_bit(hash_32(current->pid, ilog2(BITS_PER_LONG)), &pids))
3180	return true;
3181
3182	/*
3183	* Complete a scan that has already started regardless of PID access, or
3184	* some VMAs may never be scanned in multi-threaded applications:
3185	*/
3186	if (mm->numa_scan_offset > vma->vm_start) {
3187	trace_sched_skip_vma_numa(mm, vma, reason: NUMAB_SKIP_IGNORE_PID);
3188	return true;
3189	}
3190
3191	return false;
3192	}
3193
3194	#define VMA_PID_RESET_PERIOD (4 * sysctl_numa_balancing_scan_delay)
3195
3196	/*
3197	* The expensive part of numa migration is done from task_work context.
3198	* Triggered from task_tick_numa().
3199	*/
3200	static void task_numa_work(struct callback_head *work)
3201	{
3202	unsigned long migrate, next_scan, now = jiffies;
3203	struct task_struct *p = current;
3204	struct mm_struct *mm = p->mm;
3205	u64 runtime = p->se.sum_exec_runtime;
3206	struct vm_area_struct *vma;
3207	unsigned long start, end;
3208	unsigned long nr_pte_updates = 0;
3209	long pages, virtpages;
3210	struct vma_iterator vmi;
3211	bool vma_pids_skipped;
3212	bool vma_pids_forced = false;
3213
3214	SCHED_WARN_ON(p != container_of(work, struct task_struct, numa_work));
3215
3216	work->next = work;
3217	/*
3218	* Who cares about NUMA placement when they're dying.
3219	*
3220	* NOTE: make sure not to dereference p->mm before this check,
3221	* exit_task_work() happens _after_ exit_mm() so we could be called
3222	* without p->mm even though we still had it when we enqueued this
3223	* work.
3224	*/
3225	if (p->flags & PF_EXITING)
3226	return;
3227
3228	if (!mm->numa_next_scan) {
3229	mm->numa_next_scan = now +
3230	msecs_to_jiffies(m: sysctl_numa_balancing_scan_delay);
3231	}
3232
3233	/*
3234	* Enforce maximal scan/migration frequency..
3235	*/
3236	migrate = mm->numa_next_scan;
3237	if (time_before(now, migrate))
3238	return;
3239
3240	if (p->numa_scan_period == 0) {
3241	p->numa_scan_period_max = task_scan_max(p);
3242	p->numa_scan_period = task_scan_start(p);
3243	}
3244
3245	next_scan = now + msecs_to_jiffies(m: p->numa_scan_period);
3246	if (!try_cmpxchg(&mm->numa_next_scan, &migrate, next_scan))
3247	return;
3248
3249	/*
3250	* Delay this task enough that another task of this mm will likely win
3251	* the next time around.
3252	*/
3253	p->node_stamp += 2 * TICK_NSEC;
3254
3255	pages = sysctl_numa_balancing_scan_size;
3256	pages <<= 20 - PAGE_SHIFT; /* MB in pages */
3257	virtpages = pages * 8; /* Scan up to this much virtual space */
3258	if (!pages)
3259	return;
3260
3261
3262	if (!mmap_read_trylock(mm))
3263	return;
3264
3265	/*
3266	* VMAs are skipped if the current PID has not trapped a fault within
3267	* the VMA recently. Allow scanning to be forced if there is no
3268	* suitable VMA remaining.
3269	*/
3270	vma_pids_skipped = false;
3271
3272	retry_pids:
3273	start = mm->numa_scan_offset;
3274	vma_iter_init(vmi: &vmi, mm, addr: start);
3275	vma = vma_next(vmi: &vmi);
3276	if (!vma) {
3277	reset_ptenuma_scan(p);
3278	start = 0;
3279	vma_iter_set(vmi: &vmi, addr: start);
3280	vma = vma_next(vmi: &vmi);
3281	}
3282
3283	do {
3284	if (!vma_migratable(vma) || !vma_policy_mof(vma) ||
3285	is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_MIXEDMAP)) {
3286	trace_sched_skip_vma_numa(mm, vma, reason: NUMAB_SKIP_UNSUITABLE);
3287	continue;
3288	}
3289
3290	/*
3291	* Shared library pages mapped by multiple processes are not
3292	* migrated as it is expected they are cache replicated. Avoid
3293	* hinting faults in read-only file-backed mappings or the vdso
3294	* as migrating the pages will be of marginal benefit.
3295	*/
3296	if (!vma->vm_mm ||
3297	(vma->vm_file && (vma->vm_flags & (VM_READ|VM_WRITE)) == (VM_READ))) {
3298	trace_sched_skip_vma_numa(mm, vma, reason: NUMAB_SKIP_SHARED_RO);
3299	continue;
3300	}
3301
3302	/*
3303	* Skip inaccessible VMAs to avoid any confusion between
3304	* PROT_NONE and NUMA hinting ptes
3305	*/
3306	if (!vma_is_accessible(vma)) {
3307	trace_sched_skip_vma_numa(mm, vma, reason: NUMAB_SKIP_INACCESSIBLE);
3308	continue;
3309	}
3310
3311	/* Initialise new per-VMA NUMAB state. */
3312	if (!vma->numab_state) {
3313	vma->numab_state = kzalloc(size: sizeof(struct vma_numab_state),
3314	GFP_KERNEL);
3315	if (!vma->numab_state)
3316	continue;
3317
3318	vma->numab_state->start_scan_seq = mm->numa_scan_seq;
3319
3320	vma->numab_state->next_scan = now +
3321	msecs_to_jiffies(m: sysctl_numa_balancing_scan_delay);
3322
3323	/* Reset happens after 4 times scan delay of scan start */
3324	vma->numab_state->pids_active_reset = vma->numab_state->next_scan +
3325	msecs_to_jiffies(VMA_PID_RESET_PERIOD);
3326
3327	/*
3328	* Ensure prev_scan_seq does not match numa_scan_seq,
3329	* to prevent VMAs being skipped prematurely on the
3330	* first scan:
3331	*/
3332	vma->numab_state->prev_scan_seq = mm->numa_scan_seq - 1;
3333	}
3334
3335	/*
3336	* Scanning the VMA's of short lived tasks add more overhead. So
3337	* delay the scan for new VMAs.
3338	*/
3339	if (mm->numa_scan_seq && time_before(jiffies,
3340	vma->numab_state->next_scan)) {
3341	trace_sched_skip_vma_numa(mm, vma, reason: NUMAB_SKIP_SCAN_DELAY);
3342	continue;
3343	}
3344
3345	/* RESET access PIDs regularly for old VMAs. */
3346	if (mm->numa_scan_seq &&
3347	time_after(jiffies, vma->numab_state->pids_active_reset)) {
3348	vma->numab_state->pids_active_reset = vma->numab_state->pids_active_reset +
3349	msecs_to_jiffies(VMA_PID_RESET_PERIOD);
3350	vma->numab_state->pids_active[0] = READ_ONCE(vma->numab_state->pids_active[1]);
3351	vma->numab_state->pids_active[1] = 0;
3352	}
3353
3354	/* Do not rescan VMAs twice within the same sequence. */
3355	if (vma->numab_state->prev_scan_seq == mm->numa_scan_seq) {
3356	mm->numa_scan_offset = vma->vm_end;
3357	trace_sched_skip_vma_numa(mm, vma, reason: NUMAB_SKIP_SEQ_COMPLETED);
3358	continue;
3359	}
3360
3361	/*
3362	* Do not scan the VMA if task has not accessed it, unless no other
3363	* VMA candidate exists.
3364	*/
3365	if (!vma_pids_forced && !vma_is_accessed(mm, vma)) {
3366	vma_pids_skipped = true;
3367	trace_sched_skip_vma_numa(mm, vma, reason: NUMAB_SKIP_PID_INACTIVE);
3368	continue;
3369	}
3370
3371	do {
3372	start = max(start, vma->vm_start);
3373	end = ALIGN(start + (pages << PAGE_SHIFT), HPAGE_SIZE);
3374	end = min(end, vma->vm_end);
3375	nr_pte_updates = change_prot_numa(vma, start, end);
3376
3377	/*
3378	* Try to scan sysctl_numa_balancing_size worth of
3379	* hpages that have at least one present PTE that
3380	* is not already pte-numa. If the VMA contains
3381	* areas that are unused or already full of prot_numa
3382	* PTEs, scan up to virtpages, to skip through those
3383	* areas faster.
3384	*/
3385	if (nr_pte_updates)
3386	pages -= (end - start) >> PAGE_SHIFT;
3387	virtpages -= (end - start) >> PAGE_SHIFT;
3388
3389	start = end;
3390	if (pages <= 0 || virtpages <= 0)
3391	goto out;
3392
3393	cond_resched();
3394	} while (end != vma->vm_end);
3395
3396	/* VMA scan is complete, do not scan until next sequence. */
3397	vma->numab_state->prev_scan_seq = mm->numa_scan_seq;
3398
3399	/*
3400	* Only force scan within one VMA at a time, to limit the
3401	* cost of scanning a potentially uninteresting VMA.
3402	*/
3403	if (vma_pids_forced)
3404	break;
3405	} for_each_vma(vmi, vma);
3406
3407	/*
3408	* If no VMAs are remaining and VMAs were skipped due to the PID
3409	* not accessing the VMA previously, then force a scan to ensure
3410	* forward progress:
3411	*/
3412	if (!vma && !vma_pids_forced && vma_pids_skipped) {
3413	vma_pids_forced = true;
3414	goto retry_pids;
3415	}
3416
3417	out:
3418	/*
3419	* It is possible to reach the end of the VMA list but the last few
3420	* VMAs are not guaranteed to the vma_migratable. If they are not, we
3421	* would find the !migratable VMA on the next scan but not reset the
3422	* scanner to the start so check it now.
3423	*/
3424	if (vma)
3425	mm->numa_scan_offset = start;
3426	else
3427	reset_ptenuma_scan(p);
3428	mmap_read_unlock(mm);
3429
3430	/*
3431	* Make sure tasks use at least 32x as much time to run other code
3432	* than they used here, to limit NUMA PTE scanning overhead to 3% max.
3433	* Usually update_task_scan_period slows down scanning enough; on an
3434	* overloaded system we need to limit overhead on a per task basis.
3435	*/
3436	if (unlikely(p->se.sum_exec_runtime != runtime)) {
3437	u64 diff = p->se.sum_exec_runtime - runtime;
3438	p->node_stamp += 32 * diff;
3439	}
3440	}
3441
3442	void init_numa_balancing(unsigned long clone_flags, struct task_struct *p)
3443	{
3444	int mm_users = 0;
3445	struct mm_struct *mm = p->mm;
3446
3447	if (mm) {
3448	mm_users = atomic_read(v: &mm->mm_users);
3449	if (mm_users == 1) {
3450	mm->numa_next_scan = jiffies + msecs_to_jiffies(m: sysctl_numa_balancing_scan_delay);
3451	mm->numa_scan_seq = 0;
3452	}
3453	}
3454	p->node_stamp = 0;
3455	p->numa_scan_seq = mm ? mm->numa_scan_seq : 0;
3456	p->numa_scan_period = sysctl_numa_balancing_scan_delay;
3457	p->numa_migrate_retry = 0;
3458	/* Protect against double add, see task_tick_numa and task_numa_work */
3459	p->numa_work.next = &p->numa_work;
3460	p->numa_faults = NULL;
3461	p->numa_pages_migrated = 0;
3462	p->total_numa_faults = 0;
3463	RCU_INIT_POINTER(p->numa_group, NULL);
3464	p->last_task_numa_placement = 0;
3465	p->last_sum_exec_runtime = 0;
3466
3467	init_task_work(twork: &p->numa_work, func: task_numa_work);
3468
3469	/* New address space, reset the preferred nid */
3470	if (!(clone_flags & CLONE_VM)) {
3471	p->numa_preferred_nid = NUMA_NO_NODE;
3472	return;
3473	}
3474
3475	/*
3476	* New thread, keep existing numa_preferred_nid which should be copied
3477	* already by arch_dup_task_struct but stagger when scans start.
3478	*/
3479	if (mm) {
3480	unsigned int delay;
3481
3482	delay = min_t(unsigned int, task_scan_max(current),
3483	current->numa_scan_period * mm_users * NSEC_PER_MSEC);
3484	delay += 2 * TICK_NSEC;
3485	p->node_stamp = delay;
3486	}
3487	}
3488
3489	/*
3490	* Drive the periodic memory faults..
3491	*/
3492	static void task_tick_numa(struct rq *rq, struct task_struct *curr)
3493	{
3494	struct callback_head *work = &curr->numa_work;
3495	u64 period, now;
3496
3497	/*
3498	* We don't care about NUMA placement if we don't have memory.
3499	*/
3500	if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) || work->next != work)
3501	return;
3502
3503	/*
3504	* Using runtime rather than walltime has the dual advantage that
3505	* we (mostly) drive the selection from busy threads and that the
3506	* task needs to have done some actual work before we bother with
3507	* NUMA placement.
3508	*/
3509	now = curr->se.sum_exec_runtime;
3510	period = (u64)curr->numa_scan_period * NSEC_PER_MSEC;
3511
3512	if (now > curr->node_stamp + period) {
3513	if (!curr->node_stamp)
3514	curr->numa_scan_period = task_scan_start(p: curr);
3515	curr->node_stamp += period;
3516
3517	if (!time_before(jiffies, curr->mm->numa_next_scan))
3518	task_work_add(task: curr, twork: work, mode: TWA_RESUME);
3519	}
3520	}
3521
3522	static void update_scan_period(struct task_struct *p, int new_cpu)
3523	{
3524	int src_nid = cpu_to_node(cpu: task_cpu(p));
3525	int dst_nid = cpu_to_node(cpu: new_cpu);
3526
3527	if (!static_branch_likely(&sched_numa_balancing))
3528	return;
3529
3530	if (!p->mm || !p->numa_faults || (p->flags & PF_EXITING))
3531	return;
3532
3533	if (src_nid == dst_nid)
3534	return;
3535
3536	/*
3537	* Allow resets if faults have been trapped before one scan
3538	* has completed. This is most likely due to a new task that
3539	* is pulled cross-node due to wakeups or load balancing.
3540	*/
3541	if (p->numa_scan_seq) {
3542	/*
3543	* Avoid scan adjustments if moving to the preferred
3544	* node or if the task was not previously running on
3545	* the preferred node.
3546	*/
3547	if (dst_nid == p->numa_preferred_nid ||
3548	(p->numa_preferred_nid != NUMA_NO_NODE &&
3549	src_nid != p->numa_preferred_nid))
3550	return;
3551	}
3552
3553	p->numa_scan_period = task_scan_start(p);
3554	}
3555
3556	#else
3557	static void task_tick_numa(struct rq *rq, struct task_struct *curr)
3558	{
3559	}
3560
3561	static inline void account_numa_enqueue(struct rq *rq, struct task_struct *p)
3562	{
3563	}
3564
3565	static inline void account_numa_dequeue(struct rq *rq, struct task_struct *p)
3566	{
3567	}
3568
3569	static inline void update_scan_period(struct task_struct *p, int new_cpu)
3570	{
3571	}
3572
3573	#endif /* CONFIG_NUMA_BALANCING */
3574
3575	static void
3576	account_entity_enqueue(struct cfs_rq *cfs_rq, struct sched_entity *se)
3577	{
3578	update_load_add(lw: &cfs_rq->load, inc: se->load.weight);
3579	#ifdef CONFIG_SMP
3580	if (entity_is_task(se)) {
3581	struct rq *rq = rq_of(cfs_rq);
3582
3583	account_numa_enqueue(rq, p: task_of(se));
3584	list_add(new: &se->group_node, head: &rq->cfs_tasks);
3585	}
3586	#endif
3587	cfs_rq->nr_running++;
3588	if (se_is_idle(se))
3589	cfs_rq->idle_nr_running++;
3590	}
3591
3592	static void
3593	account_entity_dequeue(struct cfs_rq *cfs_rq, struct sched_entity *se)
3594	{
3595	update_load_sub(lw: &cfs_rq->load, dec: se->load.weight);
3596	#ifdef CONFIG_SMP
3597	if (entity_is_task(se)) {
3598	account_numa_dequeue(rq: rq_of(cfs_rq), p: task_of(se));
3599	list_del_init(entry: &se->group_node);
3600	}
3601	#endif
3602	cfs_rq->nr_running--;
3603	if (se_is_idle(se))
3604	cfs_rq->idle_nr_running--;
3605	}
3606
3607	/*
3608	* Signed add and clamp on underflow.
3609	*
3610	* Explicitly do a load-store to ensure the intermediate value never hits
3611	* memory. This allows lockless observations without ever seeing the negative
3612	* values.
3613	*/
3614	#define add_positive(_ptr, _val) do { \
3615	typeof(_ptr) ptr = (_ptr); \
3616	typeof(_val) val = (_val); \
3617	typeof(*ptr) res, var = READ_ONCE(*ptr); \
3618	\
3619	res = var + val; \
3620	\
3621	if (val < 0 && res > var) \
3622	res = 0; \
3623	\
3624	WRITE_ONCE(*ptr, res); \
3625	} while (0)
3626
3627	/*
3628	* Unsigned subtract and clamp on underflow.
3629	*
3630	* Explicitly do a load-store to ensure the intermediate value never hits
3631	* memory. This allows lockless observations without ever seeing the negative
3632	* values.
3633	*/
3634	#define sub_positive(_ptr, _val) do { \
3635	typeof(_ptr) ptr = (_ptr); \
3636	typeof(*ptr) val = (_val); \
3637	typeof(*ptr) res, var = READ_ONCE(*ptr); \
3638	res = var - val; \
3639	if (res > var) \
3640	res = 0; \
3641	WRITE_ONCE(*ptr, res); \
3642	} while (0)
3643
3644	/*
3645	* Remove and clamp on negative, from a local variable.
3646	*
3647	* A variant of sub_positive(), which does not use explicit load-store
3648	* and is thus optimized for local variable updates.
3649	*/
3650	#define lsub_positive(_ptr, _val) do { \
3651	typeof(_ptr) ptr = (_ptr); \
3652	*ptr -= min_t(typeof(*ptr), *ptr, _val); \
3653	} while (0)
3654
3655	#ifdef CONFIG_SMP
3656	static inline void
3657	enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3658	{
3659	cfs_rq->avg.load_avg += se->avg.load_avg;
3660	cfs_rq->avg.load_sum += se_weight(se) * se->avg.load_sum;
3661	}
3662
3663	static inline void
3664	dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
3665	{
3666	sub_positive(&cfs_rq->avg.load_avg, se->avg.load_avg);
3667	sub_positive(&cfs_rq->avg.load_sum, se_weight(se) * se->avg.load_sum);
3668	/* See update_cfs_rq_load_avg() */
3669	cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum,
3670	cfs_rq->avg.load_avg * PELT_MIN_DIVIDER);
3671	}
3672	#else
3673	static inline void
3674	enqueue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
3675	static inline void
3676	dequeue_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { }
3677	#endif
3678
3679	static void reweight_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *se,
3680	unsigned long weight)
3681	{
3682	unsigned long old_weight = se->load.weight;
3683	u64 avruntime = avg_vruntime(cfs_rq);
3684	s64 vlag, vslice;
3685
3686	/*
3687	* VRUNTIME
3688	* ========
3689	*
3690	* COROLLARY #1: The virtual runtime of the entity needs to be
3691	* adjusted if re-weight at !0-lag point.
3692	*
3693	* Proof: For contradiction assume this is not true, so we can
3694	* re-weight without changing vruntime at !0-lag point.
3695	*
3696	* Weight VRuntime Avg-VRuntime
3697	* before w v V
3698	* after w' v' V'
3699	*
3700	* Since lag needs to be preserved through re-weight:
3701	*
3702	* lag = (V - v)*w = (V'- v')*w', where v = v'
3703	* ==> V' = (V - v)*w/w' + v (1)
3704	*
3705	* Let W be the total weight of the entities before reweight,
3706	* since V' is the new weighted average of entities:
3707	*
3708	* V' = (WV + w'v - wv) / (W + w' - w) (2)
3709	*
3710	* by using (1) & (2) we obtain:
3711	*
3712	* (WV + w'v - wv) / (W + w' - w) = (V - v)*w/w' + v
3713	* ==> (WV-Wv+Wv+w'v-wv)/(W+w'-w) = (V - v)*w/w' + v
3714	* ==> (WV - Wv)/(W + w' - w) + v = (V - v)*w/w' + v
3715	* ==> (V - v)*W/(W + w' - w) = (V - v)*w/w' (3)
3716	*
3717	* Since we are doing at !0-lag point which means V != v, we
3718	* can simplify (3):
3719	*
3720	* ==> W / (W + w' - w) = w / w'
3721	* ==> Ww' = Ww + ww' - ww
3722	* ==> W * (w' - w) = w * (w' - w)
3723	* ==> W = w (re-weight indicates w' != w)
3724	*
3725	* So the cfs_rq contains only one entity, hence vruntime of
3726	* the entity @v should always equal to the cfs_rq's weighted
3727	* average vruntime @V, which means we will always re-weight
3728	* at 0-lag point, thus breach assumption. Proof completed.
3729	*
3730	*
3731	* COROLLARY #2: Re-weight does NOT affect weighted average
3732	* vruntime of all the entities.
3733	*
3734	* Proof: According to corollary #1, Eq. (1) should be:
3735	*
3736	* (V - v)*w = (V' - v')*w'
3737	* ==> v' = V' - (V - v)*w/w' (4)
3738	*
3739	* According to the weighted average formula, we have:
3740	*
3741	* V' = (WV - wv + w'v') / (W - w + w')
3742	* = (WV - wv + w'(V' - (V - v)w/w')) / (W - w + w')
3743	* = (WV - wv + w'V' - Vw + wv) / (W - w + w')
3744	* = (WV + w'V' - Vw) / (W - w + w')
3745	*
3746	* ==> V'*(W - w + w') = WV + w'V' - Vw
3747	* ==> V' * (W - w) = (W - w) * V (5)
3748	*
3749	* If the entity is the only one in the cfs_rq, then reweight
3750	* always occurs at 0-lag point, so V won't change. Or else
3751	* there are other entities, hence W != w, then Eq. (5) turns
3752	* into V' = V. So V won't change in either case, proof done.
3753	*
3754	*
3755	* So according to corollary #1 & #2, the effect of re-weight
3756	* on vruntime should be:
3757	*
3758	* v' = V' - (V - v) * w / w' (4)
3759	* = V - (V - v) * w / w'
3760	* = V - vl * w / w'
3761	* = V - vl'
3762	*/
3763	if (avruntime != se->vruntime) {
3764	vlag = (s64)(avruntime - se->vruntime);
3765	vlag = div_s64(dividend: vlag * old_weight, divisor: weight);
3766	se->vruntime = avruntime - vlag;
3767	}
3768
3769	/*
3770	* DEADLINE
3771	* ========
3772	*
3773	* When the weight changes, the virtual time slope changes and
3774	* we should adjust the relative virtual deadline accordingly.
3775	*
3776	* d' = v' + (d - v)*w/w'
3777	* = V' - (V - v)*w/w' + (d - v)*w/w'
3778	* = V - (V - v)*w/w' + (d - v)*w/w'
3779	* = V + (d - V)*w/w'
3780	*/
3781	vslice = (s64)(se->deadline - avruntime);
3782	vslice = div_s64(dividend: vslice * old_weight, divisor: weight);
3783	se->deadline = avruntime + vslice;
3784	}
3785
3786	static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se,
3787	unsigned long weight)
3788	{
3789	bool curr = cfs_rq->curr == se;
3790
3791	if (se->on_rq) {
3792	/* commit outstanding execution time */
3793	if (curr)
3794	update_curr(cfs_rq);
3795	else
3796	__dequeue_entity(cfs_rq, se);
3797	update_load_sub(lw: &cfs_rq->load, dec: se->load.weight);
3798	}
3799	dequeue_load_avg(cfs_rq, se);
3800
3801	if (!se->on_rq) {
3802	/*
3803	* Because we keep se->vlag = V - v_i, while: lag_i = w_i*(V - v_i),
3804	* we need to scale se->vlag when w_i changes.
3805	*/
3806	se->vlag = div_s64(dividend: se->vlag * se->load.weight, divisor: weight);
3807	} else {
3808	reweight_eevdf(cfs_rq, se, weight);
3809	}
3810
3811	update_load_set(lw: &se->load, w: weight);
3812
3813	#ifdef CONFIG_SMP
3814	do {
3815	u32 divider = get_pelt_divider(avg: &se->avg);
3816
3817	se->avg.load_avg = div_u64(dividend: se_weight(se) * se->avg.load_sum, divisor: divider);
3818	} while (0);
3819	#endif
3820
3821	enqueue_load_avg(cfs_rq, se);
3822	if (se->on_rq) {
3823	update_load_add(lw: &cfs_rq->load, inc: se->load.weight);
3824	if (!curr)
3825	__enqueue_entity(cfs_rq, se);
3826
3827	/*
3828	* The entity's vruntime has been adjusted, so let's check
3829	* whether the rq-wide min_vruntime needs updated too. Since
3830	* the calculations above require stable min_vruntime rather
3831	* than up-to-date one, we do the update at the end of the
3832	* reweight process.
3833	*/
3834	update_min_vruntime(cfs_rq);
3835	}
3836	}
3837
3838	void reweight_task(struct task_struct *p, int prio)
3839	{
3840	struct sched_entity *se = &p->se;
3841	struct cfs_rq *cfs_rq = cfs_rq_of(se);
3842	struct load_weight *load = &se->load;
3843	unsigned long weight = scale_load(sched_prio_to_weight[prio]);
3844
3845	reweight_entity(cfs_rq, se, weight);
3846	load->inv_weight = sched_prio_to_wmult[prio];
3847	}
3848
3849	static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
3850
3851	#ifdef CONFIG_FAIR_GROUP_SCHED
3852	#ifdef CONFIG_SMP
3853	/*
3854	* All this does is approximate the hierarchical proportion which includes that
3855	* global sum we all love to hate.
3856	*
3857	* That is, the weight of a group entity, is the proportional share of the
3858	* group weight based on the group runqueue weights. That is:
3859	*
3860	* tg->weight * grq->load.weight
3861	* ge->load.weight = ----------------------------- (1)
3862	* \Sum grq->load.weight
3863	*
3864	* Now, because computing that sum is prohibitively expensive to compute (been
3865	* there, done that) we approximate it with this average stuff. The average
3866	* moves slower and therefore the approximation is cheaper and more stable.
3867	*
3868	* So instead of the above, we substitute:
3869	*
3870	* grq->load.weight -> grq->avg.load_avg (2)
3871	*
3872	* which yields the following:
3873	*
3874	* tg->weight * grq->avg.load_avg
3875	* ge->load.weight = ------------------------------ (3)
3876	* tg->load_avg
3877	*
3878	* Where: tg->load_avg ~= \Sum grq->avg.load_avg
3879	*
3880	* That is shares_avg, and it is right (given the approximation (2)).
3881	*
3882	* The problem with it is that because the average is slow -- it was designed
3883	* to be exactly that of course -- this leads to transients in boundary
3884	* conditions. In specific, the case where the group was idle and we start the
3885	* one task. It takes time for our CPU's grq->avg.load_avg to build up,
3886	* yielding bad latency etc..
3887	*
3888	* Now, in that special case (1) reduces to:
3889	*
3890	* tg->weight * grq->load.weight
3891	* ge->load.weight = ----------------------------- = tg->weight (4)
3892	* grp->load.weight
3893	*
3894	* That is, the sum collapses because all other CPUs are idle; the UP scenario.
3895	*
3896	* So what we do is modify our approximation (3) to approach (4) in the (near)
3897	* UP case, like:
3898	*
3899	* ge->load.weight =
3900	*
3901	* tg->weight * grq->load.weight
3902	* --------------------------------------------------- (5)
3903	* tg->load_avg - grq->avg.load_avg + grq->load.weight
3904	*
3905	* But because grq->load.weight can drop to 0, resulting in a divide by zero,
3906	* we need to use grq->avg.load_avg as its lower bound, which then gives:
3907	*
3908	*
3909	* tg->weight * grq->load.weight
3910	* ge->load.weight = ----------------------------- (6)
3911	* tg_load_avg'
3912	*
3913	* Where:
3914	*
3915	* tg_load_avg' = tg->load_avg - grq->avg.load_avg +
3916	* max(grq->load.weight, grq->avg.load_avg)
3917	*
3918	* And that is shares_weight and is icky. In the (near) UP case it approaches
3919	* (4) while in the normal case it approaches (3). It consistently
3920	* overestimates the ge->load.weight and therefore:
3921	*
3922	* \Sum ge->load.weight >= tg->weight
3923	*
3924	* hence icky!
3925	*/
3926	static long calc_group_shares(struct cfs_rq *cfs_rq)
3927	{
3928	long tg_weight, tg_shares, load, shares;
3929	struct task_group *tg = cfs_rq->tg;
3930
3931	tg_shares = READ_ONCE(tg->shares);
3932
3933	load = max(scale_load_down(cfs_rq->load.weight), cfs_rq->avg.load_avg);
3934
3935	tg_weight = atomic_long_read(v: &tg->load_avg);
3936
3937	/* Ensure tg_weight >= load */
3938	tg_weight -= cfs_rq->tg_load_avg_contrib;
3939	tg_weight += load;
3940
3941	shares = (tg_shares * load);
3942	if (tg_weight)
3943	shares /= tg_weight;
3944
3945	/*
3946	* MIN_SHARES has to be unscaled here to support per-CPU partitioning
3947	* of a group with small tg->shares value. It is a floor value which is
3948	* assigned as a minimum load.weight to the sched_entity representing
3949	* the group on a CPU.
3950	*
3951	* E.g. on 64-bit for a group with tg->shares of scale_load(15)=15*1024
3952	* on an 8-core system with 8 tasks each runnable on one CPU shares has
3953	* to be 15*1024*1/8=1920 instead of scale_load(MIN_SHARES)=2*1024. In
3954	* case no task is runnable on a CPU MIN_SHARES=2 should be returned
3955	* instead of 0.
3956	*/
3957	return clamp_t(long, shares, MIN_SHARES, tg_shares);
3958	}
3959	#endif /* CONFIG_SMP */
3960
3961	/*
3962	* Recomputes the group entity based on the current state of its group
3963	* runqueue.
3964	*/
3965	static void update_cfs_group(struct sched_entity *se)
3966	{
3967	struct cfs_rq *gcfs_rq = group_cfs_rq(grp: se);
3968	long shares;
3969
3970	if (!gcfs_rq)
3971	return;
3972
3973	if (throttled_hierarchy(cfs_rq: gcfs_rq))
3974	return;
3975
3976	#ifndef CONFIG_SMP
3977	shares = READ_ONCE(gcfs_rq->tg->shares);
3978	#else
3979	shares = calc_group_shares(cfs_rq: gcfs_rq);
3980	#endif
3981	if (unlikely(se->load.weight != shares))
3982	reweight_entity(cfs_rq: cfs_rq_of(se), se, weight: shares);
3983	}
3984
3985	#else /* CONFIG_FAIR_GROUP_SCHED */
3986	static inline void update_cfs_group(struct sched_entity *se)
3987	{
3988	}
3989	#endif /* CONFIG_FAIR_GROUP_SCHED */
3990
3991	static inline void cfs_rq_util_change(struct cfs_rq *cfs_rq, int flags)
3992	{
3993	struct rq *rq = rq_of(cfs_rq);
3994
3995	if (&rq->cfs == cfs_rq) {
3996	/*
3997	* There are a few boundary cases this might miss but it should
3998	* get called often enough that that should (hopefully) not be
3999	* a real problem.
4000	*
4001	* It will not get called when we go idle, because the idle
4002	* thread is a different class (!fair), nor will the utilization
4003	* number include things like RT tasks.
4004	*
4005	* As is, the util number is not freq-invariant (we'd have to
4006	* implement arch_scale_freq_capacity() for that).
4007	*
4008	* See cpu_util_cfs().
4009	*/
4010	cpufreq_update_util(rq, flags);
4011	}
4012	}
4013
4014	#ifdef CONFIG_SMP
4015	static inline bool load_avg_is_decayed(struct sched_avg *sa)
4016	{
4017	if (sa->load_sum)
4018	return false;
4019
4020	if (sa->util_sum)
4021	return false;
4022
4023	if (sa->runnable_sum)
4024	return false;
4025
4026	/*
4027	* _avg must be null when _sum are null because _avg = _sum / divider
4028	* Make sure that rounding and/or propagation of PELT values never
4029	* break this.
4030	*/
4031	SCHED_WARN_ON(sa->load_avg ||
4032	sa->util_avg ||
4033	sa->runnable_avg);
4034
4035	return true;
4036	}
4037
4038	static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq)
4039	{
4040	return u64_u32_load_copy(cfs_rq->avg.last_update_time,
4041	cfs_rq->last_update_time_copy);
4042	}
4043	#ifdef CONFIG_FAIR_GROUP_SCHED
4044	/*
4045	* Because list_add_leaf_cfs_rq always places a child cfs_rq on the list
4046	* immediately before a parent cfs_rq, and cfs_rqs are removed from the list
4047	* bottom-up, we only have to test whether the cfs_rq before us on the list
4048	* is our child.
4049	* If cfs_rq is not on the list, test whether a child needs its to be added to
4050	* connect a branch to the tree * (see list_add_leaf_cfs_rq() for details).
4051	*/
4052	static inline bool child_cfs_rq_on_list(struct cfs_rq *cfs_rq)
4053	{
4054	struct cfs_rq *prev_cfs_rq;
4055	struct list_head *prev;
4056
4057	if (cfs_rq->on_list) {
4058	prev = cfs_rq->leaf_cfs_rq_list.prev;
4059	} else {
4060	struct rq *rq = rq_of(cfs_rq);
4061
4062	prev = rq->tmp_alone_branch;
4063	}
4064
4065	prev_cfs_rq = container_of(prev, struct cfs_rq, leaf_cfs_rq_list);
4066
4067	return (prev_cfs_rq->tg->parent == cfs_rq->tg);
4068	}
4069
4070	static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
4071	{
4072	if (cfs_rq->load.weight)
4073	return false;
4074
4075	if (!load_avg_is_decayed(sa: &cfs_rq->avg))
4076	return false;
4077
4078	if (child_cfs_rq_on_list(cfs_rq))
4079	return false;
4080
4081	return true;
4082	}
4083
4084	/**
4085	* update_tg_load_avg - update the tg's load avg
4086	* @cfs_rq: the cfs_rq whose avg changed
4087	*
4088	* This function 'ensures': tg->load_avg := \Sum tg->cfs_rq[]->avg.load.
4089	* However, because tg->load_avg is a global value there are performance
4090	* considerations.
4091	*
4092	* In order to avoid having to look at the other cfs_rq's, we use a
4093	* differential update where we store the last value we propagated. This in
4094	* turn allows skipping updates if the differential is 'small'.
4095	*
4096	* Updating tg's load_avg is necessary before update_cfs_share().
4097	*/
4098	static inline void update_tg_load_avg(struct cfs_rq *cfs_rq)
4099	{
4100	long delta;
4101	u64 now;
4102
4103	/*
4104	* No need to update load_avg for root_task_group as it is not used.
4105	*/
4106	if (cfs_rq->tg == &root_task_group)
4107	return;
4108
4109	/* rq has been offline and doesn't contribute to the share anymore: */
4110	if (!cpu_active(cpu: cpu_of(rq: rq_of(cfs_rq))))
4111	return;
4112
4113	/*
4114	* For migration heavy workloads, access to tg->load_avg can be
4115	* unbound. Limit the update rate to at most once per ms.
4116	*/
4117	now = sched_clock_cpu(cpu: cpu_of(rq: rq_of(cfs_rq)));
4118	if (now - cfs_rq->last_update_tg_load_avg < NSEC_PER_MSEC)
4119	return;
4120
4121	delta = cfs_rq->avg.load_avg - cfs_rq->tg_load_avg_contrib;
4122	if (abs(delta) > cfs_rq->tg_load_avg_contrib / 64) {
4123	atomic_long_add(i: delta, v: &cfs_rq->tg->load_avg);
4124	cfs_rq->tg_load_avg_contrib = cfs_rq->avg.load_avg;
4125	cfs_rq->last_update_tg_load_avg = now;
4126	}
4127	}
4128
4129	static inline void clear_tg_load_avg(struct cfs_rq *cfs_rq)
4130	{
4131	long delta;
4132	u64 now;
4133
4134	/*
4135	* No need to update load_avg for root_task_group, as it is not used.
4136	*/
4137	if (cfs_rq->tg == &root_task_group)
4138	return;
4139
4140	now = sched_clock_cpu(cpu: cpu_of(rq: rq_of(cfs_rq)));
4141	delta = 0 - cfs_rq->tg_load_avg_contrib;
4142	atomic_long_add(i: delta, v: &cfs_rq->tg->load_avg);
4143	cfs_rq->tg_load_avg_contrib = 0;
4144	cfs_rq->last_update_tg_load_avg = now;
4145	}
4146
4147	/* CPU offline callback: */
4148	static void __maybe_unused clear_tg_offline_cfs_rqs(struct rq *rq)
4149	{
4150	struct task_group *tg;
4151
4152	lockdep_assert_rq_held(rq);
4153
4154	/*
4155	* The rq clock has already been updated in
4156	* set_rq_offline(), so we should skip updating
4157	* the rq clock again in unthrottle_cfs_rq().
4158	*/
4159	rq_clock_start_loop_update(rq);
4160
4161	rcu_read_lock();
4162	list_for_each_entry_rcu(tg, &task_groups, list) {
4163	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
4164
4165	clear_tg_load_avg(cfs_rq);
4166	}
4167	rcu_read_unlock();
4168
4169	rq_clock_stop_loop_update(rq);
4170	}
4171
4172	/*
4173	* Called within set_task_rq() right before setting a task's CPU. The
4174	* caller only guarantees p->pi_lock is held; no other assumptions,
4175	* including the state of rq->lock, should be made.
4176	*/
4177	void set_task_rq_fair(struct sched_entity *se,
4178	struct cfs_rq *prev, struct cfs_rq *next)
4179	{
4180	u64 p_last_update_time;
4181	u64 n_last_update_time;
4182
4183	if (!sched_feat(ATTACH_AGE_LOAD))
4184	return;
4185
4186	/*
4187	* We are supposed to update the task to "current" time, then its up to
4188	* date and ready to go to new CPU/cfs_rq. But we have difficulty in
4189	* getting what current time is, so simply throw away the out-of-date
4190	* time. This will result in the wakee task is less decayed, but giving
4191	* the wakee more load sounds not bad.
4192	*/
4193	if (!(se->avg.last_update_time && prev))
4194	return;
4195
4196	p_last_update_time = cfs_rq_last_update_time(cfs_rq: prev);
4197	n_last_update_time = cfs_rq_last_update_time(cfs_rq: next);
4198
4199	__update_load_avg_blocked_se(now: p_last_update_time, se);
4200	se->avg.last_update_time = n_last_update_time;
4201	}
4202
4203	/*
4204	* When on migration a sched_entity joins/leaves the PELT hierarchy, we need to
4205	* propagate its contribution. The key to this propagation is the invariant
4206	* that for each group:
4207	*
4208	* ge->avg == grq->avg (1)
4209	*
4210	* _IFF_ we look at the pure running and runnable sums. Because they
4211	* represent the very same entity, just at different points in the hierarchy.
4212	*
4213	* Per the above update_tg_cfs_util() and update_tg_cfs_runnable() are trivial
4214	* and simply copies the running/runnable sum over (but still wrong, because
4215	* the group entity and group rq do not have their PELT windows aligned).
4216	*
4217	* However, update_tg_cfs_load() is more complex. So we have:
4218	*
4219	* ge->avg.load_avg = ge->load.weight * ge->avg.runnable_avg (2)
4220	*
4221	* And since, like util, the runnable part should be directly transferable,
4222	* the following would _appear_ to be the straight forward approach:
4223	*
4224	* grq->avg.load_avg = grq->load.weight * grq->avg.runnable_avg (3)
4225	*
4226	* And per (1) we have:
4227	*
4228	* ge->avg.runnable_avg == grq->avg.runnable_avg
4229	*
4230	* Which gives:
4231	*
4232	* ge->load.weight * grq->avg.load_avg
4233	* ge->avg.load_avg = ----------------------------------- (4)
4234	* grq->load.weight
4235	*
4236	* Except that is wrong!
4237	*
4238	* Because while for entities historical weight is not important and we
4239	* really only care about our future and therefore can consider a pure
4240	* runnable sum, runqueues can NOT do this.
4241	*
4242	* We specifically want runqueues to have a load_avg that includes
4243	* historical weights. Those represent the blocked load, the load we expect
4244	* to (shortly) return to us. This only works by keeping the weights as
4245	* integral part of the sum. We therefore cannot decompose as per (3).
4246	*
4247	* Another reason this doesn't work is that runnable isn't a 0-sum entity.
4248	* Imagine a rq with 2 tasks that each are runnable 2/3 of the time. Then the
4249	* rq itself is runnable anywhere between 2/3 and 1 depending on how the
4250	* runnable section of these tasks overlap (or not). If they were to perfectly
4251	* align the rq as a whole would be runnable 2/3 of the time. If however we
4252	* always have at least 1 runnable task, the rq as a whole is always runnable.
4253	*
4254	* So we'll have to approximate.. :/
4255	*
4256	* Given the constraint:
4257	*
4258	* ge->avg.running_sum <= ge->avg.runnable_sum <= LOAD_AVG_MAX
4259	*
4260	* We can construct a rule that adds runnable to a rq by assuming minimal
4261	* overlap.
4262	*
4263	* On removal, we'll assume each task is equally runnable; which yields:
4264	*
4265	* grq->avg.runnable_sum = grq->avg.load_sum / grq->load.weight
4266	*
4267	* XXX: only do this for the part of runnable > running ?
4268	*
4269	*/
4270	static inline void
4271	update_tg_cfs_util(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
4272	{
4273	long delta_sum, delta_avg = gcfs_rq->avg.util_avg - se->avg.util_avg;
4274	u32 new_sum, divider;
4275
4276	/* Nothing to update */
4277	if (!delta_avg)
4278	return;
4279
4280	/*
4281	* cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
4282	* See ___update_load_avg() for details.
4283	*/
4284	divider = get_pelt_divider(avg: &cfs_rq->avg);
4285
4286
4287	/* Set new sched_entity's utilization */
4288	se->avg.util_avg = gcfs_rq->avg.util_avg;
4289	new_sum = se->avg.util_avg * divider;
4290	delta_sum = (long)new_sum - (long)se->avg.util_sum;
4291	se->avg.util_sum = new_sum;
4292
4293	/* Update parent cfs_rq utilization */
4294	add_positive(&cfs_rq->avg.util_avg, delta_avg);
4295	add_positive(&cfs_rq->avg.util_sum, delta_sum);
4296
4297	/* See update_cfs_rq_load_avg() */
4298	cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum,
4299	cfs_rq->avg.util_avg * PELT_MIN_DIVIDER);
4300	}
4301
4302	static inline void
4303	update_tg_cfs_runnable(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
4304	{
4305	long delta_sum, delta_avg = gcfs_rq->avg.runnable_avg - se->avg.runnable_avg;
4306	u32 new_sum, divider;
4307
4308	/* Nothing to update */
4309	if (!delta_avg)
4310	return;
4311
4312	/*
4313	* cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
4314	* See ___update_load_avg() for details.
4315	*/
4316	divider = get_pelt_divider(avg: &cfs_rq->avg);
4317
4318	/* Set new sched_entity's runnable */
4319	se->avg.runnable_avg = gcfs_rq->avg.runnable_avg;
4320	new_sum = se->avg.runnable_avg * divider;
4321	delta_sum = (long)new_sum - (long)se->avg.runnable_sum;
4322	se->avg.runnable_sum = new_sum;
4323
4324	/* Update parent cfs_rq runnable */
4325	add_positive(&cfs_rq->avg.runnable_avg, delta_avg);
4326	add_positive(&cfs_rq->avg.runnable_sum, delta_sum);
4327	/* See update_cfs_rq_load_avg() */
4328	cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum,
4329	cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER);
4330	}
4331
4332	static inline void
4333	update_tg_cfs_load(struct cfs_rq *cfs_rq, struct sched_entity *se, struct cfs_rq *gcfs_rq)
4334	{
4335	long delta_avg, running_sum, runnable_sum = gcfs_rq->prop_runnable_sum;
4336	unsigned long load_avg;
4337	u64 load_sum = 0;
4338	s64 delta_sum;
4339	u32 divider;
4340
4341	if (!runnable_sum)
4342	return;
4343
4344	gcfs_rq->prop_runnable_sum = 0;
4345
4346	/*
4347	* cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
4348	* See ___update_load_avg() for details.
4349	*/
4350	divider = get_pelt_divider(avg: &cfs_rq->avg);
4351
4352	if (runnable_sum >= 0) {
4353	/*
4354	* Add runnable; clip at LOAD_AVG_MAX. Reflects that until
4355	* the CPU is saturated running == runnable.
4356	*/
4357	runnable_sum += se->avg.load_sum;
4358	runnable_sum = min_t(long, runnable_sum, divider);
4359	} else {
4360	/*
4361	* Estimate the new unweighted runnable_sum of the gcfs_rq by
4362	* assuming all tasks are equally runnable.
4363	*/
4364	if (scale_load_down(gcfs_rq->load.weight)) {
4365	load_sum = div_u64(dividend: gcfs_rq->avg.load_sum,
4366	scale_load_down(gcfs_rq->load.weight));
4367	}
4368
4369	/* But make sure to not inflate se's runnable */
4370	runnable_sum = min(se->avg.load_sum, load_sum);
4371	}
4372
4373	/*
4374	* runnable_sum can't be lower than running_sum
4375	* Rescale running sum to be in the same range as runnable sum
4376	* running_sum is in [0 : LOAD_AVG_MAX << SCHED_CAPACITY_SHIFT]
4377	* runnable_sum is in [0 : LOAD_AVG_MAX]
4378	*/
4379	running_sum = se->avg.util_sum >> SCHED_CAPACITY_SHIFT;
4380	runnable_sum = max(runnable_sum, running_sum);
4381
4382	load_sum = se_weight(se) * runnable_sum;
4383	load_avg = div_u64(dividend: load_sum, divisor: divider);
4384
4385	delta_avg = load_avg - se->avg.load_avg;
4386	if (!delta_avg)
4387	return;
4388
4389	delta_sum = load_sum - (s64)se_weight(se) * se->avg.load_sum;
4390
4391	se->avg.load_sum = runnable_sum;
4392	se->avg.load_avg = load_avg;
4393	add_positive(&cfs_rq->avg.load_avg, delta_avg);
4394	add_positive(&cfs_rq->avg.load_sum, delta_sum);
4395	/* See update_cfs_rq_load_avg() */
4396	cfs_rq->avg.load_sum = max_t(u32, cfs_rq->avg.load_sum,
4397	cfs_rq->avg.load_avg * PELT_MIN_DIVIDER);
4398	}
4399
4400	static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum)
4401	{
4402	cfs_rq->propagate = 1;
4403	cfs_rq->prop_runnable_sum += runnable_sum;
4404	}
4405
4406	/* Update task and its cfs_rq load average */
4407	static inline int propagate_entity_load_avg(struct sched_entity *se)
4408	{
4409	struct cfs_rq *cfs_rq, *gcfs_rq;
4410
4411	if (entity_is_task(se))
4412	return 0;
4413
4414	gcfs_rq = group_cfs_rq(grp: se);
4415	if (!gcfs_rq->propagate)
4416	return 0;
4417
4418	gcfs_rq->propagate = 0;
4419
4420	cfs_rq = cfs_rq_of(se);
4421
4422	add_tg_cfs_propagate(cfs_rq, runnable_sum: gcfs_rq->prop_runnable_sum);
4423
4424	update_tg_cfs_util(cfs_rq, se, gcfs_rq);
4425	update_tg_cfs_runnable(cfs_rq, se, gcfs_rq);
4426	update_tg_cfs_load(cfs_rq, se, gcfs_rq);
4427
4428	trace_pelt_cfs_tp(cfs_rq);
4429	trace_pelt_se_tp(se);
4430
4431	return 1;
4432	}
4433
4434	/*
4435	* Check if we need to update the load and the utilization of a blocked
4436	* group_entity:
4437	*/
4438	static inline bool skip_blocked_update(struct sched_entity *se)
4439	{
4440	struct cfs_rq *gcfs_rq = group_cfs_rq(grp: se);
4441
4442	/*
4443	* If sched_entity still have not zero load or utilization, we have to
4444	* decay it:
4445	*/
4446	if (se->avg.load_avg || se->avg.util_avg)
4447	return false;
4448
4449	/*
4450	* If there is a pending propagation, we have to update the load and
4451	* the utilization of the sched_entity:
4452	*/
4453	if (gcfs_rq->propagate)
4454	return false;
4455
4456	/*
4457	* Otherwise, the load and the utilization of the sched_entity is
4458	* already zero and there is no pending propagation, so it will be a
4459	* waste of time to try to decay it:
4460	*/
4461	return true;
4462	}
4463
4464	#else /* CONFIG_FAIR_GROUP_SCHED */
4465
4466	static inline void update_tg_load_avg(struct cfs_rq *cfs_rq) {}
4467
4468	static inline void clear_tg_offline_cfs_rqs(struct rq *rq) {}
4469
4470	static inline int propagate_entity_load_avg(struct sched_entity *se)
4471	{
4472	return 0;
4473	}
4474
4475	static inline void add_tg_cfs_propagate(struct cfs_rq *cfs_rq, long runnable_sum) {}
4476
4477	#endif /* CONFIG_FAIR_GROUP_SCHED */
4478
4479	#ifdef CONFIG_NO_HZ_COMMON
4480	static inline void migrate_se_pelt_lag(struct sched_entity *se)
4481	{
4482	u64 throttled = 0, now, lut;
4483	struct cfs_rq *cfs_rq;
4484	struct rq *rq;
4485	bool is_idle;
4486
4487	if (load_avg_is_decayed(sa: &se->avg))
4488	return;
4489
4490	cfs_rq = cfs_rq_of(se);
4491	rq = rq_of(cfs_rq);
4492
4493	rcu_read_lock();
4494	is_idle = is_idle_task(rcu_dereference(rq->curr));
4495	rcu_read_unlock();
4496
4497	/*
4498	* The lag estimation comes with a cost we don't want to pay all the
4499	* time. Hence, limiting to the case where the source CPU is idle and
4500	* we know we are at the greatest risk to have an outdated clock.
4501	*/
4502	if (!is_idle)
4503	return;
4504
4505	/*
4506	* Estimated "now" is: last_update_time + cfs_idle_lag + rq_idle_lag, where:
4507	*
4508	* last_update_time (the cfs_rq's last_update_time)
4509	* = cfs_rq_clock_pelt()@cfs_rq_idle
4510	* = rq_clock_pelt()@cfs_rq_idle
4511	* - cfs->throttled_clock_pelt_time@cfs_rq_idle
4512	*
4513	* cfs_idle_lag (delta between rq's update and cfs_rq's update)
4514	* = rq_clock_pelt()@rq_idle - rq_clock_pelt()@cfs_rq_idle
4515	*
4516	* rq_idle_lag (delta between now and rq's update)
4517	* = sched_clock_cpu() - rq_clock()@rq_idle
4518	*
4519	* We can then write:
4520	*
4521	* now = rq_clock_pelt()@rq_idle - cfs->throttled_clock_pelt_time +
4522	* sched_clock_cpu() - rq_clock()@rq_idle
4523	* Where:
4524	* rq_clock_pelt()@rq_idle is rq->clock_pelt_idle
4525	* rq_clock()@rq_idle is rq->clock_idle
4526	* cfs->throttled_clock_pelt_time@cfs_rq_idle
4527	* is cfs_rq->throttled_pelt_idle
4528	*/
4529
4530	#ifdef CONFIG_CFS_BANDWIDTH
4531	throttled = u64_u32_load(cfs_rq->throttled_pelt_idle);
4532	/* The clock has been stopped for throttling */
4533	if (throttled == U64_MAX)
4534	return;
4535	#endif
4536	now = u64_u32_load(rq->clock_pelt_idle);
4537	/*
4538	* Paired with _update_idle_rq_clock_pelt(). It ensures at the worst case
4539	* is observed the old clock_pelt_idle value and the new clock_idle,
4540	* which lead to an underestimation. The opposite would lead to an
4541	* overestimation.
4542	*/
4543	smp_rmb();
4544	lut = cfs_rq_last_update_time(cfs_rq);
4545
4546	now -= throttled;
4547	if (now < lut)
4548	/*
4549	* cfs_rq->avg.last_update_time is more recent than our
4550	* estimation, let's use it.
4551	*/
4552	now = lut;
4553	else
4554	now += sched_clock_cpu(cpu: cpu_of(rq)) - u64_u32_load(rq->clock_idle);
4555
4556	__update_load_avg_blocked_se(now, se);
4557	}
4558	#else
4559	static void migrate_se_pelt_lag(struct sched_entity *se) {}
4560	#endif
4561
4562	/**
4563	* update_cfs_rq_load_avg - update the cfs_rq's load/util averages
4564	* @now: current time, as per cfs_rq_clock_pelt()
4565	* @cfs_rq: cfs_rq to update
4566	*
4567	* The cfs_rq avg is the direct sum of all its entities (blocked and runnable)
4568	* avg. The immediate corollary is that all (fair) tasks must be attached.
4569	*
4570	* cfs_rq->avg is used for task_h_load() and update_cfs_share() for example.
4571	*
4572	* Return: true if the load decayed or we removed load.
4573	*
4574	* Since both these conditions indicate a changed cfs_rq->avg.load we should
4575	* call update_tg_load_avg() when this function returns true.
4576	*/
4577	static inline int
4578	update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq)
4579	{
4580	unsigned long removed_load = 0, removed_util = 0, removed_runnable = 0;
4581	struct sched_avg *sa = &cfs_rq->avg;
4582	int decayed = 0;
4583
4584	if (cfs_rq->removed.nr) {
4585	unsigned long r;
4586	u32 divider = get_pelt_divider(avg: &cfs_rq->avg);
4587
4588	raw_spin_lock(&cfs_rq->removed.lock);
4589	swap(cfs_rq->removed.util_avg, removed_util);
4590	swap(cfs_rq->removed.load_avg, removed_load);
4591	swap(cfs_rq->removed.runnable_avg, removed_runnable);
4592	cfs_rq->removed.nr = 0;
4593	raw_spin_unlock(&cfs_rq->removed.lock);
4594
4595	r = removed_load;
4596	sub_positive(&sa->load_avg, r);
4597	sub_positive(&sa->load_sum, r * divider);
4598	/* See sa->util_sum below */
4599	sa->load_sum = max_t(u32, sa->load_sum, sa->load_avg * PELT_MIN_DIVIDER);
4600
4601	r = removed_util;
4602	sub_positive(&sa->util_avg, r);
4603	sub_positive(&sa->util_sum, r * divider);
4604	/*
4605	* Because of rounding, se->util_sum might ends up being +1 more than
4606	* cfs->util_sum. Although this is not a problem by itself, detaching
4607	* a lot of tasks with the rounding problem between 2 updates of
4608	* util_avg (~1ms) can make cfs->util_sum becoming null whereas
4609	* cfs_util_avg is not.
4610	* Check that util_sum is still above its lower bound for the new
4611	* util_avg. Given that period_contrib might have moved since the last
4612	* sync, we are only sure that util_sum must be above or equal to
4613	* util_avg * minimum possible divider
4614	*/
4615	sa->util_sum = max_t(u32, sa->util_sum, sa->util_avg * PELT_MIN_DIVIDER);
4616
4617	r = removed_runnable;
4618	sub_positive(&sa->runnable_avg, r);
4619	sub_positive(&sa->runnable_sum, r * divider);
4620	/* See sa->util_sum above */
4621	sa->runnable_sum = max_t(u32, sa->runnable_sum,
4622	sa->runnable_avg * PELT_MIN_DIVIDER);
4623
4624	/*
4625	* removed_runnable is the unweighted version of removed_load so we
4626	* can use it to estimate removed_load_sum.
4627	*/
4628	add_tg_cfs_propagate(cfs_rq,
4629	runnable_sum: -(long)(removed_runnable * divider) >> SCHED_CAPACITY_SHIFT);
4630
4631	decayed = 1;
4632	}
4633
4634	decayed |= __update_load_avg_cfs_rq(now, cfs_rq);
4635	u64_u32_store_copy(sa->last_update_time,
4636	cfs_rq->last_update_time_copy,
4637	sa->last_update_time);
4638	return decayed;
4639	}
4640
4641	/**
4642	* attach_entity_load_avg - attach this entity to its cfs_rq load avg
4643	* @cfs_rq: cfs_rq to attach to
4644	* @se: sched_entity to attach
4645	*
4646	* Must call update_cfs_rq_load_avg() before this, since we rely on
4647	* cfs_rq->avg.last_update_time being current.
4648	*/
4649	static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
4650	{
4651	/*
4652	* cfs_rq->avg.period_contrib can be used for both cfs_rq and se.
4653	* See ___update_load_avg() for details.
4654	*/
4655	u32 divider = get_pelt_divider(avg: &cfs_rq->avg);
4656
4657	/*
4658	* When we attach the @se to the @cfs_rq, we must align the decay
4659	* window because without that, really weird and wonderful things can
4660	* happen.
4661	*
4662	* XXX illustrate
4663	*/
4664	se->avg.last_update_time = cfs_rq->avg.last_update_time;
4665	se->avg.period_contrib = cfs_rq->avg.period_contrib;
4666
4667	/*
4668	* Hell(o) Nasty stuff.. we need to recompute _sum based on the new
4669	* period_contrib. This isn't strictly correct, but since we're
4670	* entirely outside of the PELT hierarchy, nobody cares if we truncate
4671	* _sum a little.
4672	*/
4673	se->avg.util_sum = se->avg.util_avg * divider;
4674
4675	se->avg.runnable_sum = se->avg.runnable_avg * divider;
4676
4677	se->avg.load_sum = se->avg.load_avg * divider;
4678	if (se_weight(se) < se->avg.load_sum)
4679	se->avg.load_sum = div_u64(dividend: se->avg.load_sum, divisor: se_weight(se));
4680	else
4681	se->avg.load_sum = 1;
4682
4683	enqueue_load_avg(cfs_rq, se);
4684	cfs_rq->avg.util_avg += se->avg.util_avg;
4685	cfs_rq->avg.util_sum += se->avg.util_sum;
4686	cfs_rq->avg.runnable_avg += se->avg.runnable_avg;
4687	cfs_rq->avg.runnable_sum += se->avg.runnable_sum;
4688
4689	add_tg_cfs_propagate(cfs_rq, runnable_sum: se->avg.load_sum);
4690
4691	cfs_rq_util_change(cfs_rq, flags: 0);
4692
4693	trace_pelt_cfs_tp(cfs_rq);
4694	}
4695
4696	/**
4697	* detach_entity_load_avg - detach this entity from its cfs_rq load avg
4698	* @cfs_rq: cfs_rq to detach from
4699	* @se: sched_entity to detach
4700	*
4701	* Must call update_cfs_rq_load_avg() before this, since we rely on
4702	* cfs_rq->avg.last_update_time being current.
4703	*/
4704	static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se)
4705	{
4706	dequeue_load_avg(cfs_rq, se);
4707	sub_positive(&cfs_rq->avg.util_avg, se->avg.util_avg);
4708	sub_positive(&cfs_rq->avg.util_sum, se->avg.util_sum);
4709	/* See update_cfs_rq_load_avg() */
4710	cfs_rq->avg.util_sum = max_t(u32, cfs_rq->avg.util_sum,
4711	cfs_rq->avg.util_avg * PELT_MIN_DIVIDER);
4712
4713	sub_positive(&cfs_rq->avg.runnable_avg, se->avg.runnable_avg);
4714	sub_positive(&cfs_rq->avg.runnable_sum, se->avg.runnable_sum);
4715	/* See update_cfs_rq_load_avg() */
4716	cfs_rq->avg.runnable_sum = max_t(u32, cfs_rq->avg.runnable_sum,
4717	cfs_rq->avg.runnable_avg * PELT_MIN_DIVIDER);
4718
4719	add_tg_cfs_propagate(cfs_rq, runnable_sum: -se->avg.load_sum);
4720
4721	cfs_rq_util_change(cfs_rq, flags: 0);
4722
4723	trace_pelt_cfs_tp(cfs_rq);
4724	}
4725
4726	/*
4727	* Optional action to be done while updating the load average
4728	*/
4729	#define UPDATE_TG 0x1
4730	#define SKIP_AGE_LOAD 0x2
4731	#define DO_ATTACH 0x4
4732	#define DO_DETACH 0x8
4733
4734	/* Update task and its cfs_rq load average */
4735	static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
4736	{
4737	u64 now = cfs_rq_clock_pelt(cfs_rq);
4738	int decayed;
4739
4740	/*
4741	* Track task load average for carrying it to new CPU after migrated, and
4742	* track group sched_entity load average for task_h_load calc in migration
4743	*/
4744	if (se->avg.last_update_time && !(flags & SKIP_AGE_LOAD))
4745	__update_load_avg_se(now, cfs_rq, se);
4746
4747	decayed = update_cfs_rq_load_avg(now, cfs_rq);
4748	decayed |= propagate_entity_load_avg(se);
4749
4750	if (!se->avg.last_update_time && (flags & DO_ATTACH)) {
4751
4752	/*
4753	* DO_ATTACH means we're here from enqueue_entity().
4754	* !last_update_time means we've passed through
4755	* migrate_task_rq_fair() indicating we migrated.
4756	*
4757	* IOW we're enqueueing a task on a new CPU.
4758	*/
4759	attach_entity_load_avg(cfs_rq, se);
4760	update_tg_load_avg(cfs_rq);
4761
4762	} else if (flags & DO_DETACH) {
4763	/*
4764	* DO_DETACH means we're here from dequeue_entity()
4765	* and we are migrating task out of the CPU.
4766	*/
4767	detach_entity_load_avg(cfs_rq, se);
4768	update_tg_load_avg(cfs_rq);
4769	} else if (decayed) {
4770	cfs_rq_util_change(cfs_rq, flags: 0);
4771
4772	if (flags & UPDATE_TG)
4773	update_tg_load_avg(cfs_rq);
4774	}
4775	}
4776
4777	/*
4778	* Synchronize entity load avg of dequeued entity without locking
4779	* the previous rq.
4780	*/
4781	static void sync_entity_load_avg(struct sched_entity *se)
4782	{
4783	struct cfs_rq *cfs_rq = cfs_rq_of(se);
4784	u64 last_update_time;
4785
4786	last_update_time = cfs_rq_last_update_time(cfs_rq);
4787	__update_load_avg_blocked_se(now: last_update_time, se);
4788	}
4789
4790	/*
4791	* Task first catches up with cfs_rq, and then subtract
4792	* itself from the cfs_rq (task must be off the queue now).
4793	*/
4794	static void remove_entity_load_avg(struct sched_entity *se)
4795	{
4796	struct cfs_rq *cfs_rq = cfs_rq_of(se);
4797	unsigned long flags;
4798
4799	/*
4800	* tasks cannot exit without having gone through wake_up_new_task() ->
4801	* enqueue_task_fair() which will have added things to the cfs_rq,
4802	* so we can remove unconditionally.
4803	*/
4804
4805	sync_entity_load_avg(se);
4806
4807	raw_spin_lock_irqsave(&cfs_rq->removed.lock, flags);
4808	++cfs_rq->removed.nr;
4809	cfs_rq->removed.util_avg += se->avg.util_avg;
4810	cfs_rq->removed.load_avg += se->avg.load_avg;
4811	cfs_rq->removed.runnable_avg += se->avg.runnable_avg;
4812	raw_spin_unlock_irqrestore(&cfs_rq->removed.lock, flags);
4813	}
4814
4815	static inline unsigned long cfs_rq_runnable_avg(struct cfs_rq *cfs_rq)
4816	{
4817	return cfs_rq->avg.runnable_avg;
4818	}
4819
4820	static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq)
4821	{
4822	return cfs_rq->avg.load_avg;
4823	}
4824
4825	static int newidle_balance(struct rq *this_rq, struct rq_flags *rf);
4826
4827	static inline unsigned long task_util(struct task_struct *p)
4828	{
4829	return READ_ONCE(p->se.avg.util_avg);
4830	}
4831
4832	static inline unsigned long task_runnable(struct task_struct *p)
4833	{
4834	return READ_ONCE(p->se.avg.runnable_avg);
4835	}
4836
4837	static inline unsigned long _task_util_est(struct task_struct *p)
4838	{
4839	return READ_ONCE(p->se.avg.util_est) & ~UTIL_AVG_UNCHANGED;
4840	}
4841
4842	static inline unsigned long task_util_est(struct task_struct *p)
4843	{
4844	return max(task_util(p), _task_util_est(p));
4845	}
4846
4847	static inline void util_est_enqueue(struct cfs_rq *cfs_rq,
4848	struct task_struct *p)
4849	{
4850	unsigned int enqueued;
4851
4852	if (!sched_feat(UTIL_EST))
4853	return;
4854
4855	/* Update root cfs_rq's estimated utilization */
4856	enqueued = cfs_rq->avg.util_est;
4857	enqueued += _task_util_est(p);
4858	WRITE_ONCE(cfs_rq->avg.util_est, enqueued);
4859
4860	trace_sched_util_est_cfs_tp(cfs_rq);
4861	}
4862
4863	static inline void util_est_dequeue(struct cfs_rq *cfs_rq,
4864	struct task_struct *p)
4865	{
4866	unsigned int enqueued;
4867
4868	if (!sched_feat(UTIL_EST))
4869	return;
4870
4871	/* Update root cfs_rq's estimated utilization */
4872	enqueued = cfs_rq->avg.util_est;
4873	enqueued -= min_t(unsigned int, enqueued, _task_util_est(p));
4874	WRITE_ONCE(cfs_rq->avg.util_est, enqueued);
4875
4876	trace_sched_util_est_cfs_tp(cfs_rq);
4877	}
4878
4879	#define UTIL_EST_MARGIN (SCHED_CAPACITY_SCALE / 100)
4880
4881	static inline void util_est_update(struct cfs_rq *cfs_rq,
4882	struct task_struct *p,
4883	bool task_sleep)
4884	{
4885	unsigned int ewma, dequeued, last_ewma_diff;
4886
4887	if (!sched_feat(UTIL_EST))
4888	return;
4889
4890	/*
4891	* Skip update of task's estimated utilization when the task has not
4892	* yet completed an activation, e.g. being migrated.
4893	*/
4894	if (!task_sleep)
4895	return;
4896
4897	/* Get current estimate of utilization */
4898	ewma = READ_ONCE(p->se.avg.util_est);
4899
4900	/*
4901	* If the PELT values haven't changed since enqueue time,
4902	* skip the util_est update.
4903	*/
4904	if (ewma & UTIL_AVG_UNCHANGED)
4905	return;
4906
4907	/* Get utilization at dequeue */
4908	dequeued = task_util(p);
4909
4910	/*
4911	* Reset EWMA on utilization increases, the moving average is used only
4912	* to smooth utilization decreases.
4913	*/
4914	if (ewma <= dequeued) {
4915	ewma = dequeued;
4916	goto done;
4917	}
4918
4919	/*
4920	* Skip update of task's estimated utilization when its members are
4921	* already ~1% close to its last activation value.
4922	*/
4923	last_ewma_diff = ewma - dequeued;
4924	if (last_ewma_diff < UTIL_EST_MARGIN)
4925	goto done;
4926
4927	/*
4928	* To avoid overestimation of actual task utilization, skip updates if
4929	* we cannot grant there is idle time in this CPU.
4930	*/
4931	if (dequeued > arch_scale_cpu_capacity(cpu: cpu_of(rq: rq_of(cfs_rq))))
4932	return;
4933
4934	/*
4935	* To avoid underestimate of task utilization, skip updates of EWMA if
4936	* we cannot grant that thread got all CPU time it wanted.
4937	*/
4938	if ((dequeued + UTIL_EST_MARGIN) < task_runnable(p))
4939	goto done;
4940
4941
4942	/*
4943	* Update Task's estimated utilization
4944	*
4945	* When *p completes an activation we can consolidate another sample
4946	* of the task size. This is done by using this value to update the
4947	* Exponential Weighted Moving Average (EWMA):
4948	*
4949	* ewma(t) = w * task_util(p) + (1-w) * ewma(t-1)
4950	* = w * task_util(p) + ewma(t-1) - w * ewma(t-1)
4951	* = w * (task_util(p) - ewma(t-1)) + ewma(t-1)
4952	* = w * ( -last_ewma_diff ) + ewma(t-1)
4953	* = w * (-last_ewma_diff + ewma(t-1) / w)
4954	*
4955	* Where 'w' is the weight of new samples, which is configured to be
4956	* 0.25, thus making w=1/4 ( >>= UTIL_EST_WEIGHT_SHIFT)
4957	*/
4958	ewma <<= UTIL_EST_WEIGHT_SHIFT;
4959	ewma -= last_ewma_diff;
4960	ewma >>= UTIL_EST_WEIGHT_SHIFT;
4961	done:
4962	ewma |= UTIL_AVG_UNCHANGED;
4963	WRITE_ONCE(p->se.avg.util_est, ewma);
4964
4965	trace_sched_util_est_se_tp(se: &p->se);
4966	}
4967
4968	static inline int util_fits_cpu(unsigned long util,
4969	unsigned long uclamp_min,
4970	unsigned long uclamp_max,
4971	int cpu)
4972	{
4973	unsigned long capacity_orig, capacity_orig_thermal;
4974	unsigned long capacity = capacity_of(cpu);
4975	bool fits, uclamp_max_fits;
4976
4977	/*
4978	* Check if the real util fits without any uclamp boost/cap applied.
4979	*/
4980	fits = fits_capacity(util, capacity);
4981
4982	if (!uclamp_is_used())
4983	return fits;
4984
4985	/*
4986	* We must use arch_scale_cpu_capacity() for comparing against uclamp_min and
4987	* uclamp_max. We only care about capacity pressure (by using
4988	* capacity_of()) for comparing against the real util.
4989	*
4990	* If a task is boosted to 1024 for example, we don't want a tiny
4991	* pressure to skew the check whether it fits a CPU or not.
4992	*
4993	* Similarly if a task is capped to arch_scale_cpu_capacity(little_cpu), it
4994	* should fit a little cpu even if there's some pressure.
4995	*
4996	* Only exception is for thermal pressure since it has a direct impact
4997	* on available OPP of the system.
4998	*
4999	* We honour it for uclamp_min only as a drop in performance level
5000	* could result in not getting the requested minimum performance level.
5001	*
5002	* For uclamp_max, we can tolerate a drop in performance level as the
5003	* goal is to cap the task. So it's okay if it's getting less.
5004	*/
5005	capacity_orig = arch_scale_cpu_capacity(cpu);
5006	capacity_orig_thermal = capacity_orig - arch_scale_thermal_pressure(cpu);
5007
5008	/*
5009	* We want to force a task to fit a cpu as implied by uclamp_max.
5010	* But we do have some corner cases to cater for..
5011	*
5012	*
5013	* C=z
5014	* | ___
5015	* | C=y | |
5016	* |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max
5017	* | C=x | | | |
5018	* | ___ | | | |
5019	* | | | | | | | (util somewhere in this region)
5020	* | | | | | | |
5021	* | | | | | | |
5022	* +----------------------------------------
5023	* cpu0 cpu1 cpu2
5024	*
5025	* In the above example if a task is capped to a specific performance
5026	* point, y, then when:
5027	*
5028	* * util = 80% of x then it does not fit on cpu0 and should migrate
5029	* to cpu1
5030	* * util = 80% of y then it is forced to fit on cpu1 to honour
5031	* uclamp_max request.
5032	*
5033	* which is what we're enforcing here. A task always fits if
5034	* uclamp_max <= capacity_orig. But when uclamp_max > capacity_orig,
5035	* the normal upmigration rules should withhold still.
5036	*
5037	* Only exception is when we are on max capacity, then we need to be
5038	* careful not to block overutilized state. This is so because:
5039	*
5040	* 1. There's no concept of capping at max_capacity! We can't go
5041	* beyond this performance level anyway.
5042	* 2. The system is being saturated when we're operating near
5043	* max capacity, it doesn't make sense to block overutilized.
5044	*/
5045	uclamp_max_fits = (capacity_orig == SCHED_CAPACITY_SCALE) && (uclamp_max == SCHED_CAPACITY_SCALE);
5046	uclamp_max_fits = !uclamp_max_fits && (uclamp_max <= capacity_orig);
5047	fits = fits || uclamp_max_fits;
5048
5049	/*
5050	*
5051	* C=z
5052	* | ___ (region a, capped, util >= uclamp_max)
5053	* | C=y | |
5054	* |_ _ _ _ _ _ _ _ _ ___ _ _ _ | _ | _ _ _ _ _ uclamp_max
5055	* | C=x | | | |
5056	* | ___ | | | | (region b, uclamp_min <= util <= uclamp_max)
5057	* |_ _ _|_ _|_ _ _ _| _ | _ _ _| _ | _ _ _ _ _ uclamp_min
5058	* | | | | | | |
5059	* | | | | | | | (region c, boosted, util < uclamp_min)
5060	* +----------------------------------------
5061	* cpu0 cpu1 cpu2
5062	*
5063	* a) If util > uclamp_max, then we're capped, we don't care about
5064	* actual fitness value here. We only care if uclamp_max fits
5065	* capacity without taking margin/pressure into account.
5066	* See comment above.
5067	*
5068	* b) If uclamp_min <= util <= uclamp_max, then the normal
5069	* fits_capacity() rules apply. Except we need to ensure that we
5070	* enforce we remain within uclamp_max, see comment above.
5071	*
5072	* c) If util < uclamp_min, then we are boosted. Same as (b) but we
5073	* need to take into account the boosted value fits the CPU without
5074	* taking margin/pressure into account.
5075	*
5076	* Cases (a) and (b) are handled in the 'fits' variable already. We
5077	* just need to consider an extra check for case (c) after ensuring we
5078	* handle the case uclamp_min > uclamp_max.
5079	*/
5080	uclamp_min = min(uclamp_min, uclamp_max);
5081	if (fits && (util < uclamp_min) && (uclamp_min > capacity_orig_thermal))
5082	return -1;
5083
5084	return fits;
5085	}
5086
5087	static inline int task_fits_cpu(struct task_struct *p, int cpu)
5088	{
5089	unsigned long uclamp_min = uclamp_eff_value(p, clamp_id: UCLAMP_MIN);
5090	unsigned long uclamp_max = uclamp_eff_value(p, clamp_id: UCLAMP_MAX);
5091	unsigned long util = task_util_est(p);
5092	/*
5093	* Return true only if the cpu fully fits the task requirements, which
5094	* include the utilization but also the performance hints.
5095	*/
5096	return (util_fits_cpu(util, uclamp_min, uclamp_max, cpu) > 0);
5097	}
5098
5099	static inline void update_misfit_status(struct task_struct *p, struct rq *rq)
5100	{
5101	if (!sched_asym_cpucap_active())
5102	return;
5103
5104	if (!p || p->nr_cpus_allowed == 1) {
5105	rq->misfit_task_load = 0;
5106	return;
5107	}
5108
5109	if (task_fits_cpu(p, cpu: cpu_of(rq))) {
5110	rq->misfit_task_load = 0;
5111	return;
5112	}
5113
5114	/*
5115	* Make sure that misfit_task_load will not be null even if
5116	* task_h_load() returns 0.
5117	*/
5118	rq->misfit_task_load = max_t(unsigned long, task_h_load(p), 1);
5119	}
5120
5121	#else /* CONFIG_SMP */
5122
5123	static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq)
5124	{
5125	return !cfs_rq->nr_running;
5126	}
5127
5128	#define UPDATE_TG 0x0
5129	#define SKIP_AGE_LOAD 0x0
5130	#define DO_ATTACH 0x0
5131	#define DO_DETACH 0x0
5132
5133	static inline void update_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se, int not_used1)
5134	{
5135	cfs_rq_util_change(cfs_rq, 0);
5136	}
5137
5138	static inline void remove_entity_load_avg(struct sched_entity *se) {}
5139
5140	static inline void
5141	attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
5142	static inline void
5143	detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {}
5144
5145	static inline int newidle_balance(struct rq *rq, struct rq_flags *rf)
5146	{
5147	return 0;
5148	}
5149
5150	static inline void
5151	util_est_enqueue(struct cfs_rq *cfs_rq, struct task_struct *p) {}
5152
5153	static inline void
5154	util_est_dequeue(struct cfs_rq *cfs_rq, struct task_struct *p) {}
5155
5156	static inline void
5157	util_est_update(struct cfs_rq *cfs_rq, struct task_struct *p,
5158	bool task_sleep) {}
5159	static inline void update_misfit_status(struct task_struct *p, struct rq *rq) {}
5160
5161	#endif /* CONFIG_SMP */
5162
5163	static void
5164	place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
5165	{
5166	u64 vslice, vruntime = avg_vruntime(cfs_rq);
5167	s64 lag = 0;
5168
5169	se->slice = sysctl_sched_base_slice;
5170	vslice = calc_delta_fair(delta: se->slice, se);
5171
5172	/*
5173	* Due to how V is constructed as the weighted average of entities,
5174	* adding tasks with positive lag, or removing tasks with negative lag
5175	* will move 'time' backwards, this can screw around with the lag of
5176	* other tasks.
5177	*
5178	* EEVDF: placement strategy #1 / #2
5179	*/
5180	if (sched_feat(PLACE_LAG) && cfs_rq->nr_running) {
5181	struct sched_entity *curr = cfs_rq->curr;
5182	unsigned long load;
5183
5184	lag = se->vlag;
5185
5186	/*
5187	* If we want to place a task and preserve lag, we have to
5188	* consider the effect of the new entity on the weighted
5189	* average and compensate for this, otherwise lag can quickly
5190	* evaporate.
5191	*
5192	* Lag is defined as:
5193	*
5194	* lag_i = S - s_i = w_i * (V - v_i)
5195	*
5196	* To avoid the 'w_i' term all over the place, we only track
5197	* the virtual lag:
5198	*
5199	* vl_i = V - v_i <=> v_i = V - vl_i
5200	*
5201	* And we take V to be the weighted average of all v:
5202	*
5203	* V = (\Sum w_j*v_j) / W
5204	*
5205	* Where W is: \Sum w_j
5206	*
5207	* Then, the weighted average after adding an entity with lag
5208	* vl_i is given by:
5209	*
5210	* V' = (\Sum w_j*v_j + w_i*v_i) / (W + w_i)
5211	* = (W*V + w_i*(V - vl_i)) / (W + w_i)
5212	* = (W*V + w_i*V - w_i*vl_i) / (W + w_i)
5213	* = (V*(W + w_i) - w_i*l) / (W + w_i)
5214	* = V - w_i*vl_i / (W + w_i)
5215	*
5216	* And the actual lag after adding an entity with vl_i is:
5217	*
5218	* vl'_i = V' - v_i
5219	* = V - w_i*vl_i / (W + w_i) - (V - vl_i)
5220	* = vl_i - w_i*vl_i / (W + w_i)
5221	*
5222	* Which is strictly less than vl_i. So in order to preserve lag
5223	* we should inflate the lag before placement such that the
5224	* effective lag after placement comes out right.
5225	*
5226	* As such, invert the above relation for vl'_i to get the vl_i
5227	* we need to use such that the lag after placement is the lag
5228	* we computed before dequeue.
5229	*
5230	* vl'_i = vl_i - w_i*vl_i / (W + w_i)
5231	* = ((W + w_i)*vl_i - w_i*vl_i) / (W + w_i)
5232	*
5233	* (W + w_i)*vl'_i = (W + w_i)*vl_i - w_i*vl_i
5234	* = W*vl_i
5235	*
5236	* vl_i = (W + w_i)*vl'_i / W
5237	*/
5238	load = cfs_rq->avg_load;
5239	if (curr && curr->on_rq)
5240	load += scale_load_down(curr->load.weight);
5241
5242	lag *= load + scale_load_down(se->load.weight);
5243	if (WARN_ON_ONCE(!load))
5244	load = 1;
5245	lag = div_s64(dividend: lag, divisor: load);
5246	}
5247
5248	se->vruntime = vruntime - lag;
5249
5250	/*
5251	* When joining the competition; the exisiting tasks will be,
5252	* on average, halfway through their slice, as such start tasks
5253	* off with half a slice to ease into the competition.
5254	*/
5255	if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL))
5256	vslice /= 2;
5257
5258	/*
5259	* EEVDF: vd_i = ve_i + r_i/w_i
5260	*/
5261	se->deadline = se->vruntime + vslice;
5262	}
5263
5264	static void check_enqueue_throttle(struct cfs_rq *cfs_rq);
5265	static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq);
5266
5267	static inline bool cfs_bandwidth_used(void);
5268
5269	static void
5270	enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
5271	{
5272	bool curr = cfs_rq->curr == se;
5273
5274	/*
5275	* If we're the current task, we must renormalise before calling
5276	* update_curr().
5277	*/
5278	if (curr)
5279	place_entity(cfs_rq, se, flags);
5280
5281	update_curr(cfs_rq);
5282
5283	/*
5284	* When enqueuing a sched_entity, we must:
5285	* - Update loads to have both entity and cfs_rq synced with now.
5286	* - For group_entity, update its runnable_weight to reflect the new
5287	* h_nr_running of its group cfs_rq.
5288	* - For group_entity, update its weight to reflect the new share of
5289	* its group cfs_rq
5290	* - Add its new weight to cfs_rq->load.weight
5291	*/
5292	update_load_avg(cfs_rq, se, UPDATE_TG | DO_ATTACH);
5293	se_update_runnable(se);
5294	/*
5295	* XXX update_load_avg() above will have attached us to the pelt sum;
5296	* but update_cfs_group() here will re-adjust the weight and have to
5297	* undo/redo all that. Seems wasteful.
5298	*/
5299	update_cfs_group(se);
5300
5301	/*
5302	* XXX now that the entity has been re-weighted, and it's lag adjusted,
5303	* we can place the entity.
5304	*/
5305	if (!curr)
5306	place_entity(cfs_rq, se, flags);
5307
5308	account_entity_enqueue(cfs_rq, se);
5309
5310	/* Entity has migrated, no longer consider this task hot */
5311	if (flags & ENQUEUE_MIGRATED)
5312	se->exec_start = 0;
5313
5314	check_schedstat_required();
5315	update_stats_enqueue_fair(cfs_rq, se, flags);
5316	if (!curr)
5317	__enqueue_entity(cfs_rq, se);
5318	se->on_rq = 1;
5319
5320	if (cfs_rq->nr_running == 1) {
5321	check_enqueue_throttle(cfs_rq);
5322	if (!throttled_hierarchy(cfs_rq)) {
5323	list_add_leaf_cfs_rq(cfs_rq);
5324	} else {
5325	#ifdef CONFIG_CFS_BANDWIDTH
5326	struct rq *rq = rq_of(cfs_rq);
5327
5328	if (cfs_rq_throttled(cfs_rq) && !cfs_rq->throttled_clock)
5329	cfs_rq->throttled_clock = rq_clock(rq);
5330	if (!cfs_rq->throttled_clock_self)
5331	cfs_rq->throttled_clock_self = rq_clock(rq);
5332	#endif
5333	}
5334	}
5335	}
5336
5337	static void __clear_buddies_next(struct sched_entity *se)
5338	{
5339	for_each_sched_entity(se) {
5340	struct cfs_rq *cfs_rq = cfs_rq_of(se);
5341	if (cfs_rq->next != se)
5342	break;
5343
5344	cfs_rq->next = NULL;
5345	}
5346	}
5347
5348	static void clear_buddies(struct cfs_rq *cfs_rq, struct sched_entity *se)
5349	{
5350	if (cfs_rq->next == se)
5351	__clear_buddies_next(se);
5352	}
5353
5354	static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq);
5355
5356	static void
5357	dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
5358	{
5359	int action = UPDATE_TG;
5360
5361	if (entity_is_task(se) && task_on_rq_migrating(p: task_of(se)))
5362	action |= DO_DETACH;
5363
5364	/*
5365	* Update run-time statistics of the 'current'.
5366	*/
5367	update_curr(cfs_rq);
5368
5369	/*
5370	* When dequeuing a sched_entity, we must:
5371	* - Update loads to have both entity and cfs_rq synced with now.
5372	* - For group_entity, update its runnable_weight to reflect the new
5373	* h_nr_running of its group cfs_rq.
5374	* - Subtract its previous weight from cfs_rq->load.weight.
5375	* - For group entity, update its weight to reflect the new share
5376	* of its group cfs_rq.
5377	*/
5378	update_load_avg(cfs_rq, se, flags: action);
5379	se_update_runnable(se);
5380
5381	update_stats_dequeue_fair(cfs_rq, se, flags);
5382
5383	clear_buddies(cfs_rq, se);
5384
5385	update_entity_lag(cfs_rq, se);
5386	if (se != cfs_rq->curr)
5387	__dequeue_entity(cfs_rq, se);
5388	se->on_rq = 0;
5389	account_entity_dequeue(cfs_rq, se);
5390
5391	/* return excess runtime on last dequeue */
5392	return_cfs_rq_runtime(cfs_rq);
5393
5394	update_cfs_group(se);
5395
5396	/*
5397	* Now advance min_vruntime if @se was the entity holding it back,
5398	* except when: DEQUEUE_SAVE && !DEQUEUE_MOVE, in this case we'll be
5399	* put back on, and if we advance min_vruntime, we'll be placed back
5400	* further than we started -- ie. we'll be penalized.
5401	*/
5402	if ((flags & (DEQUEUE_SAVE | DEQUEUE_MOVE)) != DEQUEUE_SAVE)
5403	update_min_vruntime(cfs_rq);
5404
5405	if (cfs_rq->nr_running == 0)
5406	update_idle_cfs_rq_clock_pelt(cfs_rq);
5407	}
5408
5409	static void
5410	set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
5411	{
5412	clear_buddies(cfs_rq, se);
5413
5414	/* 'current' is not kept within the tree. */
5415	if (se->on_rq) {
5416	/*
5417	* Any task has to be enqueued before it get to execute on
5418	* a CPU. So account for the time it spent waiting on the
5419	* runqueue.
5420	*/
5421	update_stats_wait_end_fair(cfs_rq, se);
5422	__dequeue_entity(cfs_rq, se);
5423	update_load_avg(cfs_rq, se, UPDATE_TG);
5424	/*
5425	* HACK, stash a copy of deadline at the point of pick in vlag,
5426	* which isn't used until dequeue.
5427	*/
5428	se->vlag = se->deadline;
5429	}
5430
5431	update_stats_curr_start(cfs_rq, se);
5432	cfs_rq->curr = se;
5433
5434	/*
5435	* Track our maximum slice length, if the CPU's load is at
5436	* least twice that of our own weight (i.e. dont track it
5437	* when there are only lesser-weight tasks around):
5438	*/
5439	if (schedstat_enabled() &&
5440	rq_of(cfs_rq)->cfs.load.weight >= 2*se->load.weight) {
5441	struct sched_statistics *stats;
5442
5443	stats = __schedstats_from_se(se);
5444	__schedstat_set(stats->slice_max,
5445	max((u64)stats->slice_max,
5446	se->sum_exec_runtime - se->prev_sum_exec_runtime));
5447	}
5448
5449	se->prev_sum_exec_runtime = se->sum_exec_runtime;
5450	}
5451
5452	/*
5453	* Pick the next process, keeping these things in mind, in this order:
5454	* 1) keep things fair between processes/task groups
5455	* 2) pick the "next" process, since someone really wants that to run
5456	* 3) pick the "last" process, for cache locality
5457	* 4) do not run the "skip" process, if something else is available
5458	*/
5459	static struct sched_entity *
5460	pick_next_entity(struct cfs_rq *cfs_rq)
5461	{
5462	/*
5463	* Enabling NEXT_BUDDY will affect latency but not fairness.
5464	*/
5465	if (sched_feat(NEXT_BUDDY) &&
5466	cfs_rq->next && entity_eligible(cfs_rq, se: cfs_rq->next))
5467	return cfs_rq->next;
5468
5469	return pick_eevdf(cfs_rq);
5470	}
5471
5472	static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq);
5473
5474	static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev)
5475	{
5476	/*
5477	* If still on the runqueue then deactivate_task()
5478	* was not called and update_curr() has to be done:
5479	*/
5480	if (prev->on_rq)
5481	update_curr(cfs_rq);
5482
5483	/* throttle cfs_rqs exceeding runtime */
5484	check_cfs_rq_runtime(cfs_rq);
5485
5486	if (prev->on_rq) {
5487	update_stats_wait_start_fair(cfs_rq, se: prev);
5488	/* Put 'current' back into the tree. */
5489	__enqueue_entity(cfs_rq, se: prev);
5490	/* in !on_rq case, update occurred at dequeue */
5491	update_load_avg(cfs_rq, se: prev, flags: 0);
5492	}
5493	cfs_rq->curr = NULL;
5494	}
5495
5496	static void
5497	entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued)
5498	{
5499	/*
5500	* Update run-time statistics of the 'current'.
5501	*/
5502	update_curr(cfs_rq);
5503
5504	/*
5505	* Ensure that runnable average is periodically updated.
5506	*/
5507	update_load_avg(cfs_rq, se: curr, UPDATE_TG);
5508	update_cfs_group(se: curr);
5509
5510	#ifdef CONFIG_SCHED_HRTICK
5511	/*
5512	* queued ticks are scheduled to match the slice, so don't bother
5513	* validating it and just reschedule.
5514	*/
5515	if (queued) {
5516	resched_curr(rq: rq_of(cfs_rq));
5517	return;
5518	}
5519	/*
5520	* don't let the period tick interfere with the hrtick preemption
5521	*/
5522	if (!sched_feat(DOUBLE_TICK) &&
5523	hrtimer_active(timer: &rq_of(cfs_rq)->hrtick_timer))
5524	return;
5525	#endif
5526	}
5527
5528
5529	/**************************************************
5530	* CFS bandwidth control machinery
5531	*/
5532
5533	#ifdef CONFIG_CFS_BANDWIDTH
5534
5535	#ifdef CONFIG_JUMP_LABEL
5536	static struct static_key __cfs_bandwidth_used;
5537
5538	static inline bool cfs_bandwidth_used(void)
5539	{
5540	return static_key_false(key: &__cfs_bandwidth_used);
5541	}
5542
5543	void cfs_bandwidth_usage_inc(void)
5544	{
5545	static_key_slow_inc_cpuslocked(key: &__cfs_bandwidth_used);
5546	}
5547
5548	void cfs_bandwidth_usage_dec(void)
5549	{
5550	static_key_slow_dec_cpuslocked(key: &__cfs_bandwidth_used);
5551	}
5552	#else /* CONFIG_JUMP_LABEL */
5553	static bool cfs_bandwidth_used(void)
5554	{
5555	return true;
5556	}
5557
5558	void cfs_bandwidth_usage_inc(void) {}
5559	void cfs_bandwidth_usage_dec(void) {}
5560	#endif /* CONFIG_JUMP_LABEL */
5561
5562	/*
5563	* default period for cfs group bandwidth.
5564	* default: 0.1s, units: nanoseconds
5565	*/
5566	static inline u64 default_cfs_period(void)
5567	{
5568	return 100000000ULL;
5569	}
5570
5571	static inline u64 sched_cfs_bandwidth_slice(void)
5572	{
5573	return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
5574	}
5575
5576	/*
5577	* Replenish runtime according to assigned quota. We use sched_clock_cpu
5578	* directly instead of rq->clock to avoid adding additional synchronization
5579	* around rq->lock.
5580	*
5581	* requires cfs_b->lock
5582	*/
5583	void __refill_cfs_bandwidth_runtime(struct cfs_bandwidth *cfs_b)
5584	{
5585	s64 runtime;
5586
5587	if (unlikely(cfs_b->quota == RUNTIME_INF))
5588	return;
5589
5590	cfs_b->runtime += cfs_b->quota;
5591	runtime = cfs_b->runtime_snap - cfs_b->runtime;
5592	if (runtime > 0) {
5593	cfs_b->burst_time += runtime;
5594	cfs_b->nr_burst++;
5595	}
5596
5597	cfs_b->runtime = min(cfs_b->runtime, cfs_b->quota + cfs_b->burst);
5598	cfs_b->runtime_snap = cfs_b->runtime;
5599	}
5600
5601	static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
5602	{
5603	return &tg->cfs_bandwidth;
5604	}
5605
5606	/* returns 0 on failure to allocate runtime */
5607	static int __assign_cfs_rq_runtime(struct cfs_bandwidth *cfs_b,
5608	struct cfs_rq *cfs_rq, u64 target_runtime)
5609	{
5610	u64 min_amount, amount = 0;
5611
5612	lockdep_assert_held(&cfs_b->lock);
5613
5614	/* note: this is a positive sum as runtime_remaining <= 0 */
5615	min_amount = target_runtime - cfs_rq->runtime_remaining;
5616
5617	if (cfs_b->quota == RUNTIME_INF)
5618	amount = min_amount;
5619	else {
5620	start_cfs_bandwidth(cfs_b);
5621
5622	if (cfs_b->runtime > 0) {
5623	amount = min(cfs_b->runtime, min_amount);
5624	cfs_b->runtime -= amount;
5625	cfs_b->idle = 0;
5626	}
5627	}
5628
5629	cfs_rq->runtime_remaining += amount;
5630
5631	return cfs_rq->runtime_remaining > 0;
5632	}
5633
5634	/* returns 0 on failure to allocate runtime */
5635	static int assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
5636	{
5637	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg: cfs_rq->tg);
5638	int ret;
5639
5640	raw_spin_lock(&cfs_b->lock);
5641	ret = __assign_cfs_rq_runtime(cfs_b, cfs_rq, target_runtime: sched_cfs_bandwidth_slice());
5642	raw_spin_unlock(&cfs_b->lock);
5643
5644	return ret;
5645	}
5646
5647	static void __account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
5648	{
5649	/* dock delta_exec before expiring quota (as it could span periods) */
5650	cfs_rq->runtime_remaining -= delta_exec;
5651
5652	if (likely(cfs_rq->runtime_remaining > 0))
5653	return;
5654
5655	if (cfs_rq->throttled)
5656	return;
5657	/*
5658	* if we're unable to extend our runtime we resched so that the active
5659	* hierarchy can be throttled
5660	*/
5661	if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr))
5662	resched_curr(rq: rq_of(cfs_rq));
5663	}
5664
5665	static __always_inline
5666	void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec)
5667	{
5668	if (!cfs_bandwidth_used() || !cfs_rq->runtime_enabled)
5669	return;
5670
5671	__account_cfs_rq_runtime(cfs_rq, delta_exec);
5672	}
5673
5674	static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
5675	{
5676	return cfs_bandwidth_used() && cfs_rq->throttled;
5677	}
5678
5679	/* check whether cfs_rq, or any parent, is throttled */
5680	static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
5681	{
5682	return cfs_bandwidth_used() && cfs_rq->throttle_count;
5683	}
5684
5685	/*
5686	* Ensure that neither of the group entities corresponding to src_cpu or
5687	* dest_cpu are members of a throttled hierarchy when performing group
5688	* load-balance operations.
5689	*/
5690	static inline int throttled_lb_pair(struct task_group *tg,
5691	int src_cpu, int dest_cpu)
5692	{
5693	struct cfs_rq *src_cfs_rq, *dest_cfs_rq;
5694
5695	src_cfs_rq = tg->cfs_rq[src_cpu];
5696	dest_cfs_rq = tg->cfs_rq[dest_cpu];
5697
5698	return throttled_hierarchy(cfs_rq: src_cfs_rq) ||
5699	throttled_hierarchy(cfs_rq: dest_cfs_rq);
5700	}
5701
5702	static int tg_unthrottle_up(struct task_group *tg, void *data)
5703	{
5704	struct rq *rq = data;
5705	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5706
5707	cfs_rq->throttle_count--;
5708	if (!cfs_rq->throttle_count) {
5709	cfs_rq->throttled_clock_pelt_time += rq_clock_pelt(rq) -
5710	cfs_rq->throttled_clock_pelt;
5711
5712	/* Add cfs_rq with load or one or more already running entities to the list */
5713	if (!cfs_rq_is_decayed(cfs_rq))
5714	list_add_leaf_cfs_rq(cfs_rq);
5715
5716	if (cfs_rq->throttled_clock_self) {
5717	u64 delta = rq_clock(rq) - cfs_rq->throttled_clock_self;
5718
5719	cfs_rq->throttled_clock_self = 0;
5720
5721	if (SCHED_WARN_ON((s64)delta < 0))
5722	delta = 0;
5723
5724	cfs_rq->throttled_clock_self_time += delta;
5725	}
5726	}
5727
5728	return 0;
5729	}
5730
5731	static int tg_throttle_down(struct task_group *tg, void *data)
5732	{
5733	struct rq *rq = data;
5734	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
5735
5736	/* group is entering throttled state, stop time */
5737	if (!cfs_rq->throttle_count) {
5738	cfs_rq->throttled_clock_pelt = rq_clock_pelt(rq);
5739	list_del_leaf_cfs_rq(cfs_rq);
5740
5741	SCHED_WARN_ON(cfs_rq->throttled_clock_self);
5742	if (cfs_rq->nr_running)
5743	cfs_rq->throttled_clock_self = rq_clock(rq);
5744	}
5745	cfs_rq->throttle_count++;
5746
5747	return 0;
5748	}
5749
5750	static bool throttle_cfs_rq(struct cfs_rq *cfs_rq)
5751	{
5752	struct rq *rq = rq_of(cfs_rq);
5753	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg: cfs_rq->tg);
5754	struct sched_entity *se;
5755	long task_delta, idle_task_delta, dequeue = 1;
5756
5757	raw_spin_lock(&cfs_b->lock);
5758	/* This will start the period timer if necessary */
5759	if (__assign_cfs_rq_runtime(cfs_b, cfs_rq, target_runtime: 1)) {
5760	/*
5761	* We have raced with bandwidth becoming available, and if we
5762	* actually throttled the timer might not unthrottle us for an
5763	* entire period. We additionally needed to make sure that any
5764	* subsequent check_cfs_rq_runtime calls agree not to throttle
5765	* us, as we may commit to do cfs put_prev+pick_next, so we ask
5766	* for 1ns of runtime rather than just check cfs_b.
5767	*/
5768	dequeue = 0;
5769	} else {
5770	list_add_tail_rcu(new: &cfs_rq->throttled_list,
5771	head: &cfs_b->throttled_cfs_rq);
5772	}
5773	raw_spin_unlock(&cfs_b->lock);
5774
5775	if (!dequeue)
5776	return false; /* Throttle no longer required. */
5777
5778	se = cfs_rq->tg->se[cpu_of(rq: rq_of(cfs_rq))];
5779
5780	/* freeze hierarchy runnable averages while throttled */
5781	rcu_read_lock();
5782	walk_tg_tree_from(from: cfs_rq->tg, down: tg_throttle_down, up: tg_nop, data: (void *)rq);
5783	rcu_read_unlock();
5784
5785	task_delta = cfs_rq->h_nr_running;
5786	idle_task_delta = cfs_rq->idle_h_nr_running;
5787	for_each_sched_entity(se) {
5788	struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5789	/* throttled entity or throttle-on-deactivate */
5790	if (!se->on_rq)
5791	goto done;
5792
5793	dequeue_entity(cfs_rq: qcfs_rq, se, DEQUEUE_SLEEP);
5794
5795	if (cfs_rq_is_idle(cfs_rq: group_cfs_rq(grp: se)))
5796	idle_task_delta = cfs_rq->h_nr_running;
5797
5798	qcfs_rq->h_nr_running -= task_delta;
5799	qcfs_rq->idle_h_nr_running -= idle_task_delta;
5800
5801	if (qcfs_rq->load.weight) {
5802	/* Avoid re-evaluating load for this entity: */
5803	se = parent_entity(se);
5804	break;
5805	}
5806	}
5807
5808	for_each_sched_entity(se) {
5809	struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5810	/* throttled entity or throttle-on-deactivate */
5811	if (!se->on_rq)
5812	goto done;
5813
5814	update_load_avg(cfs_rq: qcfs_rq, se, flags: 0);
5815	se_update_runnable(se);
5816
5817	if (cfs_rq_is_idle(cfs_rq: group_cfs_rq(grp: se)))
5818	idle_task_delta = cfs_rq->h_nr_running;
5819
5820	qcfs_rq->h_nr_running -= task_delta;
5821	qcfs_rq->idle_h_nr_running -= idle_task_delta;
5822	}
5823
5824	/* At this point se is NULL and we are at root level*/
5825	sub_nr_running(rq, count: task_delta);
5826
5827	done:
5828	/*
5829	* Note: distribution will already see us throttled via the
5830	* throttled-list. rq->lock protects completion.
5831	*/
5832	cfs_rq->throttled = 1;
5833	SCHED_WARN_ON(cfs_rq->throttled_clock);
5834	if (cfs_rq->nr_running)
5835	cfs_rq->throttled_clock = rq_clock(rq);
5836	return true;
5837	}
5838
5839	void unthrottle_cfs_rq(struct cfs_rq *cfs_rq)
5840	{
5841	struct rq *rq = rq_of(cfs_rq);
5842	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg: cfs_rq->tg);
5843	struct sched_entity *se;
5844	long task_delta, idle_task_delta;
5845
5846	se = cfs_rq->tg->se[cpu_of(rq)];
5847
5848	cfs_rq->throttled = 0;
5849
5850	update_rq_clock(rq);
5851
5852	raw_spin_lock(&cfs_b->lock);
5853	if (cfs_rq->throttled_clock) {
5854	cfs_b->throttled_time += rq_clock(rq) - cfs_rq->throttled_clock;
5855	cfs_rq->throttled_clock = 0;
5856	}
5857	list_del_rcu(entry: &cfs_rq->throttled_list);
5858	raw_spin_unlock(&cfs_b->lock);
5859
5860	/* update hierarchical throttle state */
5861	walk_tg_tree_from(from: cfs_rq->tg, down: tg_nop, up: tg_unthrottle_up, data: (void *)rq);
5862
5863	if (!cfs_rq->load.weight) {
5864	if (!cfs_rq->on_list)
5865	return;
5866	/*
5867	* Nothing to run but something to decay (on_list)?
5868	* Complete the branch.
5869	*/
5870	for_each_sched_entity(se) {
5871	if (list_add_leaf_cfs_rq(cfs_rq: cfs_rq_of(se)))
5872	break;
5873	}
5874	goto unthrottle_throttle;
5875	}
5876
5877	task_delta = cfs_rq->h_nr_running;
5878	idle_task_delta = cfs_rq->idle_h_nr_running;
5879	for_each_sched_entity(se) {
5880	struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5881
5882	if (se->on_rq)
5883	break;
5884	enqueue_entity(cfs_rq: qcfs_rq, se, ENQUEUE_WAKEUP);
5885
5886	if (cfs_rq_is_idle(cfs_rq: group_cfs_rq(grp: se)))
5887	idle_task_delta = cfs_rq->h_nr_running;
5888
5889	qcfs_rq->h_nr_running += task_delta;
5890	qcfs_rq->idle_h_nr_running += idle_task_delta;
5891
5892	/* end evaluation on encountering a throttled cfs_rq */
5893	if (cfs_rq_throttled(cfs_rq: qcfs_rq))
5894	goto unthrottle_throttle;
5895	}
5896
5897	for_each_sched_entity(se) {
5898	struct cfs_rq *qcfs_rq = cfs_rq_of(se);
5899
5900	update_load_avg(cfs_rq: qcfs_rq, se, UPDATE_TG);
5901	se_update_runnable(se);
5902
5903	if (cfs_rq_is_idle(cfs_rq: group_cfs_rq(grp: se)))
5904	idle_task_delta = cfs_rq->h_nr_running;
5905
5906	qcfs_rq->h_nr_running += task_delta;
5907	qcfs_rq->idle_h_nr_running += idle_task_delta;
5908
5909	/* end evaluation on encountering a throttled cfs_rq */
5910	if (cfs_rq_throttled(cfs_rq: qcfs_rq))
5911	goto unthrottle_throttle;
5912	}
5913
5914	/* At this point se is NULL and we are at root level*/
5915	add_nr_running(rq, count: task_delta);
5916
5917	unthrottle_throttle:
5918	assert_list_leaf_cfs_rq(rq);
5919
5920	/* Determine whether we need to wake up potentially idle CPU: */
5921	if (rq->curr == rq->idle && rq->cfs.nr_running)
5922	resched_curr(rq);
5923	}
5924
5925	#ifdef CONFIG_SMP
5926	static void __cfsb_csd_unthrottle(void *arg)
5927	{
5928	struct cfs_rq *cursor, *tmp;
5929	struct rq *rq = arg;
5930	struct rq_flags rf;
5931
5932	rq_lock(rq, rf: &rf);
5933
5934	/*
5935	* Iterating over the list can trigger several call to
5936	* update_rq_clock() in unthrottle_cfs_rq().
5937	* Do it once and skip the potential next ones.
5938	*/
5939	update_rq_clock(rq);
5940	rq_clock_start_loop_update(rq);
5941
5942	/*
5943	* Since we hold rq lock we're safe from concurrent manipulation of
5944	* the CSD list. However, this RCU critical section annotates the
5945	* fact that we pair with sched_free_group_rcu(), so that we cannot
5946	* race with group being freed in the window between removing it
5947	* from the list and advancing to the next entry in the list.
5948	*/
5949	rcu_read_lock();
5950
5951	list_for_each_entry_safe(cursor, tmp, &rq->cfsb_csd_list,
5952	throttled_csd_list) {
5953	list_del_init(entry: &cursor->throttled_csd_list);
5954
5955	if (cfs_rq_throttled(cfs_rq: cursor))
5956	unthrottle_cfs_rq(cfs_rq: cursor);
5957	}
5958
5959	rcu_read_unlock();
5960
5961	rq_clock_stop_loop_update(rq);
5962	rq_unlock(rq, rf: &rf);
5963	}
5964
5965	static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
5966	{
5967	struct rq *rq = rq_of(cfs_rq);
5968	bool first;
5969
5970	if (rq == this_rq()) {
5971	unthrottle_cfs_rq(cfs_rq);
5972	return;
5973	}
5974
5975	/* Already enqueued */
5976	if (SCHED_WARN_ON(!list_empty(&cfs_rq->throttled_csd_list)))
5977	return;
5978
5979	first = list_empty(head: &rq->cfsb_csd_list);
5980	list_add_tail(new: &cfs_rq->throttled_csd_list, head: &rq->cfsb_csd_list);
5981	if (first)
5982	smp_call_function_single_async(cpu: cpu_of(rq), csd: &rq->cfsb_csd);
5983	}
5984	#else
5985	static inline void __unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
5986	{
5987	unthrottle_cfs_rq(cfs_rq);
5988	}
5989	#endif
5990
5991	static void unthrottle_cfs_rq_async(struct cfs_rq *cfs_rq)
5992	{
5993	lockdep_assert_rq_held(rq: rq_of(cfs_rq));
5994
5995	if (SCHED_WARN_ON(!cfs_rq_throttled(cfs_rq) ||
5996	cfs_rq->runtime_remaining <= 0))
5997	return;
5998
5999	__unthrottle_cfs_rq_async(cfs_rq);
6000	}
6001
6002	static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
6003	{
6004	int this_cpu = smp_processor_id();
6005	u64 runtime, remaining = 1;
6006	bool throttled = false;
6007	struct cfs_rq *cfs_rq, *tmp;
6008	struct rq_flags rf;
6009	struct rq *rq;
6010	LIST_HEAD(local_unthrottle);
6011
6012	rcu_read_lock();
6013	list_for_each_entry_rcu(cfs_rq, &cfs_b->throttled_cfs_rq,
6014	throttled_list) {
6015	rq = rq_of(cfs_rq);
6016
6017	if (!remaining) {
6018	throttled = true;
6019	break;
6020	}
6021
6022	rq_lock_irqsave(rq, rf: &rf);
6023	if (!cfs_rq_throttled(cfs_rq))
6024	goto next;
6025
6026	/* Already queued for async unthrottle */
6027	if (!list_empty(head: &cfs_rq->throttled_csd_list))
6028	goto next;
6029
6030	/* By the above checks, this should never be true */
6031	SCHED_WARN_ON(cfs_rq->runtime_remaining > 0);
6032
6033	raw_spin_lock(&cfs_b->lock);
6034	runtime = -cfs_rq->runtime_remaining + 1;
6035	if (runtime > cfs_b->runtime)
6036	runtime = cfs_b->runtime;
6037	cfs_b->runtime -= runtime;
6038	remaining = cfs_b->runtime;
6039	raw_spin_unlock(&cfs_b->lock);
6040
6041	cfs_rq->runtime_remaining += runtime;
6042
6043	/* we check whether we're throttled above */
6044	if (cfs_rq->runtime_remaining > 0) {
6045	if (cpu_of(rq) != this_cpu) {
6046	unthrottle_cfs_rq_async(cfs_rq);
6047	} else {
6048	/*
6049	* We currently only expect to be unthrottling
6050	* a single cfs_rq locally.
6051	*/
6052	SCHED_WARN_ON(!list_empty(&local_unthrottle));
6053	list_add_tail(new: &cfs_rq->throttled_csd_list,
6054	head: &local_unthrottle);
6055	}
6056	} else {
6057	throttled = true;
6058	}
6059
6060	next:
6061	rq_unlock_irqrestore(rq, rf: &rf);
6062	}
6063
6064	list_for_each_entry_safe(cfs_rq, tmp, &local_unthrottle,
6065	throttled_csd_list) {
6066	struct rq *rq = rq_of(cfs_rq);
6067
6068	rq_lock_irqsave(rq, rf: &rf);
6069
6070	list_del_init(entry: &cfs_rq->throttled_csd_list);
6071
6072	if (cfs_rq_throttled(cfs_rq))
6073	unthrottle_cfs_rq(cfs_rq);
6074
6075	rq_unlock_irqrestore(rq, rf: &rf);
6076	}
6077	SCHED_WARN_ON(!list_empty(&local_unthrottle));
6078
6079	rcu_read_unlock();
6080
6081	return throttled;
6082	}
6083
6084	/*
6085	* Responsible for refilling a task_group's bandwidth and unthrottling its
6086	* cfs_rqs as appropriate. If there has been no activity within the last
6087	* period the timer is deactivated until scheduling resumes; cfs_b->idle is
6088	* used to track this state.
6089	*/
6090	static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)
6091	{
6092	int throttled;
6093
6094	/* no need to continue the timer with no bandwidth constraint */
6095	if (cfs_b->quota == RUNTIME_INF)
6096	goto out_deactivate;
6097
6098	throttled = !list_empty(head: &cfs_b->throttled_cfs_rq);
6099	cfs_b->nr_periods += overrun;
6100
6101	/* Refill extra burst quota even if cfs_b->idle */
6102	__refill_cfs_bandwidth_runtime(cfs_b);
6103
6104	/*
6105	* idle depends on !throttled (for the case of a large deficit), and if
6106	* we're going inactive then everything else can be deferred
6107	*/
6108	if (cfs_b->idle && !throttled)
6109	goto out_deactivate;
6110
6111	if (!throttled) {
6112	/* mark as potentially idle for the upcoming period */
6113	cfs_b->idle = 1;
6114	return 0;
6115	}
6116
6117	/* account preceding periods in which throttling occurred */
6118	cfs_b->nr_throttled += overrun;
6119
6120	/*
6121	* This check is repeated as we release cfs_b->lock while we unthrottle.
6122	*/
6123	while (throttled && cfs_b->runtime > 0) {
6124	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
6125	/* we can't nest cfs_b->lock while distributing bandwidth */
6126	throttled = distribute_cfs_runtime(cfs_b);
6127	raw_spin_lock_irqsave(&cfs_b->lock, flags);
6128	}
6129
6130	/*
6131	* While we are ensured activity in the period following an
6132	* unthrottle, this also covers the case in which the new bandwidth is
6133	* insufficient to cover the existing bandwidth deficit. (Forcing the
6134	* timer to remain active while there are any throttled entities.)
6135	*/
6136	cfs_b->idle = 0;
6137
6138	return 0;
6139
6140	out_deactivate:
6141	return 1;
6142	}
6143
6144	/* a cfs_rq won't donate quota below this amount */
6145	static const u64 min_cfs_rq_runtime = 1 * NSEC_PER_MSEC;
6146	/* minimum remaining period time to redistribute slack quota */
6147	static const u64 min_bandwidth_expiration = 2 * NSEC_PER_MSEC;
6148	/* how long we wait to gather additional slack before distributing */
6149	static const u64 cfs_bandwidth_slack_period = 5 * NSEC_PER_MSEC;
6150
6151	/*
6152	* Are we near the end of the current quota period?
6153	*
6154	* Requires cfs_b->lock for hrtimer_expires_remaining to be safe against the
6155	* hrtimer base being cleared by hrtimer_start. In the case of
6156	* migrate_hrtimers, base is never cleared, so we are fine.
6157	*/
6158	static int runtime_refresh_within(struct cfs_bandwidth *cfs_b, u64 min_expire)
6159	{
6160	struct hrtimer *refresh_timer = &cfs_b->period_timer;
6161	s64 remaining;
6162
6163	/* if the call-back is running a quota refresh is already occurring */
6164	if (hrtimer_callback_running(timer: refresh_timer))
6165	return 1;
6166
6167	/* is a quota refresh about to occur? */
6168	remaining = ktime_to_ns(kt: hrtimer_expires_remaining(timer: refresh_timer));
6169	if (remaining < (s64)min_expire)
6170	return 1;
6171
6172	return 0;
6173	}
6174
6175	static void start_cfs_slack_bandwidth(struct cfs_bandwidth *cfs_b)
6176	{
6177	u64 min_left = cfs_bandwidth_slack_period + min_bandwidth_expiration;
6178
6179	/* if there's a quota refresh soon don't bother with slack */
6180	if (runtime_refresh_within(cfs_b, min_expire: min_left))
6181	return;
6182
6183	/* don't push forwards an existing deferred unthrottle */
6184	if (cfs_b->slack_started)
6185	return;
6186	cfs_b->slack_started = true;
6187
6188	hrtimer_start(timer: &cfs_b->slack_timer,
6189	tim: ns_to_ktime(ns: cfs_bandwidth_slack_period),
6190	mode: HRTIMER_MODE_REL);
6191	}
6192
6193	/* we know any runtime found here is valid as update_curr() precedes return */
6194	static void __return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
6195	{
6196	struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg: cfs_rq->tg);
6197	s64 slack_runtime = cfs_rq->runtime_remaining - min_cfs_rq_runtime;
6198
6199	if (slack_runtime <= 0)
6200	return;
6201
6202	raw_spin_lock(&cfs_b->lock);
6203	if (cfs_b->quota != RUNTIME_INF) {
6204	cfs_b->runtime += slack_runtime;
6205
6206	/* we are under rq->lock, defer unthrottling using a timer */
6207	if (cfs_b->runtime > sched_cfs_bandwidth_slice() &&
6208	!list_empty(head: &cfs_b->throttled_cfs_rq))
6209	start_cfs_slack_bandwidth(cfs_b);
6210	}
6211	raw_spin_unlock(&cfs_b->lock);
6212
6213	/* even if it's not valid for return we don't want to try again */
6214	cfs_rq->runtime_remaining -= slack_runtime;
6215	}
6216
6217	static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
6218	{
6219	if (!cfs_bandwidth_used())
6220	return;
6221
6222	if (!cfs_rq->runtime_enabled || cfs_rq->nr_running)
6223	return;
6224
6225	__return_cfs_rq_runtime(cfs_rq);
6226	}
6227
6228	/*
6229	* This is done with a timer (instead of inline with bandwidth return) since
6230	* it's necessary to juggle rq->locks to unthrottle their respective cfs_rqs.
6231	*/
6232	static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
6233	{
6234	u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
6235	unsigned long flags;
6236
6237	/* confirm we're still not at a refresh boundary */
6238	raw_spin_lock_irqsave(&cfs_b->lock, flags);
6239	cfs_b->slack_started = false;
6240
6241	if (runtime_refresh_within(cfs_b, min_expire: min_bandwidth_expiration)) {
6242	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
6243	return;
6244	}
6245
6246	if (cfs_b->quota != RUNTIME_INF && cfs_b->runtime > slice)
6247	runtime = cfs_b->runtime;
6248
6249	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
6250
6251	if (!runtime)
6252	return;
6253
6254	distribute_cfs_runtime(cfs_b);
6255	}
6256
6257	/*
6258	* When a group wakes up we want to make sure that its quota is not already
6259	* expired/exceeded, otherwise it may be allowed to steal additional ticks of
6260	* runtime as update_curr() throttling can not trigger until it's on-rq.
6261	*/
6262	static void check_enqueue_throttle(struct cfs_rq *cfs_rq)
6263	{
6264	if (!cfs_bandwidth_used())
6265	return;
6266
6267	/* an active group must be handled by the update_curr()->put() path */
6268	if (!cfs_rq->runtime_enabled || cfs_rq->curr)
6269	return;
6270
6271	/* ensure the group is not already throttled */
6272	if (cfs_rq_throttled(cfs_rq))
6273	return;
6274
6275	/* update runtime allocation */
6276	account_cfs_rq_runtime(cfs_rq, delta_exec: 0);
6277	if (cfs_rq->runtime_remaining <= 0)
6278	throttle_cfs_rq(cfs_rq);
6279	}
6280
6281	static void sync_throttle(struct task_group *tg, int cpu)
6282	{
6283	struct cfs_rq *pcfs_rq, *cfs_rq;
6284
6285	if (!cfs_bandwidth_used())
6286	return;
6287
6288	if (!tg->parent)
6289	return;
6290
6291	cfs_rq = tg->cfs_rq[cpu];
6292	pcfs_rq = tg->parent->cfs_rq[cpu];
6293
6294	cfs_rq->throttle_count = pcfs_rq->throttle_count;
6295	cfs_rq->throttled_clock_pelt = rq_clock_pelt(cpu_rq(cpu));
6296	}
6297
6298	/* conditionally throttle active cfs_rq's from put_prev_entity() */
6299	static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq)
6300	{
6301	if (!cfs_bandwidth_used())
6302	return false;
6303
6304	if (likely(!cfs_rq->runtime_enabled || cfs_rq->runtime_remaining > 0))
6305	return false;
6306
6307	/*
6308	* it's possible for a throttled entity to be forced into a running
6309	* state (e.g. set_curr_task), in this case we're finished.
6310	*/
6311	if (cfs_rq_throttled(cfs_rq))
6312	return true;
6313
6314	return throttle_cfs_rq(cfs_rq);
6315	}
6316
6317	static enum hrtimer_restart sched_cfs_slack_timer(struct hrtimer *timer)
6318	{
6319	struct cfs_bandwidth *cfs_b =
6320	container_of(timer, struct cfs_bandwidth, slack_timer);
6321
6322	do_sched_cfs_slack_timer(cfs_b);
6323
6324	return HRTIMER_NORESTART;
6325	}
6326
6327	extern const u64 max_cfs_quota_period;
6328
6329	static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
6330	{
6331	struct cfs_bandwidth *cfs_b =
6332	container_of(timer, struct cfs_bandwidth, period_timer);
6333	unsigned long flags;
6334	int overrun;
6335	int idle = 0;
6336	int count = 0;
6337
6338	raw_spin_lock_irqsave(&cfs_b->lock, flags);
6339	for (;;) {
6340	overrun = hrtimer_forward_now(timer, interval: cfs_b->period);
6341	if (!overrun)
6342	break;
6343
6344	idle = do_sched_cfs_period_timer(cfs_b, overrun, flags);
6345
6346	if (++count > 3) {
6347	u64 new, old = ktime_to_ns(kt: cfs_b->period);
6348
6349	/*
6350	* Grow period by a factor of 2 to avoid losing precision.
6351	* Precision loss in the quota/period ratio can cause __cfs_schedulable
6352	* to fail.
6353	*/
6354	new = old * 2;
6355	if (new < max_cfs_quota_period) {
6356	cfs_b->period = ns_to_ktime(ns: new);
6357	cfs_b->quota *= 2;
6358	cfs_b->burst *= 2;
6359
6360	pr_warn_ratelimited(
6361	"cfs_period_timer[cpu%d]: period too short, scaling up (new cfs_period_us = %lld, cfs_quota_us = %lld)\n",
6362	smp_processor_id(),
6363	div_u64(new, NSEC_PER_USEC),
6364	div_u64(cfs_b->quota, NSEC_PER_USEC));
6365	} else {
6366	pr_warn_ratelimited(
6367	"cfs_period_timer[cpu%d]: period too short, but cannot scale up without losing precision (cfs_period_us = %lld, cfs_quota_us = %lld)\n",
6368	smp_processor_id(),
6369	div_u64(old, NSEC_PER_USEC),
6370	div_u64(cfs_b->quota, NSEC_PER_USEC));
6371	}
6372
6373	/* reset count so we don't come right back in here */
6374	count = 0;
6375	}
6376	}
6377	if (idle)
6378	cfs_b->period_active = 0;
6379	raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
6380
6381	return idle ? HRTIMER_NORESTART : HRTIMER_RESTART;
6382	}
6383
6384	void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent)
6385	{
6386	raw_spin_lock_init(&cfs_b->lock);
6387	cfs_b->runtime = 0;
6388	cfs_b->quota = RUNTIME_INF;
6389	cfs_b->period = ns_to_ktime(ns: default_cfs_period());
6390	cfs_b->burst = 0;
6391	cfs_b->hierarchical_quota = parent ? parent->hierarchical_quota : RUNTIME_INF;
6392
6393	INIT_LIST_HEAD(list: &cfs_b->throttled_cfs_rq);
6394	hrtimer_init(timer: &cfs_b->period_timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_ABS_PINNED);
6395	cfs_b->period_timer.function = sched_cfs_period_timer;
6396
6397	/* Add a random offset so that timers interleave */
6398	hrtimer_set_expires(timer: &cfs_b->period_timer,
6399	time: get_random_u32_below(ceil: cfs_b->period));
6400	hrtimer_init(timer: &cfs_b->slack_timer, CLOCK_MONOTONIC, mode: HRTIMER_MODE_REL);
6401	cfs_b->slack_timer.function = sched_cfs_slack_timer;
6402	cfs_b->slack_started = false;
6403	}
6404
6405	static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq)
6406	{
6407	cfs_rq->runtime_enabled = 0;
6408	INIT_LIST_HEAD(list: &cfs_rq->throttled_list);
6409	INIT_LIST_HEAD(list: &cfs_rq->throttled_csd_list);
6410	}
6411
6412	void start_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
6413	{
6414	lockdep_assert_held(&cfs_b->lock);
6415
6416	if (cfs_b->period_active)
6417	return;
6418
6419	cfs_b->period_active = 1;
6420	hrtimer_forward_now(timer: &cfs_b->period_timer, interval: cfs_b->period);
6421	hrtimer_start_expires(timer: &cfs_b->period_timer, mode: HRTIMER_MODE_ABS_PINNED);
6422	}
6423
6424	static void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
6425	{
6426	int __maybe_unused i;
6427
6428	/* init_cfs_bandwidth() was not called */
6429	if (!cfs_b->throttled_cfs_rq.next)
6430	return;
6431
6432	hrtimer_cancel(timer: &cfs_b->period_timer);
6433	hrtimer_cancel(timer: &cfs_b->slack_timer);
6434
6435	/*
6436	* It is possible that we still have some cfs_rq's pending on a CSD
6437	* list, though this race is very rare. In order for this to occur, we
6438	* must have raced with the last task leaving the group while there
6439	* exist throttled cfs_rq(s), and the period_timer must have queued the
6440	* CSD item but the remote cpu has not yet processed it. To handle this,
6441	* we can simply flush all pending CSD work inline here. We're
6442	* guaranteed at this point that no additional cfs_rq of this group can
6443	* join a CSD list.
6444	*/
6445	#ifdef CONFIG_SMP
6446	for_each_possible_cpu(i) {
6447	struct rq *rq = cpu_rq(i);
6448	unsigned long flags;
6449
6450	if (list_empty(head: &rq->cfsb_csd_list))
6451	continue;
6452
6453	local_irq_save(flags);
6454	__cfsb_csd_unthrottle(arg: rq);
6455	local_irq_restore(flags);
6456	}
6457	#endif
6458	}
6459
6460	/*
6461	* Both these CPU hotplug callbacks race against unregister_fair_sched_group()
6462	*
6463	* The race is harmless, since modifying bandwidth settings of unhooked group
6464	* bits doesn't do much.
6465	*/
6466
6467	/* cpu online callback */
6468	static void __maybe_unused update_runtime_enabled(struct rq *rq)
6469	{
6470	struct task_group *tg;
6471
6472	lockdep_assert_rq_held(rq);
6473
6474	rcu_read_lock();
6475	list_for_each_entry_rcu(tg, &task_groups, list) {
6476	struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth;
6477	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
6478
6479	raw_spin_lock(&cfs_b->lock);
6480	cfs_rq->runtime_enabled = cfs_b->quota != RUNTIME_INF;
6481	raw_spin_unlock(&cfs_b->lock);
6482	}
6483	rcu_read_unlock();
6484	}
6485
6486	/* cpu offline callback */
6487	static void __maybe_unused unthrottle_offline_cfs_rqs(struct rq *rq)
6488	{
6489	struct task_group *tg;
6490
6491	lockdep_assert_rq_held(rq);
6492
6493	/*
6494	* The rq clock has already been updated in the
6495	* set_rq_offline(), so we should skip updating
6496	* the rq clock again in unthrottle_cfs_rq().
6497	*/
6498	rq_clock_start_loop_update(rq);
6499
6500	rcu_read_lock();
6501	list_for_each_entry_rcu(tg, &task_groups, list) {
6502	struct cfs_rq *cfs_rq = tg->cfs_rq[cpu_of(rq)];
6503
6504	if (!cfs_rq->runtime_enabled)
6505	continue;
6506
6507	/*
6508	* clock_task is not advancing so we just need to make sure
6509	* there's some valid quota amount
6510	*/
6511	cfs_rq->runtime_remaining = 1;
6512	/*
6513	* Offline rq is schedulable till CPU is completely disabled
6514	* in take_cpu_down(), so we prevent new cfs throttling here.
6515	*/
6516	cfs_rq->runtime_enabled = 0;
6517
6518	if (cfs_rq_throttled(cfs_rq))
6519	unthrottle_cfs_rq(cfs_rq);
6520	}
6521	rcu_read_unlock();
6522
6523	rq_clock_stop_loop_update(rq);
6524	}
6525
6526	bool cfs_task_bw_constrained(struct task_struct *p)
6527	{
6528	struct cfs_rq *cfs_rq = task_cfs_rq(p);
6529
6530	if (!cfs_bandwidth_used())
6531	return false;
6532
6533	if (cfs_rq->runtime_enabled ||
6534	tg_cfs_bandwidth(tg: cfs_rq->tg)->hierarchical_quota != RUNTIME_INF)
6535	return true;
6536
6537	return false;
6538	}
6539
6540	#ifdef CONFIG_NO_HZ_FULL
6541	/* called from pick_next_task_fair() */
6542	static void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p)
6543	{
6544	int cpu = cpu_of(rq);
6545
6546	if (!sched_feat(HZ_BW) || !cfs_bandwidth_used())
6547	return;
6548
6549	if (!tick_nohz_full_cpu(cpu))
6550	return;
6551
6552	if (rq->nr_running != 1)
6553	return;
6554
6555	/*
6556	* We know there is only one task runnable and we've just picked it. The
6557	* normal enqueue path will have cleared TICK_DEP_BIT_SCHED if we will
6558	* be otherwise able to stop the tick. Just need to check if we are using
6559	* bandwidth control.
6560	*/
6561	if (cfs_task_bw_constrained(p))
6562	tick_nohz_dep_set_cpu(cpu, TICK_DEP_BIT_SCHED);
6563	}
6564	#endif
6565
6566	#else /* CONFIG_CFS_BANDWIDTH */
6567
6568	static inline bool cfs_bandwidth_used(void)
6569	{
6570	return false;
6571	}
6572
6573	static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq, u64 delta_exec) {}
6574	static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq) { return false; }
6575	static void check_enqueue_throttle(struct cfs_rq *cfs_rq) {}
6576	static inline void sync_throttle(struct task_group *tg, int cpu) {}
6577	static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
6578
6579	static inline int cfs_rq_throttled(struct cfs_rq *cfs_rq)
6580	{
6581	return 0;
6582	}
6583
6584	static inline int throttled_hierarchy(struct cfs_rq *cfs_rq)
6585	{
6586	return 0;
6587	}
6588
6589	static inline int throttled_lb_pair(struct task_group *tg,
6590	int src_cpu, int dest_cpu)
6591	{
6592	return 0;
6593	}
6594
6595	#ifdef CONFIG_FAIR_GROUP_SCHED
6596	void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b, struct cfs_bandwidth *parent) {}
6597	static void init_cfs_rq_runtime(struct cfs_rq *cfs_rq) {}
6598	#endif
6599
6600	static inline struct cfs_bandwidth *tg_cfs_bandwidth(struct task_group *tg)
6601	{
6602	return NULL;
6603	}
6604	static inline void destroy_cfs_bandwidth(struct cfs_bandwidth *cfs_b) {}
6605	static inline void update_runtime_enabled(struct rq *rq) {}
6606	static inline void unthrottle_offline_cfs_rqs(struct rq *rq) {}
6607	#ifdef CONFIG_CGROUP_SCHED
6608	bool cfs_task_bw_constrained(struct task_struct *p)
6609	{
6610	return false;
6611	}
6612	#endif
6613	#endif /* CONFIG_CFS_BANDWIDTH */
6614
6615	#if !defined(CONFIG_CFS_BANDWIDTH) || !defined(CONFIG_NO_HZ_FULL)
6616	static inline void sched_fair_update_stop_tick(struct rq *rq, struct task_struct *p) {}
6617	#endif
6618
6619	/**************************************************
6620	* CFS operations on tasks:
6621	*/
6622
6623	#ifdef CONFIG_SCHED_HRTICK
6624	static void hrtick_start_fair(struct rq *rq, struct task_struct *p)
6625	{
6626	struct sched_entity *se = &p->se;
6627
6628	SCHED_WARN_ON(task_rq(p) != rq);
6629
6630	if (rq->cfs.h_nr_running > 1) {
6631	u64 ran = se->sum_exec_runtime - se->prev_sum_exec_runtime;
6632	u64 slice = se->slice;
6633	s64 delta = slice - ran;
6634
6635	if (delta < 0) {
6636	if (task_current(rq, p))
6637	resched_curr(rq);
6638	return;
6639	}
6640	hrtick_start(rq, delay: delta);
6641	}
6642	}
6643
6644	/*
6645	* called from enqueue/dequeue and updates the hrtick when the
6646	* current task is from our class and nr_running is low enough
6647	* to matter.
6648	*/
6649	static void hrtick_update(struct rq *rq)
6650	{
6651	struct task_struct *curr = rq->curr;
6652
6653	if (!hrtick_enabled_fair(rq) || curr->sched_class != &fair_sched_class)
6654	return;
6655
6656	hrtick_start_fair(rq, p: curr);
6657	}
6658	#else /* !CONFIG_SCHED_HRTICK */
6659	static inline void
6660	hrtick_start_fair(struct rq *rq, struct task_struct *p)
6661	{
6662	}
6663
6664	static inline void hrtick_update(struct rq *rq)
6665	{
6666	}
6667	#endif
6668
6669	#ifdef CONFIG_SMP
6670	static inline bool cpu_overutilized(int cpu)
6671	{
6672	unsigned long rq_util_min = uclamp_rq_get(cpu_rq(cpu), clamp_id: UCLAMP_MIN);
6673	unsigned long rq_util_max = uclamp_rq_get(cpu_rq(cpu), clamp_id: UCLAMP_MAX);
6674
6675	/* Return true only if the utilization doesn't fit CPU's capacity */
6676	return !util_fits_cpu(util: cpu_util_cfs(cpu), uclamp_min: rq_util_min, uclamp_max: rq_util_max, cpu);
6677	}
6678
6679	static inline void update_overutilized_status(struct rq *rq)
6680	{
6681	if (!READ_ONCE(rq->rd->overutilized) && cpu_overutilized(cpu: rq->cpu)) {
6682	WRITE_ONCE(rq->rd->overutilized, SG_OVERUTILIZED);
6683	trace_sched_overutilized_tp(rd: rq->rd, SG_OVERUTILIZED);
6684	}
6685	}
6686	#else
6687	static inline void update_overutilized_status(struct rq *rq) { }
6688	#endif
6689
6690	/* Runqueue only has SCHED_IDLE tasks enqueued */
6691	static int sched_idle_rq(struct rq *rq)
6692	{
6693	return unlikely(rq->nr_running == rq->cfs.idle_h_nr_running &&
6694	rq->nr_running);
6695	}
6696
6697	#ifdef CONFIG_SMP
6698	static int sched_idle_cpu(int cpu)
6699	{
6700	return sched_idle_rq(cpu_rq(cpu));
6701	}
6702	#endif
6703
6704	/*
6705	* The enqueue_task method is called before nr_running is
6706	* increased. Here we update the fair scheduling stats and
6707	* then put the task into the rbtree:
6708	*/
6709	static void
6710	enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
6711	{
6712	struct cfs_rq *cfs_rq;
6713	struct sched_entity *se = &p->se;
6714	int idle_h_nr_running = task_has_idle_policy(p);
6715	int task_new = !(flags & ENQUEUE_WAKEUP);
6716
6717	/*
6718	* The code below (indirectly) updates schedutil which looks at
6719	* the cfs_rq utilization to select a frequency.
6720	* Let's add the task's estimated utilization to the cfs_rq's
6721	* estimated utilization, before we update schedutil.
6722	*/
6723	util_est_enqueue(cfs_rq: &rq->cfs, p);
6724
6725	/*
6726	* If in_iowait is set, the code below may not trigger any cpufreq
6727	* utilization updates, so do it here explicitly with the IOWAIT flag
6728	* passed.
6729	*/
6730	if (p->in_iowait)
6731	cpufreq_update_util(rq, SCHED_CPUFREQ_IOWAIT);
6732
6733	for_each_sched_entity(se) {
6734	if (se->on_rq)
6735	break;
6736	cfs_rq = cfs_rq_of(se);
6737	enqueue_entity(cfs_rq, se, flags);
6738
6739	cfs_rq->h_nr_running++;
6740	cfs_rq->idle_h_nr_running += idle_h_nr_running;
6741
6742	if (cfs_rq_is_idle(cfs_rq))
6743	idle_h_nr_running = 1;
6744
6745	/* end evaluation on encountering a throttled cfs_rq */
6746	if (cfs_rq_throttled(cfs_rq))
6747	goto enqueue_throttle;
6748
6749	flags = ENQUEUE_WAKEUP;
6750	}
6751
6752	for_each_sched_entity(se) {
6753	cfs_rq = cfs_rq_of(se);
6754
6755	update_load_avg(cfs_rq, se, UPDATE_TG);
6756	se_update_runnable(se);
6757	update_cfs_group(se);
6758
6759	cfs_rq->h_nr_running++;
6760	cfs_rq->idle_h_nr_running += idle_h_nr_running;
6761
6762	if (cfs_rq_is_idle(cfs_rq))
6763	idle_h_nr_running = 1;
6764
6765	/* end evaluation on encountering a throttled cfs_rq */
6766	if (cfs_rq_throttled(cfs_rq))
6767	goto enqueue_throttle;
6768	}
6769
6770	/* At this point se is NULL and we are at root level*/
6771	add_nr_running(rq, count: 1);
6772
6773	/*
6774	* Since new tasks are assigned an initial util_avg equal to
6775	* half of the spare capacity of their CPU, tiny tasks have the
6776	* ability to cross the overutilized threshold, which will
6777	* result in the load balancer ruining all the task placement
6778	* done by EAS. As a way to mitigate that effect, do not account
6779	* for the first enqueue operation of new tasks during the
6780	* overutilized flag detection.
6781	*
6782	* A better way of solving this problem would be to wait for
6783	* the PELT signals of tasks to converge before taking them
6784	* into account, but that is not straightforward to implement,
6785	* and the following generally works well enough in practice.
6786	*/
6787	if (!task_new)
6788	update_overutilized_status(rq);
6789
6790	enqueue_throttle:
6791	assert_list_leaf_cfs_rq(rq);
6792
6793	hrtick_update(rq);
6794	}
6795
6796	static void set_next_buddy(struct sched_entity *se);
6797
6798	/*
6799	* The dequeue_task method is called before nr_running is
6800	* decreased. We remove the task from the rbtree and
6801	* update the fair scheduling stats:
6802	*/
6803	static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
6804	{
6805	struct cfs_rq *cfs_rq;
6806	struct sched_entity *se = &p->se;
6807	int task_sleep = flags & DEQUEUE_SLEEP;
6808	int idle_h_nr_running = task_has_idle_policy(p);
6809	bool was_sched_idle = sched_idle_rq(rq);
6810
6811	util_est_dequeue(cfs_rq: &rq->cfs, p);
6812
6813	for_each_sched_entity(se) {
6814	cfs_rq = cfs_rq_of(se);
6815	dequeue_entity(cfs_rq, se, flags);
6816
6817	cfs_rq->h_nr_running--;
6818	cfs_rq->idle_h_nr_running -= idle_h_nr_running;
6819
6820	if (cfs_rq_is_idle(cfs_rq))
6821	idle_h_nr_running = 1;
6822
6823	/* end evaluation on encountering a throttled cfs_rq */
6824	if (cfs_rq_throttled(cfs_rq))
6825	goto dequeue_throttle;
6826
6827	/* Don't dequeue parent if it has other entities besides us */
6828	if (cfs_rq->load.weight) {
6829	/* Avoid re-evaluating load for this entity: */
6830	se = parent_entity(se);
6831	/*
6832	* Bias pick_next to pick a task from this cfs_rq, as
6833	* p is sleeping when it is within its sched_slice.
6834	*/
6835	if (task_sleep && se && !throttled_hierarchy(cfs_rq))
6836	set_next_buddy(se);
6837	break;
6838	}
6839	flags |= DEQUEUE_SLEEP;
6840	}
6841
6842	for_each_sched_entity(se) {
6843	cfs_rq = cfs_rq_of(se);
6844
6845	update_load_avg(cfs_rq, se, UPDATE_TG);
6846	se_update_runnable(se);
6847	update_cfs_group(se);
6848
6849	cfs_rq->h_nr_running--;
6850	cfs_rq->idle_h_nr_running -= idle_h_nr_running;
6851
6852	if (cfs_rq_is_idle(cfs_rq))
6853	idle_h_nr_running = 1;
6854
6855	/* end evaluation on encountering a throttled cfs_rq */
6856	if (cfs_rq_throttled(cfs_rq))
6857	goto dequeue_throttle;
6858
6859	}
6860
6861	/* At this point se is NULL and we are at root level*/
6862	sub_nr_running(rq, count: 1);
6863
6864	/* balance early to pull high priority tasks */
6865	if (unlikely(!was_sched_idle && sched_idle_rq(rq)))
6866	rq->next_balance = jiffies;
6867
6868	dequeue_throttle:
6869	util_est_update(cfs_rq: &rq->cfs, p, task_sleep);
6870	hrtick_update(rq);
6871	}
6872
6873	#ifdef CONFIG_SMP
6874
6875	/* Working cpumask for: load_balance, load_balance_newidle. */
6876	static DEFINE_PER_CPU(cpumask_var_t, load_balance_mask);
6877	static DEFINE_PER_CPU(cpumask_var_t, select_rq_mask);
6878	static DEFINE_PER_CPU(cpumask_var_t, should_we_balance_tmpmask);
6879
6880	#ifdef CONFIG_NO_HZ_COMMON
6881
6882	static struct {
6883	cpumask_var_t idle_cpus_mask;
6884	atomic_t nr_cpus;
6885	int has_blocked; /* Idle CPUS has blocked load */
6886	int needs_update; /* Newly idle CPUs need their next_balance collated */
6887	unsigned long next_balance; /* in jiffy units */
6888	unsigned long next_blocked; /* Next update of blocked load in jiffies */
6889	} nohz ____cacheline_aligned;
6890
6891	#endif /* CONFIG_NO_HZ_COMMON */
6892
6893	static unsigned long cpu_load(struct rq *rq)
6894	{
6895	return cfs_rq_load_avg(cfs_rq: &rq->cfs);
6896	}
6897
6898	/*
6899	* cpu_load_without - compute CPU load without any contributions from *p
6900	* @cpu: the CPU which load is requested
6901	* @p: the task which load should be discounted
6902	*
6903	* The load of a CPU is defined by the load of tasks currently enqueued on that
6904	* CPU as well as tasks which are currently sleeping after an execution on that
6905	* CPU.
6906	*
6907	* This method returns the load of the specified CPU by discounting the load of
6908	* the specified task, whenever the task is currently contributing to the CPU
6909	* load.
6910	*/
6911	static unsigned long cpu_load_without(struct rq *rq, struct task_struct *p)
6912	{
6913	struct cfs_rq *cfs_rq;
6914	unsigned int load;
6915
6916	/* Task has no contribution or is new */
6917	if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
6918	return cpu_load(rq);
6919
6920	cfs_rq = &rq->cfs;
6921	load = READ_ONCE(cfs_rq->avg.load_avg);
6922
6923	/* Discount task's util from CPU's util */
6924	lsub_positive(&load, task_h_load(p));
6925
6926	return load;
6927	}
6928
6929	static unsigned long cpu_runnable(struct rq *rq)
6930	{
6931	return cfs_rq_runnable_avg(cfs_rq: &rq->cfs);
6932	}
6933
6934	static unsigned long cpu_runnable_without(struct rq *rq, struct task_struct *p)
6935	{
6936	struct cfs_rq *cfs_rq;
6937	unsigned int runnable;
6938
6939	/* Task has no contribution or is new */
6940	if (cpu_of(rq) != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
6941	return cpu_runnable(rq);
6942
6943	cfs_rq = &rq->cfs;
6944	runnable = READ_ONCE(cfs_rq->avg.runnable_avg);
6945
6946	/* Discount task's runnable from CPU's runnable */
6947	lsub_positive(&runnable, p->se.avg.runnable_avg);
6948
6949	return runnable;
6950	}
6951
6952	static unsigned long capacity_of(int cpu)
6953	{
6954	return cpu_rq(cpu)->cpu_capacity;
6955	}
6956
6957	static void record_wakee(struct task_struct *p)
6958	{
6959	/*
6960	* Only decay a single time; tasks that have less then 1 wakeup per
6961	* jiffy will not have built up many flips.
6962	*/
6963	if (time_after(jiffies, current->wakee_flip_decay_ts + HZ)) {
6964	current->wakee_flips >>= 1;
6965	current->wakee_flip_decay_ts = jiffies;
6966	}
6967
6968	if (current->last_wakee != p) {
6969	current->last_wakee = p;
6970	current->wakee_flips++;
6971	}
6972	}
6973
6974	/*
6975	* Detect M:N waker/wakee relationships via a switching-frequency heuristic.
6976	*
6977	* A waker of many should wake a different task than the one last awakened
6978	* at a frequency roughly N times higher than one of its wakees.
6979	*
6980	* In order to determine whether we should let the load spread vs consolidating
6981	* to shared cache, we look for a minimum 'flip' frequency of llc_size in one
6982	* partner, and a factor of lls_size higher frequency in the other.
6983	*
6984	* With both conditions met, we can be relatively sure that the relationship is
6985	* non-monogamous, with partner count exceeding socket size.
6986	*
6987	* Waker/wakee being client/server, worker/dispatcher, interrupt source or
6988	* whatever is irrelevant, spread criteria is apparent partner count exceeds
6989	* socket size.
6990	*/
6991	static int wake_wide(struct task_struct *p)
6992	{
6993	unsigned int master = current->wakee_flips;
6994	unsigned int slave = p->wakee_flips;
6995	int factor = __this_cpu_read(sd_llc_size);
6996
6997	if (master < slave)
6998	swap(master, slave);
6999	if (slave < factor || master < slave * factor)
7000	return 0;
7001	return 1;
7002	}
7003
7004	/*
7005	* The purpose of wake_affine() is to quickly determine on which CPU we can run
7006	* soonest. For the purpose of speed we only consider the waking and previous
7007	* CPU.
7008	*
7009	* wake_affine_idle() - only considers 'now', it check if the waking CPU is
7010	* cache-affine and is (or will be) idle.
7011	*
7012	* wake_affine_weight() - considers the weight to reflect the average
7013	* scheduling latency of the CPUs. This seems to work
7014	* for the overloaded case.
7015	*/
7016	static int
7017	wake_affine_idle(int this_cpu, int prev_cpu, int sync)
7018	{
7019	/*
7020	* If this_cpu is idle, it implies the wakeup is from interrupt
7021	* context. Only allow the move if cache is shared. Otherwise an
7022	* interrupt intensive workload could force all tasks onto one
7023	* node depending on the IO topology or IRQ affinity settings.
7024	*
7025	* If the prev_cpu is idle and cache affine then avoid a migration.
7026	* There is no guarantee that the cache hot data from an interrupt
7027	* is more important than cache hot data on the prev_cpu and from
7028	* a cpufreq perspective, it's better to have higher utilisation
7029	* on one CPU.
7030	*/
7031	if (available_idle_cpu(cpu: this_cpu) && cpus_share_cache(this_cpu, that_cpu: prev_cpu))
7032	return available_idle_cpu(cpu: prev_cpu) ? prev_cpu : this_cpu;
7033
7034	if (sync && cpu_rq(this_cpu)->nr_running == 1)
7035	return this_cpu;
7036
7037	if (available_idle_cpu(cpu: prev_cpu))
7038	return prev_cpu;
7039
7040	return nr_cpumask_bits;
7041	}
7042
7043	static int
7044	wake_affine_weight(struct sched_domain *sd, struct task_struct *p,
7045	int this_cpu, int prev_cpu, int sync)
7046	{
7047	s64 this_eff_load, prev_eff_load;
7048	unsigned long task_load;
7049
7050	this_eff_load = cpu_load(cpu_rq(this_cpu));
7051
7052	if (sync) {
7053	unsigned long current_load = task_h_load(current);
7054
7055	if (current_load > this_eff_load)
7056	return this_cpu;
7057
7058	this_eff_load -= current_load;
7059	}
7060
7061	task_load = task_h_load(p);
7062
7063	this_eff_load += task_load;
7064	if (sched_feat(WA_BIAS))
7065	this_eff_load *= 100;
7066	this_eff_load *= capacity_of(cpu: prev_cpu);
7067
7068	prev_eff_load = cpu_load(cpu_rq(prev_cpu));
7069	prev_eff_load -= task_load;
7070	if (sched_feat(WA_BIAS))
7071	prev_eff_load *= 100 + (sd->imbalance_pct - 100) / 2;
7072	prev_eff_load *= capacity_of(cpu: this_cpu);
7073
7074	/*
7075	* If sync, adjust the weight of prev_eff_load such that if
7076	* prev_eff == this_eff that select_idle_sibling() will consider
7077	* stacking the wakee on top of the waker if no other CPU is
7078	* idle.
7079	*/
7080	if (sync)
7081	prev_eff_load += 1;
7082
7083	return this_eff_load < prev_eff_load ? this_cpu : nr_cpumask_bits;
7084	}
7085
7086	static int wake_affine(struct sched_domain *sd, struct task_struct *p,
7087	int this_cpu, int prev_cpu, int sync)
7088	{
7089	int target = nr_cpumask_bits;
7090
7091	if (sched_feat(WA_IDLE))
7092	target = wake_affine_idle(this_cpu, prev_cpu, sync);
7093
7094	if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits)
7095	target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync);
7096
7097	schedstat_inc(p->stats.nr_wakeups_affine_attempts);
7098	if (target != this_cpu)
7099	return prev_cpu;
7100
7101	schedstat_inc(sd->ttwu_move_affine);
7102	schedstat_inc(p->stats.nr_wakeups_affine);
7103	return target;
7104	}
7105
7106	static struct sched_group *
7107	find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu);
7108
7109	/*
7110	* find_idlest_group_cpu - find the idlest CPU among the CPUs in the group.
7111	*/
7112	static int
7113	find_idlest_group_cpu(struct sched_group *group, struct task_struct *p, int this_cpu)
7114	{
7115	unsigned long load, min_load = ULONG_MAX;
7116	unsigned int min_exit_latency = UINT_MAX;
7117	u64 latest_idle_timestamp = 0;
7118	int least_loaded_cpu = this_cpu;
7119	int shallowest_idle_cpu = -1;
7120	int i;
7121
7122	/* Check if we have any choice: */
7123	if (group->group_weight == 1)
7124	return cpumask_first(srcp: sched_group_span(sg: group));
7125
7126	/* Traverse only the allowed CPUs */
7127	for_each_cpu_and(i, sched_group_span(group), p->cpus_ptr) {
7128	struct rq *rq = cpu_rq(i);
7129
7130	if (!sched_core_cookie_match(rq, p))
7131	continue;
7132
7133	if (sched_idle_cpu(cpu: i))
7134	return i;
7135
7136	if (available_idle_cpu(cpu: i)) {
7137	struct cpuidle_state *idle = idle_get_state(rq);
7138	if (idle && idle->exit_latency < min_exit_latency) {
7139	/*
7140	* We give priority to a CPU whose idle state
7141	* has the smallest exit latency irrespective
7142	* of any idle timestamp.
7143	*/
7144	min_exit_latency = idle->exit_latency;
7145	latest_idle_timestamp = rq->idle_stamp;
7146	shallowest_idle_cpu = i;
7147	} else if ((!idle || idle->exit_latency == min_exit_latency) &&
7148	rq->idle_stamp > latest_idle_timestamp) {
7149	/*
7150	* If equal or no active idle state, then
7151	* the most recently idled CPU might have
7152	* a warmer cache.
7153	*/
7154	latest_idle_timestamp = rq->idle_stamp;
7155	shallowest_idle_cpu = i;
7156	}
7157	} else if (shallowest_idle_cpu == -1) {
7158	load = cpu_load(cpu_rq(i));
7159	if (load < min_load) {
7160	min_load = load;
7161	least_loaded_cpu = i;
7162	}
7163	}
7164	}
7165
7166	return shallowest_idle_cpu != -1 ? shallowest_idle_cpu : least_loaded_cpu;
7167	}
7168
7169	static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p,
7170	int cpu, int prev_cpu, int sd_flag)
7171	{
7172	int new_cpu = cpu;
7173
7174	if (!cpumask_intersects(src1p: sched_domain_span(sd), src2p: p->cpus_ptr))
7175	return prev_cpu;
7176
7177	/*
7178	* We need task's util for cpu_util_without, sync it up to
7179	* prev_cpu's last_update_time.
7180	*/
7181	if (!(sd_flag & SD_BALANCE_FORK))
7182	sync_entity_load_avg(se: &p->se);
7183
7184	while (sd) {
7185	struct sched_group *group;
7186	struct sched_domain *tmp;
7187	int weight;
7188
7189	if (!(sd->flags & sd_flag)) {
7190	sd = sd->child;
7191	continue;
7192	}
7193
7194	group = find_idlest_group(sd, p, this_cpu: cpu);
7195	if (!group) {
7196	sd = sd->child;
7197	continue;
7198	}
7199
7200	new_cpu = find_idlest_group_cpu(group, p, this_cpu: cpu);
7201	if (new_cpu == cpu) {
7202	/* Now try balancing at a lower domain level of 'cpu': */
7203	sd = sd->child;
7204	continue;
7205	}
7206
7207	/* Now try balancing at a lower domain level of 'new_cpu': */
7208	cpu = new_cpu;
7209	weight = sd->span_weight;
7210	sd = NULL;
7211	for_each_domain(cpu, tmp) {
7212	if (weight <= tmp->span_weight)
7213	break;
7214	if (tmp->flags & sd_flag)
7215	sd = tmp;
7216	}
7217	}
7218
7219	return new_cpu;
7220	}
7221
7222	static inline int __select_idle_cpu(int cpu, struct task_struct *p)
7223	{
7224	if ((available_idle_cpu(cpu) || sched_idle_cpu(cpu)) &&
7225	sched_cpu_cookie_match(cpu_rq(cpu), p))
7226	return cpu;
7227
7228	return -1;
7229	}
7230
7231	#ifdef CONFIG_SCHED_SMT
7232	DEFINE_STATIC_KEY_FALSE(sched_smt_present);
7233	EXPORT_SYMBOL_GPL(sched_smt_present);
7234
7235	static inline void set_idle_cores(int cpu, int val)
7236	{
7237	struct sched_domain_shared *sds;
7238
7239	sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
7240	if (sds)
7241	WRITE_ONCE(sds->has_idle_cores, val);
7242	}
7243
7244	static inline bool test_idle_cores(int cpu)
7245	{
7246	struct sched_domain_shared *sds;
7247
7248	sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
7249	if (sds)
7250	return READ_ONCE(sds->has_idle_cores);
7251
7252	return false;
7253	}
7254
7255	/*
7256	* Scans the local SMT mask to see if the entire core is idle, and records this
7257	* information in sd_llc_shared->has_idle_cores.
7258	*
7259	* Since SMT siblings share all cache levels, inspecting this limited remote
7260	* state should be fairly cheap.
7261	*/
7262	void __update_idle_core(struct rq *rq)
7263	{
7264	int core = cpu_of(rq);
7265	int cpu;
7266
7267	rcu_read_lock();
7268	if (test_idle_cores(cpu: core))
7269	goto unlock;
7270
7271	for_each_cpu(cpu, cpu_smt_mask(core)) {
7272	if (cpu == core)
7273	continue;
7274
7275	if (!available_idle_cpu(cpu))
7276	goto unlock;
7277	}
7278
7279	set_idle_cores(cpu: core, val: 1);
7280	unlock:
7281	rcu_read_unlock();
7282	}
7283
7284	/*
7285	* Scan the entire LLC domain for idle cores; this dynamically switches off if
7286	* there are no idle cores left in the system; tracked through
7287	* sd_llc->shared->has_idle_cores and enabled through update_idle_core() above.
7288	*/
7289	static int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
7290	{
7291	bool idle = true;
7292	int cpu;
7293
7294	for_each_cpu(cpu, cpu_smt_mask(core)) {
7295	if (!available_idle_cpu(cpu)) {
7296	idle = false;
7297	if (*idle_cpu == -1) {
7298	if (sched_idle_cpu(cpu) && cpumask_test_cpu(cpu, cpumask: cpus)) {
7299	*idle_cpu = cpu;
7300	break;
7301	}
7302	continue;
7303	}
7304	break;
7305	}
7306	if (*idle_cpu == -1 && cpumask_test_cpu(cpu, cpumask: cpus))
7307	*idle_cpu = cpu;
7308	}
7309
7310	if (idle)
7311	return core;
7312
7313	cpumask_andnot(dstp: cpus, src1p: cpus, src2p: cpu_smt_mask(cpu: core));
7314	return -1;
7315	}
7316
7317	/*
7318	* Scan the local SMT mask for idle CPUs.
7319	*/
7320	static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
7321	{
7322	int cpu;
7323
7324	for_each_cpu_and(cpu, cpu_smt_mask(target), p->cpus_ptr) {
7325	if (cpu == target)
7326	continue;
7327	/*
7328	* Check if the CPU is in the LLC scheduling domain of @target.
7329	* Due to isolcpus, there is no guarantee that all the siblings are in the domain.
7330	*/
7331	if (!cpumask_test_cpu(cpu, cpumask: sched_domain_span(sd)))
7332	continue;
7333	if (available_idle_cpu(cpu) || sched_idle_cpu(cpu))
7334	return cpu;
7335	}
7336
7337	return -1;
7338	}
7339
7340	#else /* CONFIG_SCHED_SMT */
7341
7342	static inline void set_idle_cores(int cpu, int val)
7343	{
7344	}
7345
7346	static inline bool test_idle_cores(int cpu)
7347	{
7348	return false;
7349	}
7350
7351	static inline int select_idle_core(struct task_struct *p, int core, struct cpumask *cpus, int *idle_cpu)
7352	{
7353	return __select_idle_cpu(core, p);
7354	}
7355
7356	static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int target)
7357	{
7358	return -1;
7359	}
7360
7361	#endif /* CONFIG_SCHED_SMT */
7362
7363	/*
7364	* Scan the LLC domain for idle CPUs; this is dynamically regulated by
7365	* comparing the average scan cost (tracked in sd->avg_scan_cost) against the
7366	* average idle time for this rq (as found in rq->avg_idle).
7367	*/
7368	static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, bool has_idle_core, int target)
7369	{
7370	struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
7371	int i, cpu, idle_cpu = -1, nr = INT_MAX;
7372	struct sched_domain_shared *sd_share;
7373
7374	cpumask_and(dstp: cpus, src1p: sched_domain_span(sd), src2p: p->cpus_ptr);
7375
7376	if (sched_feat(SIS_UTIL)) {
7377	sd_share = rcu_dereference(per_cpu(sd_llc_shared, target));
7378	if (sd_share) {
7379	/* because !--nr is the condition to stop scan */
7380	nr = READ_ONCE(sd_share->nr_idle_scan) + 1;
7381	/* overloaded LLC is unlikely to have idle cpu/core */
7382	if (nr == 1)
7383	return -1;
7384	}
7385	}
7386
7387	if (static_branch_unlikely(&sched_cluster_active)) {
7388	struct sched_group *sg = sd->groups;
7389
7390	if (sg->flags & SD_CLUSTER) {
7391	for_each_cpu_wrap(cpu, sched_group_span(sg), target + 1) {
7392	if (!cpumask_test_cpu(cpu, cpumask: cpus))
7393	continue;
7394
7395	if (has_idle_core) {
7396	i = select_idle_core(p, core: cpu, cpus, idle_cpu: &idle_cpu);
7397	if ((unsigned int)i < nr_cpumask_bits)
7398	return i;
7399	} else {
7400	if (--nr <= 0)
7401	return -1;
7402	idle_cpu = __select_idle_cpu(cpu, p);
7403	if ((unsigned int)idle_cpu < nr_cpumask_bits)
7404	return idle_cpu;
7405	}
7406	}
7407	cpumask_andnot(dstp: cpus, src1p: cpus, src2p: sched_group_span(sg));
7408	}
7409	}
7410
7411	for_each_cpu_wrap(cpu, cpus, target + 1) {
7412	if (has_idle_core) {
7413	i = select_idle_core(p, core: cpu, cpus, idle_cpu: &idle_cpu);
7414	if ((unsigned int)i < nr_cpumask_bits)
7415	return i;
7416
7417	} else {
7418	if (--nr <= 0)
7419	return -1;
7420	idle_cpu = __select_idle_cpu(cpu, p);
7421	if ((unsigned int)idle_cpu < nr_cpumask_bits)
7422	break;
7423	}
7424	}
7425
7426	if (has_idle_core)
7427	set_idle_cores(cpu: target, val: false);
7428
7429	return idle_cpu;
7430	}
7431
7432	/*
7433	* Scan the asym_capacity domain for idle CPUs; pick the first idle one on which
7434	* the task fits. If no CPU is big enough, but there are idle ones, try to
7435	* maximize capacity.
7436	*/
7437	static int
7438	select_idle_capacity(struct task_struct *p, struct sched_domain *sd, int target)
7439	{
7440	unsigned long task_util, util_min, util_max, best_cap = 0;
7441	int fits, best_fits = 0;
7442	int cpu, best_cpu = -1;
7443	struct cpumask *cpus;
7444
7445	cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
7446	cpumask_and(dstp: cpus, src1p: sched_domain_span(sd), src2p: p->cpus_ptr);
7447
7448	task_util = task_util_est(p);
7449	util_min = uclamp_eff_value(p, clamp_id: UCLAMP_MIN);
7450	util_max = uclamp_eff_value(p, clamp_id: UCLAMP_MAX);
7451
7452	for_each_cpu_wrap(cpu, cpus, target) {
7453	unsigned long cpu_cap = capacity_of(cpu);
7454
7455	if (!available_idle_cpu(cpu) && !sched_idle_cpu(cpu))
7456	continue;
7457
7458	fits = util_fits_cpu(util: task_util, uclamp_min: util_min, uclamp_max: util_max, cpu);
7459
7460	/* This CPU fits with all requirements */
7461	if (fits > 0)
7462	return cpu;
7463	/*
7464	* Only the min performance hint (i.e. uclamp_min) doesn't fit.
7465	* Look for the CPU with best capacity.
7466	*/
7467	else if (fits < 0)
7468	cpu_cap = arch_scale_cpu_capacity(cpu) - thermal_load_avg(cpu_rq(cpu));
7469
7470	/*
7471	* First, select CPU which fits better (-1 being better than 0).
7472	* Then, select the one with best capacity at same level.
7473	*/
7474	if ((fits < best_fits) ||
7475	((fits == best_fits) && (cpu_cap > best_cap))) {
7476	best_cap = cpu_cap;
7477	best_cpu = cpu;
7478	best_fits = fits;
7479	}
7480	}
7481
7482	return best_cpu;
7483	}
7484
7485	static inline bool asym_fits_cpu(unsigned long util,
7486	unsigned long util_min,
7487	unsigned long util_max,
7488	int cpu)
7489	{
7490	if (sched_asym_cpucap_active())
7491	/*
7492	* Return true only if the cpu fully fits the task requirements
7493	* which include the utilization and the performance hints.
7494	*/
7495	return (util_fits_cpu(util, uclamp_min: util_min, uclamp_max: util_max, cpu) > 0);
7496
7497	return true;
7498	}
7499
7500	/*
7501	* Try and locate an idle core/thread in the LLC cache domain.
7502	*/
7503	static int select_idle_sibling(struct task_struct *p, int prev, int target)
7504	{
7505	bool has_idle_core = false;
7506	struct sched_domain *sd;
7507	unsigned long task_util, util_min, util_max;
7508	int i, recent_used_cpu, prev_aff = -1;
7509
7510	/*
7511	* On asymmetric system, update task utilization because we will check
7512	* that the task fits with cpu's capacity.
7513	*/
7514	if (sched_asym_cpucap_active()) {
7515	sync_entity_load_avg(se: &p->se);
7516	task_util = task_util_est(p);
7517	util_min = uclamp_eff_value(p, clamp_id: UCLAMP_MIN);
7518	util_max = uclamp_eff_value(p, clamp_id: UCLAMP_MAX);
7519	}
7520
7521	/*
7522	* per-cpu select_rq_mask usage
7523	*/
7524	lockdep_assert_irqs_disabled();
7525
7526	if ((available_idle_cpu(cpu: target) || sched_idle_cpu(cpu: target)) &&
7527	asym_fits_cpu(util: task_util, util_min, util_max, cpu: target))
7528	return target;
7529
7530	/*
7531	* If the previous CPU is cache affine and idle, don't be stupid:
7532	*/
7533	if (prev != target && cpus_share_cache(this_cpu: prev, that_cpu: target) &&
7534	(available_idle_cpu(cpu: prev) || sched_idle_cpu(cpu: prev)) &&
7535	asym_fits_cpu(util: task_util, util_min, util_max, cpu: prev)) {
7536
7537	if (!static_branch_unlikely(&sched_cluster_active) ||
7538	cpus_share_resources(this_cpu: prev, that_cpu: target))
7539	return prev;
7540
7541	prev_aff = prev;
7542	}
7543
7544	/*
7545	* Allow a per-cpu kthread to stack with the wakee if the
7546	* kworker thread and the tasks previous CPUs are the same.
7547	* The assumption is that the wakee queued work for the
7548	* per-cpu kthread that is now complete and the wakeup is
7549	* essentially a sync wakeup. An obvious example of this
7550	* pattern is IO completions.
7551	*/
7552	if (is_per_cpu_kthread(current) &&
7553	in_task() &&
7554	prev == smp_processor_id() &&
7555	this_rq()->nr_running <= 1 &&
7556	asym_fits_cpu(util: task_util, util_min, util_max, cpu: prev)) {
7557	return prev;
7558	}
7559
7560	/* Check a recently used CPU as a potential idle candidate: */
7561	recent_used_cpu = p->recent_used_cpu;
7562	p->recent_used_cpu = prev;
7563	if (recent_used_cpu != prev &&
7564	recent_used_cpu != target &&
7565	cpus_share_cache(this_cpu: recent_used_cpu, that_cpu: target) &&
7566	(available_idle_cpu(cpu: recent_used_cpu) || sched_idle_cpu(cpu: recent_used_cpu)) &&
7567	cpumask_test_cpu(cpu: recent_used_cpu, cpumask: p->cpus_ptr) &&
7568	asym_fits_cpu(util: task_util, util_min, util_max, cpu: recent_used_cpu)) {
7569
7570	if (!static_branch_unlikely(&sched_cluster_active) ||
7571	cpus_share_resources(this_cpu: recent_used_cpu, that_cpu: target))
7572	return recent_used_cpu;
7573
7574	} else {
7575	recent_used_cpu = -1;
7576	}
7577
7578	/*
7579	* For asymmetric CPU capacity systems, our domain of interest is
7580	* sd_asym_cpucapacity rather than sd_llc.
7581	*/
7582	if (sched_asym_cpucap_active()) {
7583	sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, target));
7584	/*
7585	* On an asymmetric CPU capacity system where an exclusive
7586	* cpuset defines a symmetric island (i.e. one unique
7587	* capacity_orig value through the cpuset), the key will be set
7588	* but the CPUs within that cpuset will not have a domain with
7589	* SD_ASYM_CPUCAPACITY. These should follow the usual symmetric
7590	* capacity path.
7591	*/
7592	if (sd) {
7593	i = select_idle_capacity(p, sd, target);
7594	return ((unsigned)i < nr_cpumask_bits) ? i : target;
7595	}
7596	}
7597
7598	sd = rcu_dereference(per_cpu(sd_llc, target));
7599	if (!sd)
7600	return target;
7601
7602	if (sched_smt_active()) {
7603	has_idle_core = test_idle_cores(cpu: target);
7604
7605	if (!has_idle_core && cpus_share_cache(this_cpu: prev, that_cpu: target)) {
7606	i = select_idle_smt(p, sd, target: prev);
7607	if ((unsigned int)i < nr_cpumask_bits)
7608	return i;
7609	}
7610	}
7611
7612	i = select_idle_cpu(p, sd, has_idle_core, target);
7613	if ((unsigned)i < nr_cpumask_bits)
7614	return i;
7615
7616	/*
7617	* For cluster machines which have lower sharing cache like L2 or
7618	* LLC Tag, we tend to find an idle CPU in the target's cluster
7619	* first. But prev_cpu or recent_used_cpu may also be a good candidate,
7620	* use them if possible when no idle CPU found in select_idle_cpu().
7621	*/
7622	if ((unsigned int)prev_aff < nr_cpumask_bits)
7623	return prev_aff;
7624	if ((unsigned int)recent_used_cpu < nr_cpumask_bits)
7625	return recent_used_cpu;
7626
7627	return target;
7628	}
7629
7630	/**
7631	* cpu_util() - Estimates the amount of CPU capacity used by CFS tasks.
7632	* @cpu: the CPU to get the utilization for
7633	* @p: task for which the CPU utilization should be predicted or NULL
7634	* @dst_cpu: CPU @p migrates to, -1 if @p moves from @cpu or @p == NULL
7635	* @boost: 1 to enable boosting, otherwise 0
7636	*
7637	* The unit of the return value must be the same as the one of CPU capacity
7638	* so that CPU utilization can be compared with CPU capacity.
7639	*
7640	* CPU utilization is the sum of running time of runnable tasks plus the
7641	* recent utilization of currently non-runnable tasks on that CPU.
7642	* It represents the amount of CPU capacity currently used by CFS tasks in
7643	* the range [0..max CPU capacity] with max CPU capacity being the CPU
7644	* capacity at f_max.
7645	*
7646	* The estimated CPU utilization is defined as the maximum between CPU
7647	* utilization and sum of the estimated utilization of the currently
7648	* runnable tasks on that CPU. It preserves a utilization "snapshot" of
7649	* previously-executed tasks, which helps better deduce how busy a CPU will
7650	* be when a long-sleeping task wakes up. The contribution to CPU utilization
7651	* of such a task would be significantly decayed at this point of time.
7652	*
7653	* Boosted CPU utilization is defined as max(CPU runnable, CPU utilization).
7654	* CPU contention for CFS tasks can be detected by CPU runnable > CPU
7655	* utilization. Boosting is implemented in cpu_util() so that internal
7656	* users (e.g. EAS) can use it next to external users (e.g. schedutil),
7657	* latter via cpu_util_cfs_boost().
7658	*
7659	* CPU utilization can be higher than the current CPU capacity
7660	* (f_curr/f_max * max CPU capacity) or even the max CPU capacity because
7661	* of rounding errors as well as task migrations or wakeups of new tasks.
7662	* CPU utilization has to be capped to fit into the [0..max CPU capacity]
7663	* range. Otherwise a group of CPUs (CPU0 util = 121% + CPU1 util = 80%)
7664	* could be seen as over-utilized even though CPU1 has 20% of spare CPU
7665	* capacity. CPU utilization is allowed to overshoot current CPU capacity
7666	* though since this is useful for predicting the CPU capacity required
7667	* after task migrations (scheduler-driven DVFS).
7668	*
7669	* Return: (Boosted) (estimated) utilization for the specified CPU.
7670	*/
7671	static unsigned long
7672	cpu_util(int cpu, struct task_struct *p, int dst_cpu, int boost)
7673	{
7674	struct cfs_rq *cfs_rq = &cpu_rq(cpu)->cfs;
7675	unsigned long util = READ_ONCE(cfs_rq->avg.util_avg);
7676	unsigned long runnable;
7677
7678	if (boost) {
7679	runnable = READ_ONCE(cfs_rq->avg.runnable_avg);
7680	util = max(util, runnable);
7681	}
7682
7683	/*
7684	* If @dst_cpu is -1 or @p migrates from @cpu to @dst_cpu remove its
7685	* contribution. If @p migrates from another CPU to @cpu add its
7686	* contribution. In all the other cases @cpu is not impacted by the
7687	* migration so its util_avg is already correct.
7688	*/
7689	if (p && task_cpu(p) == cpu && dst_cpu != cpu)
7690	lsub_positive(&util, task_util(p));
7691	else if (p && task_cpu(p) != cpu && dst_cpu == cpu)
7692	util += task_util(p);
7693
7694	if (sched_feat(UTIL_EST)) {
7695	unsigned long util_est;
7696
7697	util_est = READ_ONCE(cfs_rq->avg.util_est);
7698
7699	/*
7700	* During wake-up @p isn't enqueued yet and doesn't contribute
7701	* to any cpu_rq(cpu)->cfs.avg.util_est.
7702	* If @dst_cpu == @cpu add it to "simulate" cpu_util after @p
7703	* has been enqueued.
7704	*
7705	* During exec (@dst_cpu = -1) @p is enqueued and does
7706	* contribute to cpu_rq(cpu)->cfs.util_est.
7707	* Remove it to "simulate" cpu_util without @p's contribution.
7708	*
7709	* Despite the task_on_rq_queued(@p) check there is still a
7710	* small window for a possible race when an exec
7711	* select_task_rq_fair() races with LB's detach_task().
7712	*
7713	* detach_task()
7714	* deactivate_task()
7715	* p->on_rq = TASK_ON_RQ_MIGRATING;
7716	* -------------------------------- A
7717	* dequeue_task() \
7718	* dequeue_task_fair() + Race Time
7719	* util_est_dequeue() /
7720	* -------------------------------- B
7721	*
7722	* The additional check "current == p" is required to further
7723	* reduce the race window.
7724	*/
7725	if (dst_cpu == cpu)
7726	util_est += _task_util_est(p);
7727	else if (p && unlikely(task_on_rq_queued(p) || current == p))
7728	lsub_positive(&util_est, _task_util_est(p));
7729
7730	util = max(util, util_est);
7731	}
7732
7733	return min(util, arch_scale_cpu_capacity(cpu));
7734	}
7735
7736	unsigned long cpu_util_cfs(int cpu)
7737	{
7738	return cpu_util(cpu, NULL, dst_cpu: -1, boost: 0);
7739	}
7740
7741	unsigned long cpu_util_cfs_boost(int cpu)
7742	{
7743	return cpu_util(cpu, NULL, dst_cpu: -1, boost: 1);
7744	}
7745
7746	/*
7747	* cpu_util_without: compute cpu utilization without any contributions from *p
7748	* @cpu: the CPU which utilization is requested
7749	* @p: the task which utilization should be discounted
7750	*
7751	* The utilization of a CPU is defined by the utilization of tasks currently
7752	* enqueued on that CPU as well as tasks which are currently sleeping after an
7753	* execution on that CPU.
7754	*
7755	* This method returns the utilization of the specified CPU by discounting the
7756	* utilization of the specified task, whenever the task is currently
7757	* contributing to the CPU utilization.
7758	*/
7759	static unsigned long cpu_util_without(int cpu, struct task_struct *p)
7760	{
7761	/* Task has no contribution or is new */
7762	if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
7763	p = NULL;
7764
7765	return cpu_util(cpu, p, dst_cpu: -1, boost: 0);
7766	}
7767
7768	/*
7769	* energy_env - Utilization landscape for energy estimation.
7770	* @task_busy_time: Utilization contribution by the task for which we test the
7771	* placement. Given by eenv_task_busy_time().
7772	* @pd_busy_time: Utilization of the whole perf domain without the task
7773	* contribution. Given by eenv_pd_busy_time().
7774	* @cpu_cap: Maximum CPU capacity for the perf domain.
7775	* @pd_cap: Entire perf domain capacity. (pd->nr_cpus * cpu_cap).
7776	*/
7777	struct energy_env {
7778	unsigned long task_busy_time;
7779	unsigned long pd_busy_time;
7780	unsigned long cpu_cap;
7781	unsigned long pd_cap;
7782	};
7783
7784	/*
7785	* Compute the task busy time for compute_energy(). This time cannot be
7786	* injected directly into effective_cpu_util() because of the IRQ scaling.
7787	* The latter only makes sense with the most recent CPUs where the task has
7788	* run.
7789	*/
7790	static inline void eenv_task_busy_time(struct energy_env *eenv,
7791	struct task_struct *p, int prev_cpu)
7792	{
7793	unsigned long busy_time, max_cap = arch_scale_cpu_capacity(cpu: prev_cpu);
7794	unsigned long irq = cpu_util_irq(cpu_rq(prev_cpu));
7795
7796	if (unlikely(irq >= max_cap))
7797	busy_time = max_cap;
7798	else
7799	busy_time = scale_irq_capacity(util: task_util_est(p), irq, max: max_cap);
7800
7801	eenv->task_busy_time = busy_time;
7802	}
7803
7804	/*
7805	* Compute the perf_domain (PD) busy time for compute_energy(). Based on the
7806	* utilization for each @pd_cpus, it however doesn't take into account
7807	* clamping since the ratio (utilization / cpu_capacity) is already enough to
7808	* scale the EM reported power consumption at the (eventually clamped)
7809	* cpu_capacity.
7810	*
7811	* The contribution of the task @p for which we want to estimate the
7812	* energy cost is removed (by cpu_util()) and must be calculated
7813	* separately (see eenv_task_busy_time). This ensures:
7814	*
7815	* - A stable PD utilization, no matter which CPU of that PD we want to place
7816	* the task on.
7817	*
7818	* - A fair comparison between CPUs as the task contribution (task_util())
7819	* will always be the same no matter which CPU utilization we rely on
7820	* (util_avg or util_est).
7821	*
7822	* Set @eenv busy time for the PD that spans @pd_cpus. This busy time can't
7823	* exceed @eenv->pd_cap.
7824	*/
7825	static inline void eenv_pd_busy_time(struct energy_env *eenv,
7826	struct cpumask *pd_cpus,
7827	struct task_struct *p)
7828	{
7829	unsigned long busy_time = 0;
7830	int cpu;
7831
7832	for_each_cpu(cpu, pd_cpus) {
7833	unsigned long util = cpu_util(cpu, p, dst_cpu: -1, boost: 0);
7834
7835	busy_time += effective_cpu_util(cpu, util_cfs: util, NULL, NULL);
7836	}
7837
7838	eenv->pd_busy_time = min(eenv->pd_cap, busy_time);
7839	}
7840
7841	/*
7842	* Compute the maximum utilization for compute_energy() when the task @p
7843	* is placed on the cpu @dst_cpu.
7844	*
7845	* Returns the maximum utilization among @eenv->cpus. This utilization can't
7846	* exceed @eenv->cpu_cap.
7847	*/
7848	static inline unsigned long
7849	eenv_pd_max_util(struct energy_env *eenv, struct cpumask *pd_cpus,
7850	struct task_struct *p, int dst_cpu)
7851	{
7852	unsigned long max_util = 0;
7853	int cpu;
7854
7855	for_each_cpu(cpu, pd_cpus) {
7856	struct task_struct *tsk = (cpu == dst_cpu) ? p : NULL;
7857	unsigned long util = cpu_util(cpu, p, dst_cpu, boost: 1);
7858	unsigned long eff_util, min, max;
7859
7860	/*
7861	* Performance domain frequency: utilization clamping
7862	* must be considered since it affects the selection
7863	* of the performance domain frequency.
7864	* NOTE: in case RT tasks are running, by default the
7865	* FREQUENCY_UTIL's utilization can be max OPP.
7866	*/
7867	eff_util = effective_cpu_util(cpu, util_cfs: util, min: &min, max: &max);
7868
7869	/* Task's uclamp can modify min and max value */
7870	if (tsk && uclamp_is_used()) {
7871	min = max(min, uclamp_eff_value(p, UCLAMP_MIN));
7872
7873	/*
7874	* If there is no active max uclamp constraint,
7875	* directly use task's one, otherwise keep max.
7876	*/
7877	if (uclamp_rq_is_idle(cpu_rq(cpu)))
7878	max = uclamp_eff_value(p, clamp_id: UCLAMP_MAX);
7879	else
7880	max = max(max, uclamp_eff_value(p, UCLAMP_MAX));
7881	}
7882
7883	eff_util = sugov_effective_cpu_perf(cpu, actual: eff_util, min, max);
7884	max_util = max(max_util, eff_util);
7885	}
7886
7887	return min(max_util, eenv->cpu_cap);
7888	}
7889
7890	/*
7891	* compute_energy(): Use the Energy Model to estimate the energy that @pd would
7892	* consume for a given utilization landscape @eenv. When @dst_cpu < 0, the task
7893	* contribution is ignored.
7894	*/
7895	static inline unsigned long
7896	compute_energy(struct energy_env *eenv, struct perf_domain *pd,
7897	struct cpumask *pd_cpus, struct task_struct *p, int dst_cpu)
7898	{
7899	unsigned long max_util = eenv_pd_max_util(eenv, pd_cpus, p, dst_cpu);
7900	unsigned long busy_time = eenv->pd_busy_time;
7901	unsigned long energy;
7902
7903	if (dst_cpu >= 0)
7904	busy_time = min(eenv->pd_cap, busy_time + eenv->task_busy_time);
7905
7906	energy = em_cpu_energy(pd: pd->em_pd, max_util, sum_util: busy_time, allowed_cpu_cap: eenv->cpu_cap);
7907
7908	trace_sched_compute_energy_tp(p, dst_cpu, energy, max_util, busy_time);
7909
7910	return energy;
7911	}
7912
7913	/*
7914	* find_energy_efficient_cpu(): Find most energy-efficient target CPU for the
7915	* waking task. find_energy_efficient_cpu() looks for the CPU with maximum
7916	* spare capacity in each performance domain and uses it as a potential
7917	* candidate to execute the task. Then, it uses the Energy Model to figure
7918	* out which of the CPU candidates is the most energy-efficient.
7919	*
7920	* The rationale for this heuristic is as follows. In a performance domain,
7921	* all the most energy efficient CPU candidates (according to the Energy
7922	* Model) are those for which we'll request a low frequency. When there are
7923	* several CPUs for which the frequency request will be the same, we don't
7924	* have enough data to break the tie between them, because the Energy Model
7925	* only includes active power costs. With this model, if we assume that
7926	* frequency requests follow utilization (e.g. using schedutil), the CPU with
7927	* the maximum spare capacity in a performance domain is guaranteed to be among
7928	* the best candidates of the performance domain.
7929	*
7930	* In practice, it could be preferable from an energy standpoint to pack
7931	* small tasks on a CPU in order to let other CPUs go in deeper idle states,
7932	* but that could also hurt our chances to go cluster idle, and we have no
7933	* ways to tell with the current Energy Model if this is actually a good
7934	* idea or not. So, find_energy_efficient_cpu() basically favors
7935	* cluster-packing, and spreading inside a cluster. That should at least be
7936	* a good thing for latency, and this is consistent with the idea that most
7937	* of the energy savings of EAS come from the asymmetry of the system, and
7938	* not so much from breaking the tie between identical CPUs. That's also the
7939	* reason why EAS is enabled in the topology code only for systems where
7940	* SD_ASYM_CPUCAPACITY is set.
7941	*
7942	* NOTE: Forkees are not accepted in the energy-aware wake-up path because
7943	* they don't have any useful utilization data yet and it's not possible to
7944	* forecast their impact on energy consumption. Consequently, they will be
7945	* placed by find_idlest_cpu() on the least loaded CPU, which might turn out
7946	* to be energy-inefficient in some use-cases. The alternative would be to
7947	* bias new tasks towards specific types of CPUs first, or to try to infer
7948	* their util_avg from the parent task, but those heuristics could hurt
7949	* other use-cases too. So, until someone finds a better way to solve this,
7950	* let's keep things simple by re-using the existing slow path.
7951	*/
7952	static int find_energy_efficient_cpu(struct task_struct *p, int prev_cpu)
7953	{
7954	struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
7955	unsigned long prev_delta = ULONG_MAX, best_delta = ULONG_MAX;
7956	unsigned long p_util_min = uclamp_is_used() ? uclamp_eff_value(p, clamp_id: UCLAMP_MIN) : 0;
7957	unsigned long p_util_max = uclamp_is_used() ? uclamp_eff_value(p, clamp_id: UCLAMP_MAX) : 1024;
7958	struct root_domain *rd = this_rq()->rd;
7959	int cpu, best_energy_cpu, target = -1;
7960	int prev_fits = -1, best_fits = -1;
7961	unsigned long best_thermal_cap = 0;
7962	unsigned long prev_thermal_cap = 0;
7963	struct sched_domain *sd;
7964	struct perf_domain *pd;
7965	struct energy_env eenv;
7966
7967	rcu_read_lock();
7968	pd = rcu_dereference(rd->pd);
7969	if (!pd || READ_ONCE(rd->overutilized))
7970	goto unlock;
7971
7972	/*
7973	* Energy-aware wake-up happens on the lowest sched_domain starting
7974	* from sd_asym_cpucapacity spanning over this_cpu and prev_cpu.
7975	*/
7976	sd = rcu_dereference(*this_cpu_ptr(&sd_asym_cpucapacity));
7977	while (sd && !cpumask_test_cpu(cpu: prev_cpu, cpumask: sched_domain_span(sd)))
7978	sd = sd->parent;
7979	if (!sd)
7980	goto unlock;
7981
7982	target = prev_cpu;
7983
7984	sync_entity_load_avg(se: &p->se);
7985	if (!task_util_est(p) && p_util_min == 0)
7986	goto unlock;
7987
7988	eenv_task_busy_time(eenv: &eenv, p, prev_cpu);
7989
7990	for (; pd; pd = pd->next) {
7991	unsigned long util_min = p_util_min, util_max = p_util_max;
7992	unsigned long cpu_cap, cpu_thermal_cap, util;
7993	long prev_spare_cap = -1, max_spare_cap = -1;
7994	unsigned long rq_util_min, rq_util_max;
7995	unsigned long cur_delta, base_energy;
7996	int max_spare_cap_cpu = -1;
7997	int fits, max_fits = -1;
7998
7999	cpumask_and(dstp: cpus, perf_domain_span(pd), cpu_online_mask);
8000
8001	if (cpumask_empty(srcp: cpus))
8002	continue;
8003
8004	/* Account thermal pressure for the energy estimation */
8005	cpu = cpumask_first(srcp: cpus);
8006	cpu_thermal_cap = arch_scale_cpu_capacity(cpu);
8007	cpu_thermal_cap -= arch_scale_thermal_pressure(cpu);
8008
8009	eenv.cpu_cap = cpu_thermal_cap;
8010	eenv.pd_cap = 0;
8011
8012	for_each_cpu(cpu, cpus) {
8013	struct rq *rq = cpu_rq(cpu);
8014
8015	eenv.pd_cap += cpu_thermal_cap;
8016
8017	if (!cpumask_test_cpu(cpu, cpumask: sched_domain_span(sd)))
8018	continue;
8019
8020	if (!cpumask_test_cpu(cpu, cpumask: p->cpus_ptr))
8021	continue;
8022
8023	util = cpu_util(cpu, p, dst_cpu: cpu, boost: 0);
8024	cpu_cap = capacity_of(cpu);
8025
8026	/*
8027	* Skip CPUs that cannot satisfy the capacity request.
8028	* IOW, placing the task there would make the CPU
8029	* overutilized. Take uclamp into account to see how
8030	* much capacity we can get out of the CPU; this is
8031	* aligned with sched_cpu_util().
8032	*/
8033	if (uclamp_is_used() && !uclamp_rq_is_idle(rq)) {
8034	/*
8035	* Open code uclamp_rq_util_with() except for
8036	* the clamp() part. Ie: apply max aggregation
8037	* only. util_fits_cpu() logic requires to
8038	* operate on non clamped util but must use the
8039	* max-aggregated uclamp_{min, max}.
8040	*/
8041	rq_util_min = uclamp_rq_get(rq, clamp_id: UCLAMP_MIN);
8042	rq_util_max = uclamp_rq_get(rq, clamp_id: UCLAMP_MAX);
8043
8044	util_min = max(rq_util_min, p_util_min);
8045	util_max = max(rq_util_max, p_util_max);
8046	}
8047
8048	fits = util_fits_cpu(util, uclamp_min: util_min, uclamp_max: util_max, cpu);
8049	if (!fits)
8050	continue;
8051
8052	lsub_positive(&cpu_cap, util);
8053
8054	if (cpu == prev_cpu) {
8055	/* Always use prev_cpu as a candidate. */
8056	prev_spare_cap = cpu_cap;
8057	prev_fits = fits;
8058	} else if ((fits > max_fits) ||
8059	((fits == max_fits) && ((long)cpu_cap > max_spare_cap))) {
8060	/*
8061	* Find the CPU with the maximum spare capacity
8062	* among the remaining CPUs in the performance
8063	* domain.
8064	*/
8065	max_spare_cap = cpu_cap;
8066	max_spare_cap_cpu = cpu;
8067	max_fits = fits;
8068	}
8069	}
8070
8071	if (max_spare_cap_cpu < 0 && prev_spare_cap < 0)
8072	continue;
8073
8074	eenv_pd_busy_time(eenv: &eenv, pd_cpus: cpus, p);
8075	/* Compute the 'base' energy of the pd, without @p */
8076	base_energy = compute_energy(eenv: &eenv, pd, pd_cpus: cpus, p, dst_cpu: -1);
8077
8078	/* Evaluate the energy impact of using prev_cpu. */
8079	if (prev_spare_cap > -1) {
8080	prev_delta = compute_energy(eenv: &eenv, pd, pd_cpus: cpus, p,
8081	dst_cpu: prev_cpu);
8082	/* CPU utilization has changed */
8083	if (prev_delta < base_energy)
8084	goto unlock;
8085	prev_delta -= base_energy;
8086	prev_thermal_cap = cpu_thermal_cap;
8087	best_delta = min(best_delta, prev_delta);
8088	}
8089
8090	/* Evaluate the energy impact of using max_spare_cap_cpu. */
8091	if (max_spare_cap_cpu >= 0 && max_spare_cap > prev_spare_cap) {
8092	/* Current best energy cpu fits better */
8093	if (max_fits < best_fits)
8094	continue;
8095
8096	/*
8097	* Both don't fit performance hint (i.e. uclamp_min)
8098	* but best energy cpu has better capacity.
8099	*/
8100	if ((max_fits < 0) &&
8101	(cpu_thermal_cap <= best_thermal_cap))
8102	continue;
8103
8104	cur_delta = compute_energy(eenv: &eenv, pd, pd_cpus: cpus, p,
8105	dst_cpu: max_spare_cap_cpu);
8106	/* CPU utilization has changed */
8107	if (cur_delta < base_energy)
8108	goto unlock;
8109	cur_delta -= base_energy;
8110
8111	/*
8112	* Both fit for the task but best energy cpu has lower
8113	* energy impact.
8114	*/
8115	if ((max_fits > 0) && (best_fits > 0) &&
8116	(cur_delta >= best_delta))
8117	continue;
8118
8119	best_delta = cur_delta;
8120	best_energy_cpu = max_spare_cap_cpu;
8121	best_fits = max_fits;
8122	best_thermal_cap = cpu_thermal_cap;
8123	}
8124	}
8125	rcu_read_unlock();
8126
8127	if ((best_fits > prev_fits) ||
8128	((best_fits > 0) && (best_delta < prev_delta)) ||
8129	((best_fits < 0) && (best_thermal_cap > prev_thermal_cap)))
8130	target = best_energy_cpu;
8131
8132	return target;
8133
8134	unlock:
8135	rcu_read_unlock();
8136
8137	return target;
8138	}
8139
8140	/*
8141	* select_task_rq_fair: Select target runqueue for the waking task in domains
8142	* that have the relevant SD flag set. In practice, this is SD_BALANCE_WAKE,
8143	* SD_BALANCE_FORK, or SD_BALANCE_EXEC.
8144	*
8145	* Balances load by selecting the idlest CPU in the idlest group, or under
8146	* certain conditions an idle sibling CPU if the domain has SD_WAKE_AFFINE set.
8147	*
8148	* Returns the target CPU number.
8149	*/
8150	static int
8151	select_task_rq_fair(struct task_struct *p, int prev_cpu, int wake_flags)
8152	{
8153	int sync = (wake_flags & WF_SYNC) && !(current->flags & PF_EXITING);
8154	struct sched_domain *tmp, *sd = NULL;
8155	int cpu = smp_processor_id();
8156	int new_cpu = prev_cpu;
8157	int want_affine = 0;
8158	/* SD_flags and WF_flags share the first nibble */
8159	int sd_flag = wake_flags & 0xF;
8160
8161	/*
8162	* required for stable ->cpus_allowed
8163	*/
8164	lockdep_assert_held(&p->pi_lock);
8165	if (wake_flags & WF_TTWU) {
8166	record_wakee(p);
8167
8168	if ((wake_flags & WF_CURRENT_CPU) &&
8169	cpumask_test_cpu(cpu, cpumask: p->cpus_ptr))
8170	return cpu;
8171
8172	if (sched_energy_enabled()) {
8173	new_cpu = find_energy_efficient_cpu(p, prev_cpu);
8174	if (new_cpu >= 0)
8175	return new_cpu;
8176	new_cpu = prev_cpu;
8177	}
8178
8179	want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, cpumask: p->cpus_ptr);
8180	}
8181
8182	rcu_read_lock();
8183	for_each_domain(cpu, tmp) {
8184	/*
8185	* If both 'cpu' and 'prev_cpu' are part of this domain,
8186	* cpu is a valid SD_WAKE_AFFINE target.
8187	*/
8188	if (want_affine && (tmp->flags & SD_WAKE_AFFINE) &&
8189	cpumask_test_cpu(cpu: prev_cpu, cpumask: sched_domain_span(sd: tmp))) {
8190	if (cpu != prev_cpu)
8191	new_cpu = wake_affine(sd: tmp, p, this_cpu: cpu, prev_cpu, sync);
8192
8193	sd = NULL; /* Prefer wake_affine over balance flags */
8194	break;
8195	}
8196
8197	/*
8198	* Usually only true for WF_EXEC and WF_FORK, as sched_domains
8199	* usually do not have SD_BALANCE_WAKE set. That means wakeup
8200	* will usually go to the fast path.
8201	*/
8202	if (tmp->flags & sd_flag)
8203	sd = tmp;
8204	else if (!want_affine)
8205	break;
8206	}
8207
8208	if (unlikely(sd)) {
8209	/* Slow path */
8210	new_cpu = find_idlest_cpu(sd, p, cpu, prev_cpu, sd_flag);
8211	} else if (wake_flags & WF_TTWU) { /* XXX always ? */
8212	/* Fast path */
8213	new_cpu = select_idle_sibling(p, prev: prev_cpu, target: new_cpu);
8214	}
8215	rcu_read_unlock();
8216
8217	return new_cpu;
8218	}
8219
8220	/*
8221	* Called immediately before a task is migrated to a new CPU; task_cpu(p) and
8222	* cfs_rq_of(p) references at time of call are still valid and identify the
8223	* previous CPU. The caller guarantees p->pi_lock or task_rq(p)->lock is held.
8224	*/
8225	static void migrate_task_rq_fair(struct task_struct *p, int new_cpu)
8226	{
8227	struct sched_entity *se = &p->se;
8228
8229	if (!task_on_rq_migrating(p)) {
8230	remove_entity_load_avg(se);
8231
8232	/*
8233	* Here, the task's PELT values have been updated according to
8234	* the current rq's clock. But if that clock hasn't been
8235	* updated in a while, a substantial idle time will be missed,
8236	* leading to an inflation after wake-up on the new rq.
8237	*
8238	* Estimate the missing time from the cfs_rq last_update_time
8239	* and update sched_avg to improve the PELT continuity after
8240	* migration.
8241	*/
8242	migrate_se_pelt_lag(se);
8243	}
8244
8245	/* Tell new CPU we are migrated */
8246	se->avg.last_update_time = 0;
8247
8248	update_scan_period(p, new_cpu);
8249	}
8250
8251	static void task_dead_fair(struct task_struct *p)
8252	{
8253	remove_entity_load_avg(se: &p->se);
8254	}
8255
8256	static int
8257	balance_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
8258	{
8259	if (rq->nr_running)
8260	return 1;
8261
8262	return newidle_balance(this_rq: rq, rf) != 0;
8263	}
8264	#endif /* CONFIG_SMP */
8265
8266	static void set_next_buddy(struct sched_entity *se)
8267	{
8268	for_each_sched_entity(se) {
8269	if (SCHED_WARN_ON(!se->on_rq))
8270	return;
8271	if (se_is_idle(se))
8272	return;
8273	cfs_rq_of(se)->next = se;
8274	}
8275	}
8276
8277	/*
8278	* Preempt the current task with a newly woken task if needed:
8279	*/
8280	static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int wake_flags)
8281	{
8282	struct task_struct *curr = rq->curr;
8283	struct sched_entity *se = &curr->se, *pse = &p->se;
8284	struct cfs_rq *cfs_rq = task_cfs_rq(p: curr);
8285	int cse_is_idle, pse_is_idle;
8286
8287	if (unlikely(se == pse))
8288	return;
8289
8290	/*
8291	* This is possible from callers such as attach_tasks(), in which we
8292	* unconditionally wakeup_preempt() after an enqueue (which may have
8293	* lead to a throttle). This both saves work and prevents false
8294	* next-buddy nomination below.
8295	*/
8296	if (unlikely(throttled_hierarchy(cfs_rq_of(pse))))
8297	return;
8298
8299	if (sched_feat(NEXT_BUDDY) && !(wake_flags & WF_FORK)) {
8300	set_next_buddy(pse);
8301	}
8302
8303	/*
8304	* We can come here with TIF_NEED_RESCHED already set from new task
8305	* wake up path.
8306	*
8307	* Note: this also catches the edge-case of curr being in a throttled
8308	* group (e.g. via set_curr_task), since update_curr() (in the
8309	* enqueue of curr) will have resulted in resched being set. This
8310	* prevents us from potentially nominating it as a false LAST_BUDDY
8311	* below.
8312	*/
8313	if (test_tsk_need_resched(tsk: curr))
8314	return;
8315
8316	/* Idle tasks are by definition preempted by non-idle tasks. */
8317	if (unlikely(task_has_idle_policy(curr)) &&
8318	likely(!task_has_idle_policy(p)))
8319	goto preempt;
8320
8321	/*
8322	* Batch and idle tasks do not preempt non-idle tasks (their preemption
8323	* is driven by the tick):
8324	*/
8325	if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION))
8326	return;
8327
8328	find_matching_se(se: &se, pse: &pse);
8329	WARN_ON_ONCE(!pse);
8330
8331	cse_is_idle = se_is_idle(se);
8332	pse_is_idle = se_is_idle(se: pse);
8333
8334	/*
8335	* Preempt an idle group in favor of a non-idle group (and don't preempt
8336	* in the inverse case).
8337	*/
8338	if (cse_is_idle && !pse_is_idle)
8339	goto preempt;
8340	if (cse_is_idle != pse_is_idle)
8341	return;
8342
8343	cfs_rq = cfs_rq_of(se);
8344	update_curr(cfs_rq);
8345
8346	/*
8347	* XXX pick_eevdf(cfs_rq) != se ?
8348	*/
8349	if (pick_eevdf(cfs_rq) == pse)
8350	goto preempt;
8351
8352	return;
8353
8354	preempt:
8355	resched_curr(rq);
8356	}
8357
8358	#ifdef CONFIG_SMP
8359	static struct task_struct *pick_task_fair(struct rq *rq)
8360	{
8361	struct sched_entity *se;
8362	struct cfs_rq *cfs_rq;
8363
8364	again:
8365	cfs_rq = &rq->cfs;
8366	if (!cfs_rq->nr_running)
8367	return NULL;
8368
8369	do {
8370	struct sched_entity *curr = cfs_rq->curr;
8371
8372	/* When we pick for a remote RQ, we'll not have done put_prev_entity() */
8373	if (curr) {
8374	if (curr->on_rq)
8375	update_curr(cfs_rq);
8376	else
8377	curr = NULL;
8378
8379	if (unlikely(check_cfs_rq_runtime(cfs_rq)))
8380	goto again;
8381	}
8382
8383	se = pick_next_entity(cfs_rq);
8384	cfs_rq = group_cfs_rq(grp: se);
8385	} while (cfs_rq);
8386
8387	return task_of(se);
8388	}
8389	#endif
8390
8391	struct task_struct *
8392	pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
8393	{
8394	struct cfs_rq *cfs_rq = &rq->cfs;
8395	struct sched_entity *se;
8396	struct task_struct *p;
8397	int new_tasks;
8398
8399	again:
8400	if (!sched_fair_runnable(rq))
8401	goto idle;
8402
8403	#ifdef CONFIG_FAIR_GROUP_SCHED
8404	if (!prev || prev->sched_class != &fair_sched_class)
8405	goto simple;
8406
8407	/*
8408	* Because of the set_next_buddy() in dequeue_task_fair() it is rather
8409	* likely that a next task is from the same cgroup as the current.
8410	*
8411	* Therefore attempt to avoid putting and setting the entire cgroup
8412	* hierarchy, only change the part that actually changes.
8413	*/
8414
8415	do {
8416	struct sched_entity *curr = cfs_rq->curr;
8417
8418	/*
8419	* Since we got here without doing put_prev_entity() we also
8420	* have to consider cfs_rq->curr. If it is still a runnable
8421	* entity, update_curr() will update its vruntime, otherwise
8422	* forget we've ever seen it.
8423	*/
8424	if (curr) {
8425	if (curr->on_rq)
8426	update_curr(cfs_rq);
8427	else
8428	curr = NULL;
8429
8430	/*
8431	* This call to check_cfs_rq_runtime() will do the
8432	* throttle and dequeue its entity in the parent(s).
8433	* Therefore the nr_running test will indeed
8434	* be correct.
8435	*/
8436	if (unlikely(check_cfs_rq_runtime(cfs_rq))) {
8437	cfs_rq = &rq->cfs;
8438
8439	if (!cfs_rq->nr_running)
8440	goto idle;
8441
8442	goto simple;
8443	}
8444	}
8445
8446	se = pick_next_entity(cfs_rq);
8447	cfs_rq = group_cfs_rq(grp: se);
8448	} while (cfs_rq);
8449
8450	p = task_of(se);
8451
8452	/*
8453	* Since we haven't yet done put_prev_entity and if the selected task
8454	* is a different task than we started out with, try and touch the
8455	* least amount of cfs_rqs.
8456	*/
8457	if (prev != p) {
8458	struct sched_entity *pse = &prev->se;
8459
8460	while (!(cfs_rq = is_same_group(se, pse))) {
8461	int se_depth = se->depth;
8462	int pse_depth = pse->depth;
8463
8464	if (se_depth <= pse_depth) {
8465	put_prev_entity(cfs_rq: cfs_rq_of(se: pse), prev: pse);
8466	pse = parent_entity(se: pse);
8467	}
8468	if (se_depth >= pse_depth) {
8469	set_next_entity(cfs_rq: cfs_rq_of(se), se);
8470	se = parent_entity(se);
8471	}
8472	}
8473
8474	put_prev_entity(cfs_rq, prev: pse);
8475	set_next_entity(cfs_rq, se);
8476	}
8477
8478	goto done;
8479	simple:
8480	#endif
8481	if (prev)
8482	put_prev_task(rq, prev);
8483
8484	do {
8485	se = pick_next_entity(cfs_rq);
8486	set_next_entity(cfs_rq, se);
8487	cfs_rq = group_cfs_rq(grp: se);
8488	} while (cfs_rq);
8489
8490	p = task_of(se);
8491
8492	done: __maybe_unused;
8493	#ifdef CONFIG_SMP
8494	/*
8495	* Move the next running task to the front of
8496	* the list, so our cfs_tasks list becomes MRU
8497	* one.
8498	*/
8499	list_move(list: &p->se.group_node, head: &rq->cfs_tasks);
8500	#endif
8501
8502	if (hrtick_enabled_fair(rq))
8503	hrtick_start_fair(rq, p);
8504
8505	update_misfit_status(p, rq);
8506	sched_fair_update_stop_tick(rq, p);
8507
8508	return p;
8509
8510	idle:
8511	if (!rf)
8512	return NULL;
8513
8514	new_tasks = newidle_balance(this_rq: rq, rf);
8515
8516	/*
8517	* Because newidle_balance() releases (and re-acquires) rq->lock, it is
8518	* possible for any higher priority task to appear. In that case we
8519	* must re-start the pick_next_entity() loop.
8520	*/
8521	if (new_tasks < 0)
8522	return RETRY_TASK;
8523
8524	if (new_tasks > 0)
8525	goto again;
8526
8527	/*
8528	* rq is about to be idle, check if we need to update the
8529	* lost_idle_time of clock_pelt
8530	*/
8531	update_idle_rq_clock_pelt(rq);
8532
8533	return NULL;
8534	}
8535
8536	static struct task_struct *__pick_next_task_fair(struct rq *rq)
8537	{
8538	return pick_next_task_fair(rq, NULL, NULL);
8539	}
8540
8541	/*
8542	* Account for a descheduled task:
8543	*/
8544	static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
8545	{
8546	struct sched_entity *se = &prev->se;
8547	struct cfs_rq *cfs_rq;
8548
8549	for_each_sched_entity(se) {
8550	cfs_rq = cfs_rq_of(se);
8551	put_prev_entity(cfs_rq, prev: se);
8552	}
8553	}
8554
8555	/*
8556	* sched_yield() is very simple
8557	*/
8558	static void yield_task_fair(struct rq *rq)
8559	{
8560	struct task_struct *curr = rq->curr;
8561	struct cfs_rq *cfs_rq = task_cfs_rq(p: curr);
8562	struct sched_entity *se = &curr->se;
8563
8564	/*
8565	* Are we the only task in the tree?
8566	*/
8567	if (unlikely(rq->nr_running == 1))
8568	return;
8569
8570	clear_buddies(cfs_rq, se);
8571
8572	update_rq_clock(rq);
8573	/*
8574	* Update run-time statistics of the 'current'.
8575	*/
8576	update_curr(cfs_rq);
8577	/*
8578	* Tell update_rq_clock() that we've just updated,
8579	* so we don't do microscopic update in schedule()
8580	* and double the fastpath cost.
8581	*/
8582	rq_clock_skip_update(rq);
8583
8584	se->deadline += calc_delta_fair(delta: se->slice, se);
8585	}
8586
8587	static bool yield_to_task_fair(struct rq *rq, struct task_struct *p)
8588	{
8589	struct sched_entity *se = &p->se;
8590
8591	/* throttled hierarchies are not runnable */
8592	if (!se->on_rq || throttled_hierarchy(cfs_rq: cfs_rq_of(se)))
8593	return false;
8594
8595	/* Tell the scheduler that we'd really like pse to run next. */
8596	set_next_buddy(se);
8597
8598	yield_task_fair(rq);
8599
8600	return true;
8601	}
8602
8603	#ifdef CONFIG_SMP
8604	/**************************************************
8605	* Fair scheduling class load-balancing methods.
8606	*
8607	* BASICS
8608	*
8609	* The purpose of load-balancing is to achieve the same basic fairness the
8610	* per-CPU scheduler provides, namely provide a proportional amount of compute
8611	* time to each task. This is expressed in the following equation:
8612	*
8613	* W_i,n/P_i == W_j,n/P_j for all i,j (1)
8614	*
8615	* Where W_i,n is the n-th weight average for CPU i. The instantaneous weight
8616	* W_i,0 is defined as:
8617	*
8618	* W_i,0 = \Sum_j w_i,j (2)
8619	*
8620	* Where w_i,j is the weight of the j-th runnable task on CPU i. This weight
8621	* is derived from the nice value as per sched_prio_to_weight[].
8622	*
8623	* The weight average is an exponential decay average of the instantaneous
8624	* weight:
8625	*
8626	* W'_i,n = (2^n - 1) / 2^n * W_i,n + 1 / 2^n * W_i,0 (3)
8627	*
8628	* C_i is the compute capacity of CPU i, typically it is the
8629	* fraction of 'recent' time available for SCHED_OTHER task execution. But it
8630	* can also include other factors [XXX].
8631	*
8632	* To achieve this balance we define a measure of imbalance which follows
8633	* directly from (1):
8634	*
8635	* imb_i,j = max{ avg(W/C), W_i/C_i } - min{ avg(W/C), W_j/C_j } (4)
8636	*
8637	* We them move tasks around to minimize the imbalance. In the continuous
8638	* function space it is obvious this converges, in the discrete case we get
8639	* a few fun cases generally called infeasible weight scenarios.
8640	*
8641	* [XXX expand on:
8642	* - infeasible weights;
8643	* - local vs global optima in the discrete case. ]
8644	*
8645	*
8646	* SCHED DOMAINS
8647	*
8648	* In order to solve the imbalance equation (4), and avoid the obvious O(n^2)
8649	* for all i,j solution, we create a tree of CPUs that follows the hardware
8650	* topology where each level pairs two lower groups (or better). This results
8651	* in O(log n) layers. Furthermore we reduce the number of CPUs going up the
8652	* tree to only the first of the previous level and we decrease the frequency
8653	* of load-balance at each level inv. proportional to the number of CPUs in
8654	* the groups.
8655	*
8656	* This yields:
8657	*
8658	* log_2 n 1 n
8659	* \Sum { --- * --- * 2^i } = O(n) (5)
8660	* i = 0 2^i 2^i
8661	* `- size of each group
8662	* | | `- number of CPUs doing load-balance
8663	* | `- freq
8664	* `- sum over all levels
8665	*
8666	* Coupled with a limit on how many tasks we can migrate every balance pass,
8667	* this makes (5) the runtime complexity of the balancer.
8668	*
8669	* An important property here is that each CPU is still (indirectly) connected
8670	* to every other CPU in at most O(log n) steps:
8671	*
8672	* The adjacency matrix of the resulting graph is given by:
8673	*
8674	* log_2 n
8675	* A_i,j = \Union (i % 2^k == 0) && i / 2^(k+1) == j / 2^(k+1) (6)
8676	* k = 0
8677	*
8678	* And you'll find that:
8679	*
8680	* A^(log_2 n)_i,j != 0 for all i,j (7)
8681	*
8682	* Showing there's indeed a path between every CPU in at most O(log n) steps.
8683	* The task movement gives a factor of O(m), giving a convergence complexity
8684	* of:
8685	*
8686	* O(nm log n), n := nr_cpus, m := nr_tasks (8)
8687	*
8688	*
8689	* WORK CONSERVING
8690	*
8691	* In order to avoid CPUs going idle while there's still work to do, new idle
8692	* balancing is more aggressive and has the newly idle CPU iterate up the domain
8693	* tree itself instead of relying on other CPUs to bring it work.
8694	*
8695	* This adds some complexity to both (5) and (8) but it reduces the total idle
8696	* time.
8697	*
8698	* [XXX more?]
8699	*
8700	*
8701	* CGROUPS
8702	*
8703	* Cgroups make a horror show out of (2), instead of a simple sum we get:
8704	*
8705	* s_k,i
8706	* W_i,0 = \Sum_j \Prod_k w_k * ----- (9)
8707	* S_k
8708	*
8709	* Where
8710	*
8711	* s_k,i = \Sum_j w_i,j,k and S_k = \Sum_i s_k,i (10)
8712	*
8713	* w_i,j,k is the weight of the j-th runnable task in the k-th cgroup on CPU i.
8714	*
8715	* The big problem is S_k, its a global sum needed to compute a local (W_i)
8716	* property.
8717	*
8718	* [XXX write more on how we solve this.. _after_ merging pjt's patches that
8719	* rewrite all of this once again.]
8720	*/
8721
8722	static unsigned long __read_mostly max_load_balance_interval = HZ/10;
8723
8724	enum fbq_type { regular, remote, all };
8725
8726	/*
8727	* 'group_type' describes the group of CPUs at the moment of load balancing.
8728	*
8729	* The enum is ordered by pulling priority, with the group with lowest priority
8730	* first so the group_type can simply be compared when selecting the busiest
8731	* group. See update_sd_pick_busiest().
8732	*/
8733	enum group_type {
8734	/* The group has spare capacity that can be used to run more tasks. */
8735	group_has_spare = 0,
8736	/*
8737	* The group is fully used and the tasks don't compete for more CPU
8738	* cycles. Nevertheless, some tasks might wait before running.
8739	*/
8740	group_fully_busy,
8741	/*
8742	* One task doesn't fit with CPU's capacity and must be migrated to a
8743	* more powerful CPU.
8744	*/
8745	group_misfit_task,
8746	/*
8747	* Balance SMT group that's fully busy. Can benefit from migration
8748	* a task on SMT with busy sibling to another CPU on idle core.
8749	*/
8750	group_smt_balance,
8751	/*
8752	* SD_ASYM_PACKING only: One local CPU with higher capacity is available,
8753	* and the task should be migrated to it instead of running on the
8754	* current CPU.
8755	*/
8756	group_asym_packing,
8757	/*
8758	* The tasks' affinity constraints previously prevented the scheduler
8759	* from balancing the load across the system.
8760	*/
8761	group_imbalanced,
8762	/*
8763	* The CPU is overloaded and can't provide expected CPU cycles to all
8764	* tasks.
8765	*/
8766	group_overloaded
8767	};
8768
8769	enum migration_type {
8770	migrate_load = 0,
8771	migrate_util,
8772	migrate_task,
8773	migrate_misfit
8774	};
8775
8776	#define LBF_ALL_PINNED 0x01
8777	#define LBF_NEED_BREAK 0x02
8778	#define LBF_DST_PINNED 0x04
8779	#define LBF_SOME_PINNED 0x08
8780	#define LBF_ACTIVE_LB 0x10
8781
8782	struct lb_env {
8783	struct sched_domain *sd;
8784
8785	struct rq *src_rq;
8786	int src_cpu;
8787
8788	int dst_cpu;
8789	struct rq *dst_rq;
8790
8791	struct cpumask *dst_grpmask;
8792	int new_dst_cpu;
8793	enum cpu_idle_type idle;
8794	long imbalance;
8795	/* The set of CPUs under consideration for load-balancing */
8796	struct cpumask *cpus;
8797
8798	unsigned int flags;
8799
8800	unsigned int loop;
8801	unsigned int loop_break;
8802	unsigned int loop_max;
8803
8804	enum fbq_type fbq_type;
8805	enum migration_type migration_type;
8806	struct list_head tasks;
8807	};
8808
8809	/*
8810	* Is this task likely cache-hot:
8811	*/
8812	static int task_hot(struct task_struct *p, struct lb_env *env)
8813	{
8814	s64 delta;
8815
8816	lockdep_assert_rq_held(rq: env->src_rq);
8817
8818	if (p->sched_class != &fair_sched_class)
8819	return 0;
8820
8821	if (unlikely(task_has_idle_policy(p)))
8822	return 0;
8823
8824	/* SMT siblings share cache */
8825	if (env->sd->flags & SD_SHARE_CPUCAPACITY)
8826	return 0;
8827
8828	/*
8829	* Buddy candidates are cache hot:
8830	*/
8831	if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&
8832	(&p->se == cfs_rq_of(se: &p->se)->next))
8833	return 1;
8834
8835	if (sysctl_sched_migration_cost == -1)
8836	return 1;
8837
8838	/*
8839	* Don't migrate task if the task's cookie does not match
8840	* with the destination CPU's core cookie.
8841	*/
8842	if (!sched_core_cookie_match(cpu_rq(env->dst_cpu), p))
8843	return 1;
8844
8845	if (sysctl_sched_migration_cost == 0)
8846	return 0;
8847
8848	delta = rq_clock_task(rq: env->src_rq) - p->se.exec_start;
8849
8850	return delta < (s64)sysctl_sched_migration_cost;
8851	}
8852
8853	#ifdef CONFIG_NUMA_BALANCING
8854	/*
8855	* Returns 1, if task migration degrades locality
8856	* Returns 0, if task migration improves locality i.e migration preferred.
8857	* Returns -1, if task migration is not affected by locality.
8858	*/
8859	static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
8860	{
8861	struct numa_group *numa_group = rcu_dereference(p->numa_group);
8862	unsigned long src_weight, dst_weight;
8863	int src_nid, dst_nid, dist;
8864
8865	if (!static_branch_likely(&sched_numa_balancing))
8866	return -1;
8867
8868	if (!p->numa_faults || !(env->sd->flags & SD_NUMA))
8869	return -1;
8870
8871	src_nid = cpu_to_node(cpu: env->src_cpu);
8872	dst_nid = cpu_to_node(cpu: env->dst_cpu);
8873
8874	if (src_nid == dst_nid)
8875	return -1;
8876
8877	/* Migrating away from the preferred node is always bad. */
8878	if (src_nid == p->numa_preferred_nid) {
8879	if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)
8880	return 1;
8881	else
8882	return -1;
8883	}
8884
8885	/* Encourage migration to the preferred node. */
8886	if (dst_nid == p->numa_preferred_nid)
8887	return 0;
8888
8889	/* Leaving a core idle is often worse than degrading locality. */
8890	if (env->idle == CPU_IDLE)
8891	return -1;
8892
8893	dist = node_distance(src_nid, dst_nid);
8894	if (numa_group) {
8895	src_weight = group_weight(p, nid: src_nid, dist);
8896	dst_weight = group_weight(p, nid: dst_nid, dist);
8897	} else {
8898	src_weight = task_weight(p, nid: src_nid, dist);
8899	dst_weight = task_weight(p, nid: dst_nid, dist);
8900	}
8901
8902	return dst_weight < src_weight;
8903	}
8904
8905	#else
8906	static inline int migrate_degrades_locality(struct task_struct *p,
8907	struct lb_env *env)
8908	{
8909	return -1;
8910	}
8911	#endif
8912
8913	/*
8914	* can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
8915	*/
8916	static
8917	int can_migrate_task(struct task_struct *p, struct lb_env *env)
8918	{
8919	int tsk_cache_hot;
8920
8921	lockdep_assert_rq_held(rq: env->src_rq);
8922
8923	/*
8924	* We do not migrate tasks that are:
8925	* 1) throttled_lb_pair, or
8926	* 2) cannot be migrated to this CPU due to cpus_ptr, or
8927	* 3) running (obviously), or
8928	* 4) are cache-hot on their current CPU.
8929	*/
8930	if (throttled_lb_pair(tg: task_group(p), src_cpu: env->src_cpu, dest_cpu: env->dst_cpu))
8931	return 0;
8932
8933	/* Disregard pcpu kthreads; they are where they need to be. */
8934	if (kthread_is_per_cpu(k: p))
8935	return 0;
8936
8937	if (!cpumask_test_cpu(cpu: env->dst_cpu, cpumask: p->cpus_ptr)) {
8938	int cpu;
8939
8940	schedstat_inc(p->stats.nr_failed_migrations_affine);
8941
8942	env->flags |= LBF_SOME_PINNED;
8943
8944	/*
8945	* Remember if this task can be migrated to any other CPU in
8946	* our sched_group. We may want to revisit it if we couldn't
8947	* meet load balance goals by pulling other tasks on src_cpu.
8948	*
8949	* Avoid computing new_dst_cpu
8950	* - for NEWLY_IDLE
8951	* - if we have already computed one in current iteration
8952	* - if it's an active balance
8953	*/
8954	if (env->idle == CPU_NEWLY_IDLE ||
8955	env->flags & (LBF_DST_PINNED | LBF_ACTIVE_LB))
8956	return 0;
8957
8958	/* Prevent to re-select dst_cpu via env's CPUs: */
8959	for_each_cpu_and(cpu, env->dst_grpmask, env->cpus) {
8960	if (cpumask_test_cpu(cpu, cpumask: p->cpus_ptr)) {
8961	env->flags |= LBF_DST_PINNED;
8962	env->new_dst_cpu = cpu;
8963	break;
8964	}
8965	}
8966
8967	return 0;
8968	}
8969
8970	/* Record that we found at least one task that could run on dst_cpu */
8971	env->flags &= ~LBF_ALL_PINNED;
8972
8973	if (task_on_cpu(rq: env->src_rq, p)) {
8974	schedstat_inc(p->stats.nr_failed_migrations_running);
8975	return 0;
8976	}
8977
8978	/*
8979	* Aggressive migration if:
8980	* 1) active balance
8981	* 2) destination numa is preferred
8982	* 3) task is cache cold, or
8983	* 4) too many balance attempts have failed.
8984	*/
8985	if (env->flags & LBF_ACTIVE_LB)
8986	return 1;
8987
8988	tsk_cache_hot = migrate_degrades_locality(p, env);
8989	if (tsk_cache_hot == -1)
8990	tsk_cache_hot = task_hot(p, env);
8991
8992	if (tsk_cache_hot <= 0 ||
8993	env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
8994	if (tsk_cache_hot == 1) {
8995	schedstat_inc(env->sd->lb_hot_gained[env->idle]);
8996	schedstat_inc(p->stats.nr_forced_migrations);
8997	}
8998	return 1;
8999	}
9000
9001	schedstat_inc(p->stats.nr_failed_migrations_hot);
9002	return 0;
9003	}
9004
9005	/*
9006	* detach_task() -- detach the task for the migration specified in env
9007	*/
9008	static void detach_task(struct task_struct *p, struct lb_env *env)
9009	{
9010	lockdep_assert_rq_held(rq: env->src_rq);
9011
9012	deactivate_task(rq: env->src_rq, p, DEQUEUE_NOCLOCK);
9013	set_task_cpu(p, cpu: env->dst_cpu);
9014	}
9015
9016	/*
9017	* detach_one_task() -- tries to dequeue exactly one task from env->src_rq, as
9018	* part of active balancing operations within "domain".
9019	*
9020	* Returns a task if successful and NULL otherwise.
9021	*/
9022	static struct task_struct *detach_one_task(struct lb_env *env)
9023	{
9024	struct task_struct *p;
9025
9026	lockdep_assert_rq_held(rq: env->src_rq);
9027
9028	list_for_each_entry_reverse(p,
9029	&env->src_rq->cfs_tasks, se.group_node) {
9030	if (!can_migrate_task(p, env))
9031	continue;
9032
9033	detach_task(p, env);
9034
9035	/*
9036	* Right now, this is only the second place where
9037	* lb_gained[env->idle] is updated (other is detach_tasks)
9038	* so we can safely collect stats here rather than
9039	* inside detach_tasks().
9040	*/
9041	schedstat_inc(env->sd->lb_gained[env->idle]);
9042	return p;
9043	}
9044	return NULL;
9045	}
9046
9047	/*
9048	* detach_tasks() -- tries to detach up to imbalance load/util/tasks from
9049	* busiest_rq, as part of a balancing operation within domain "sd".
9050	*
9051	* Returns number of detached tasks if successful and 0 otherwise.
9052	*/
9053	static int detach_tasks(struct lb_env *env)
9054	{
9055	struct list_head *tasks = &env->src_rq->cfs_tasks;
9056	unsigned long util, load;
9057	struct task_struct *p;
9058	int detached = 0;
9059
9060	lockdep_assert_rq_held(rq: env->src_rq);
9061
9062	/*
9063	* Source run queue has been emptied by another CPU, clear
9064	* LBF_ALL_PINNED flag as we will not test any task.
9065	*/
9066	if (env->src_rq->nr_running <= 1) {
9067	env->flags &= ~LBF_ALL_PINNED;
9068	return 0;
9069	}
9070
9071	if (env->imbalance <= 0)
9072	return 0;
9073
9074	while (!list_empty(head: tasks)) {
9075	/*
9076	* We don't want to steal all, otherwise we may be treated likewise,
9077	* which could at worst lead to a livelock crash.
9078	*/
9079	if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1)
9080	break;
9081
9082	env->loop++;
9083	/*
9084	* We've more or less seen every task there is, call it quits
9085	* unless we haven't found any movable task yet.
9086	*/
9087	if (env->loop > env->loop_max &&
9088	!(env->flags & LBF_ALL_PINNED))
9089	break;
9090
9091	/* take a breather every nr_migrate tasks */
9092	if (env->loop > env->loop_break) {
9093	env->loop_break += SCHED_NR_MIGRATE_BREAK;
9094	env->flags |= LBF_NEED_BREAK;
9095	break;
9096	}
9097
9098	p = list_last_entry(tasks, struct task_struct, se.group_node);
9099
9100	if (!can_migrate_task(p, env))
9101	goto next;
9102
9103	switch (env->migration_type) {
9104	case migrate_load:
9105	/*
9106	* Depending of the number of CPUs and tasks and the
9107	* cgroup hierarchy, task_h_load() can return a null
9108	* value. Make sure that env->imbalance decreases
9109	* otherwise detach_tasks() will stop only after
9110	* detaching up to loop_max tasks.
9111	*/
9112	load = max_t(unsigned long, task_h_load(p), 1);
9113
9114	if (sched_feat(LB_MIN) &&
9115	load < 16 && !env->sd->nr_balance_failed)
9116	goto next;
9117
9118	/*
9119	* Make sure that we don't migrate too much load.
9120	* Nevertheless, let relax the constraint if
9121	* scheduler fails to find a good waiting task to
9122	* migrate.
9123	*/
9124	if (shr_bound(load, env->sd->nr_balance_failed) > env->imbalance)
9125	goto next;
9126
9127	env->imbalance -= load;
9128	break;
9129
9130	case migrate_util:
9131	util = task_util_est(p);
9132
9133	if (shr_bound(util, env->sd->nr_balance_failed) > env->imbalance)
9134	goto next;
9135
9136	env->imbalance -= util;
9137	break;
9138
9139	case migrate_task:
9140	env->imbalance--;
9141	break;
9142
9143	case migrate_misfit:
9144	/* This is not a misfit task */
9145	if (task_fits_cpu(p, cpu: env->src_cpu))
9146	goto next;
9147
9148	env->imbalance = 0;
9149	break;
9150	}
9151
9152	detach_task(p, env);
9153	list_add(new: &p->se.group_node, head: &env->tasks);
9154
9155	detached++;
9156
9157	#ifdef CONFIG_PREEMPTION
9158	/*
9159	* NEWIDLE balancing is a source of latency, so preemptible
9160	* kernels will stop after the first task is detached to minimize
9161	* the critical section.
9162	*/
9163	if (env->idle == CPU_NEWLY_IDLE)
9164	break;
9165	#endif
9166
9167	/*
9168	* We only want to steal up to the prescribed amount of
9169	* load/util/tasks.
9170	*/
9171	if (env->imbalance <= 0)
9172	break;
9173
9174	continue;
9175	next:
9176	list_move(list: &p->se.group_node, head: tasks);
9177	}
9178
9179	/*
9180	* Right now, this is one of only two places we collect this stat
9181	* so we can safely collect detach_one_task() stats here rather
9182	* than inside detach_one_task().
9183	*/
9184	schedstat_add(env->sd->lb_gained[env->idle], detached);
9185
9186	return detached;
9187	}
9188
9189	/*
9190	* attach_task() -- attach the task detached by detach_task() to its new rq.
9191	*/
9192	static void attach_task(struct rq *rq, struct task_struct *p)
9193	{
9194	lockdep_assert_rq_held(rq);
9195
9196	WARN_ON_ONCE(task_rq(p) != rq);
9197	activate_task(rq, p, ENQUEUE_NOCLOCK);
9198	wakeup_preempt(rq, p, flags: 0);
9199	}
9200
9201	/*
9202	* attach_one_task() -- attaches the task returned from detach_one_task() to
9203	* its new rq.
9204	*/
9205	static void attach_one_task(struct rq *rq, struct task_struct *p)
9206	{
9207	struct rq_flags rf;
9208
9209	rq_lock(rq, rf: &rf);
9210	update_rq_clock(rq);
9211	attach_task(rq, p);
9212	rq_unlock(rq, rf: &rf);
9213	}
9214
9215	/*
9216	* attach_tasks() -- attaches all tasks detached by detach_tasks() to their
9217	* new rq.
9218	*/
9219	static void attach_tasks(struct lb_env *env)
9220	{
9221	struct list_head *tasks = &env->tasks;
9222	struct task_struct *p;
9223	struct rq_flags rf;
9224
9225	rq_lock(rq: env->dst_rq, rf: &rf);
9226	update_rq_clock(rq: env->dst_rq);
9227
9228	while (!list_empty(head: tasks)) {
9229	p = list_first_entry(tasks, struct task_struct, se.group_node);
9230	list_del_init(entry: &p->se.group_node);
9231
9232	attach_task(rq: env->dst_rq, p);
9233	}
9234
9235	rq_unlock(rq: env->dst_rq, rf: &rf);
9236	}
9237
9238	#ifdef CONFIG_NO_HZ_COMMON
9239	static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq)
9240	{
9241	if (cfs_rq->avg.load_avg)
9242	return true;
9243
9244	if (cfs_rq->avg.util_avg)
9245	return true;
9246
9247	return false;
9248	}
9249
9250	static inline bool others_have_blocked(struct rq *rq)
9251	{
9252	if (cpu_util_rt(rq))
9253	return true;
9254
9255	if (cpu_util_dl(rq))
9256	return true;
9257
9258	if (thermal_load_avg(rq))
9259	return true;
9260
9261	if (cpu_util_irq(rq))
9262	return true;
9263
9264	return false;
9265	}
9266
9267	static inline void update_blocked_load_tick(struct rq *rq)
9268	{
9269	WRITE_ONCE(rq->last_blocked_load_update_tick, jiffies);
9270	}
9271
9272	static inline void update_blocked_load_status(struct rq *rq, bool has_blocked)
9273	{
9274	if (!has_blocked)
9275	rq->has_blocked_load = 0;
9276	}
9277	#else
9278	static inline bool cfs_rq_has_blocked(struct cfs_rq *cfs_rq) { return false; }
9279	static inline bool others_have_blocked(struct rq *rq) { return false; }
9280	static inline void update_blocked_load_tick(struct rq *rq) {}
9281	static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {}
9282	#endif
9283
9284	static bool __update_blocked_others(struct rq *rq, bool *done)
9285	{
9286	const struct sched_class *curr_class;
9287	u64 now = rq_clock_pelt(rq);
9288	unsigned long thermal_pressure;
9289	bool decayed;
9290
9291	/*
9292	* update_load_avg() can call cpufreq_update_util(). Make sure that RT,
9293	* DL and IRQ signals have been updated before updating CFS.
9294	*/
9295	curr_class = rq->curr->sched_class;
9296
9297	thermal_pressure = arch_scale_thermal_pressure(cpu: cpu_of(rq));
9298
9299	decayed = update_rt_rq_load_avg(now, rq, running: curr_class == &rt_sched_class) |
9300	update_dl_rq_load_avg(now, rq, running: curr_class == &dl_sched_class) |
9301	update_thermal_load_avg(now: rq_clock_thermal(rq), rq, capacity: thermal_pressure) |
9302	update_irq_load_avg(rq, running: 0);
9303
9304	if (others_have_blocked(rq))
9305	*done = false;
9306
9307	return decayed;
9308	}
9309
9310	#ifdef CONFIG_FAIR_GROUP_SCHED
9311
9312	static bool __update_blocked_fair(struct rq *rq, bool *done)
9313	{
9314	struct cfs_rq *cfs_rq, *pos;
9315	bool decayed = false;
9316	int cpu = cpu_of(rq);
9317
9318	/*
9319	* Iterates the task_group tree in a bottom up fashion, see
9320	* list_add_leaf_cfs_rq() for details.
9321	*/
9322	for_each_leaf_cfs_rq_safe(rq, cfs_rq, pos) {
9323	struct sched_entity *se;
9324
9325	if (update_cfs_rq_load_avg(now: cfs_rq_clock_pelt(cfs_rq), cfs_rq)) {
9326	update_tg_load_avg(cfs_rq);
9327
9328	if (cfs_rq->nr_running == 0)
9329	update_idle_cfs_rq_clock_pelt(cfs_rq);
9330
9331	if (cfs_rq == &rq->cfs)
9332	decayed = true;
9333	}
9334
9335	/* Propagate pending load changes to the parent, if any: */
9336	se = cfs_rq->tg->se[cpu];
9337	if (se && !skip_blocked_update(se))
9338	update_load_avg(cfs_rq: cfs_rq_of(se), se, UPDATE_TG);
9339
9340	/*
9341	* There can be a lot of idle CPU cgroups. Don't let fully
9342	* decayed cfs_rqs linger on the list.
9343	*/
9344	if (cfs_rq_is_decayed(cfs_rq))
9345	list_del_leaf_cfs_rq(cfs_rq);
9346
9347	/* Don't need periodic decay once load/util_avg are null */
9348	if (cfs_rq_has_blocked(cfs_rq))
9349	*done = false;
9350	}
9351
9352	return decayed;
9353	}
9354
9355	/*
9356	* Compute the hierarchical load factor for cfs_rq and all its ascendants.
9357	* This needs to be done in a top-down fashion because the load of a child
9358	* group is a fraction of its parents load.
9359	*/
9360	static void update_cfs_rq_h_load(struct cfs_rq *cfs_rq)
9361	{
9362	struct rq *rq = rq_of(cfs_rq);
9363	struct sched_entity *se = cfs_rq->tg->se[cpu_of(rq)];
9364	unsigned long now = jiffies;
9365	unsigned long load;
9366
9367	if (cfs_rq->last_h_load_update == now)
9368	return;
9369
9370	WRITE_ONCE(cfs_rq->h_load_next, NULL);
9371	for_each_sched_entity(se) {
9372	cfs_rq = cfs_rq_of(se);
9373	WRITE_ONCE(cfs_rq->h_load_next, se);
9374	if (cfs_rq->last_h_load_update == now)
9375	break;
9376	}
9377
9378	if (!se) {
9379	cfs_rq->h_load = cfs_rq_load_avg(cfs_rq);
9380	cfs_rq->last_h_load_update = now;
9381	}
9382
9383	while ((se = READ_ONCE(cfs_rq->h_load_next)) != NULL) {
9384	load = cfs_rq->h_load;
9385	load = div64_ul(load * se->avg.load_avg,
9386	cfs_rq_load_avg(cfs_rq) + 1);
9387	cfs_rq = group_cfs_rq(grp: se);
9388	cfs_rq->h_load = load;
9389	cfs_rq->last_h_load_update = now;
9390	}
9391	}
9392
9393	static unsigned long task_h_load(struct task_struct *p)
9394	{
9395	struct cfs_rq *cfs_rq = task_cfs_rq(p);
9396
9397	update_cfs_rq_h_load(cfs_rq);
9398	return div64_ul(p->se.avg.load_avg * cfs_rq->h_load,
9399	cfs_rq_load_avg(cfs_rq) + 1);
9400	}
9401	#else
9402	static bool __update_blocked_fair(struct rq *rq, bool *done)
9403	{
9404	struct cfs_rq *cfs_rq = &rq->cfs;
9405	bool decayed;
9406
9407	decayed = update_cfs_rq_load_avg(cfs_rq_clock_pelt(cfs_rq), cfs_rq);
9408	if (cfs_rq_has_blocked(cfs_rq))
9409	*done = false;
9410
9411	return decayed;
9412	}
9413
9414	static unsigned long task_h_load(struct task_struct *p)
9415	{
9416	return p->se.avg.load_avg;
9417	}
9418	#endif
9419
9420	static void update_blocked_averages(int cpu)
9421	{
9422	bool decayed = false, done = true;
9423	struct rq *rq = cpu_rq(cpu);
9424	struct rq_flags rf;
9425
9426	rq_lock_irqsave(rq, rf: &rf);
9427	update_blocked_load_tick(rq);
9428	update_rq_clock(rq);
9429
9430	decayed |= __update_blocked_others(rq, done: &done);
9431	decayed |= __update_blocked_fair(rq, done: &done);
9432
9433	update_blocked_load_status(rq, has_blocked: !done);
9434	if (decayed)
9435	cpufreq_update_util(rq, flags: 0);
9436	rq_unlock_irqrestore(rq, rf: &rf);
9437	}
9438
9439	/********** Helpers for find_busiest_group ************************/
9440
9441	/*
9442	* sg_lb_stats - stats of a sched_group required for load_balancing
9443	*/
9444	struct sg_lb_stats {
9445	unsigned long avg_load; /*Avg load across the CPUs of the group */
9446	unsigned long group_load; /* Total load over the CPUs of the group */
9447	unsigned long group_capacity;
9448	unsigned long group_util; /* Total utilization over the CPUs of the group */
9449	unsigned long group_runnable; /* Total runnable time over the CPUs of the group */
9450	unsigned int sum_nr_running; /* Nr of tasks running in the group */
9451	unsigned int sum_h_nr_running; /* Nr of CFS tasks running in the group */
9452	unsigned int idle_cpus;
9453	unsigned int group_weight;
9454	enum group_type group_type;
9455	unsigned int group_asym_packing; /* Tasks should be moved to preferred CPU */
9456	unsigned int group_smt_balance; /* Task on busy SMT be moved */
9457	unsigned long group_misfit_task_load; /* A CPU has a task too big for its capacity */
9458	#ifdef CONFIG_NUMA_BALANCING
9459	unsigned int nr_numa_running;
9460	unsigned int nr_preferred_running;
9461	#endif
9462	};
9463
9464	/*
9465	* sd_lb_stats - Structure to store the statistics of a sched_domain
9466	* during load balancing.
9467	*/
9468	struct sd_lb_stats {
9469	struct sched_group *busiest; /* Busiest group in this sd */
9470	struct sched_group *local; /* Local group in this sd */
9471	unsigned long total_load; /* Total load of all groups in sd */
9472	unsigned long total_capacity; /* Total capacity of all groups in sd */
9473	unsigned long avg_load; /* Average load across all groups in sd */
9474	unsigned int prefer_sibling; /* tasks should go to sibling first */
9475
9476	struct sg_lb_stats busiest_stat;/* Statistics of the busiest group */
9477	struct sg_lb_stats local_stat; /* Statistics of the local group */
9478	};
9479
9480	static inline void init_sd_lb_stats(struct sd_lb_stats *sds)
9481	{
9482	/*
9483	* Skimp on the clearing to avoid duplicate work. We can avoid clearing
9484	* local_stat because update_sg_lb_stats() does a full clear/assignment.
9485	* We must however set busiest_stat::group_type and
9486	* busiest_stat::idle_cpus to the worst busiest group because
9487	* update_sd_pick_busiest() reads these before assignment.
9488	*/
9489	*sds = (struct sd_lb_stats){
9490	.busiest = NULL,
9491	.local = NULL,
9492	.total_load = 0UL,
9493	.total_capacity = 0UL,
9494	.busiest_stat = {
9495	.idle_cpus = UINT_MAX,
9496	.group_type = group_has_spare,
9497	},
9498	};
9499	}
9500
9501	static unsigned long scale_rt_capacity(int cpu)
9502	{
9503	struct rq *rq = cpu_rq(cpu);
9504	unsigned long max = arch_scale_cpu_capacity(cpu);
9505	unsigned long used, free;
9506	unsigned long irq;
9507
9508	irq = cpu_util_irq(rq);
9509
9510	if (unlikely(irq >= max))
9511	return 1;
9512
9513	/*
9514	* avg_rt.util_avg and avg_dl.util_avg track binary signals
9515	* (running and not running) with weights 0 and 1024 respectively.
9516	* avg_thermal.load_avg tracks thermal pressure and the weighted
9517	* average uses the actual delta max capacity(load).
9518	*/
9519	used = cpu_util_rt(rq);
9520	used += cpu_util_dl(rq);
9521	used += thermal_load_avg(rq);
9522
9523	if (unlikely(used >= max))
9524	return 1;
9525
9526	free = max - used;
9527
9528	return scale_irq_capacity(util: free, irq, max);
9529	}
9530
9531	static void update_cpu_capacity(struct sched_domain *sd, int cpu)
9532	{
9533	unsigned long capacity = scale_rt_capacity(cpu);
9534	struct sched_group *sdg = sd->groups;
9535
9536	if (!capacity)
9537	capacity = 1;
9538
9539	cpu_rq(cpu)->cpu_capacity = capacity;
9540	trace_sched_cpu_capacity_tp(cpu_rq(cpu));
9541
9542	sdg->sgc->capacity = capacity;
9543	sdg->sgc->min_capacity = capacity;
9544	sdg->sgc->max_capacity = capacity;
9545	}
9546
9547	void update_group_capacity(struct sched_domain *sd, int cpu)
9548	{
9549	struct sched_domain *child = sd->child;
9550	struct sched_group *group, *sdg = sd->groups;
9551	unsigned long capacity, min_capacity, max_capacity;
9552	unsigned long interval;
9553
9554	interval = msecs_to_jiffies(m: sd->balance_interval);
9555	interval = clamp(interval, 1UL, max_load_balance_interval);
9556	sdg->sgc->next_update = jiffies + interval;
9557
9558	if (!child) {
9559	update_cpu_capacity(sd, cpu);
9560	return;
9561	}
9562
9563	capacity = 0;
9564	min_capacity = ULONG_MAX;
9565	max_capacity = 0;
9566
9567	if (child->flags & SD_OVERLAP) {
9568	/*
9569	* SD_OVERLAP domains cannot assume that child groups
9570	* span the current group.
9571	*/
9572
9573	for_each_cpu(cpu, sched_group_span(sdg)) {
9574	unsigned long cpu_cap = capacity_of(cpu);
9575
9576	capacity += cpu_cap;
9577	min_capacity = min(cpu_cap, min_capacity);
9578	max_capacity = max(cpu_cap, max_capacity);
9579	}
9580	} else {
9581	/*
9582	* !SD_OVERLAP domains can assume that child groups
9583	* span the current group.
9584	*/
9585
9586	group = child->groups;
9587	do {
9588	struct sched_group_capacity *sgc = group->sgc;
9589
9590	capacity += sgc->capacity;
9591	min_capacity = min(sgc->min_capacity, min_capacity);
9592	max_capacity = max(sgc->max_capacity, max_capacity);
9593	group = group->next;
9594	} while (group != child->groups);
9595	}
9596
9597	sdg->sgc->capacity = capacity;
9598	sdg->sgc->min_capacity = min_capacity;
9599	sdg->sgc->max_capacity = max_capacity;
9600	}
9601
9602	/*
9603	* Check whether the capacity of the rq has been noticeably reduced by side
9604	* activity. The imbalance_pct is used for the threshold.
9605	* Return true is the capacity is reduced
9606	*/
9607	static inline int
9608	check_cpu_capacity(struct rq *rq, struct sched_domain *sd)
9609	{
9610	return ((rq->cpu_capacity * sd->imbalance_pct) <
9611	(arch_scale_cpu_capacity(cpu: cpu_of(rq)) * 100));
9612	}
9613
9614	/*
9615	* Check whether a rq has a misfit task and if it looks like we can actually
9616	* help that task: we can migrate the task to a CPU of higher capacity, or
9617	* the task's current CPU is heavily pressured.
9618	*/
9619	static inline int check_misfit_status(struct rq *rq, struct sched_domain *sd)
9620	{
9621	return rq->misfit_task_load &&
9622	(arch_scale_cpu_capacity(cpu: rq->cpu) < rq->rd->max_cpu_capacity ||
9623	check_cpu_capacity(rq, sd));
9624	}
9625
9626	/*
9627	* Group imbalance indicates (and tries to solve) the problem where balancing
9628	* groups is inadequate due to ->cpus_ptr constraints.
9629	*
9630	* Imagine a situation of two groups of 4 CPUs each and 4 tasks each with a
9631	* cpumask covering 1 CPU of the first group and 3 CPUs of the second group.
9632	* Something like:
9633	*
9634	* { 0 1 2 3 } { 4 5 6 7 }
9635	* * * * *
9636	*
9637	* If we were to balance group-wise we'd place two tasks in the first group and
9638	* two tasks in the second group. Clearly this is undesired as it will overload
9639	* cpu 3 and leave one of the CPUs in the second group unused.
9640	*
9641	* The current solution to this issue is detecting the skew in the first group
9642	* by noticing the lower domain failed to reach balance and had difficulty
9643	* moving tasks due to affinity constraints.
9644	*
9645	* When this is so detected; this group becomes a candidate for busiest; see
9646	* update_sd_pick_busiest(). And calculate_imbalance() and
9647	* find_busiest_group() avoid some of the usual balance conditions to allow it
9648	* to create an effective group imbalance.
9649	*
9650	* This is a somewhat tricky proposition since the next run might not find the
9651	* group imbalance and decide the groups need to be balanced again. A most
9652	* subtle and fragile situation.
9653	*/
9654
9655	static inline int sg_imbalanced(struct sched_group *group)
9656	{
9657	return group->sgc->imbalance;
9658	}
9659
9660	/*
9661	* group_has_capacity returns true if the group has spare capacity that could
9662	* be used by some tasks.
9663	* We consider that a group has spare capacity if the number of task is
9664	* smaller than the number of CPUs or if the utilization is lower than the
9665	* available capacity for CFS tasks.
9666	* For the latter, we use a threshold to stabilize the state, to take into
9667	* account the variance of the tasks' load and to return true if the available
9668	* capacity in meaningful for the load balancer.
9669	* As an example, an available capacity of 1% can appear but it doesn't make
9670	* any benefit for the load balance.
9671	*/
9672	static inline bool
9673	group_has_capacity(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
9674	{
9675	if (sgs->sum_nr_running < sgs->group_weight)
9676	return true;
9677
9678	if ((sgs->group_capacity * imbalance_pct) <
9679	(sgs->group_runnable * 100))
9680	return false;
9681
9682	if ((sgs->group_capacity * 100) >
9683	(sgs->group_util * imbalance_pct))
9684	return true;
9685
9686	return false;
9687	}
9688
9689	/*
9690	* group_is_overloaded returns true if the group has more tasks than it can
9691	* handle.
9692	* group_is_overloaded is not equals to !group_has_capacity because a group
9693	* with the exact right number of tasks, has no more spare capacity but is not
9694	* overloaded so both group_has_capacity and group_is_overloaded return
9695	* false.
9696	*/
9697	static inline bool
9698	group_is_overloaded(unsigned int imbalance_pct, struct sg_lb_stats *sgs)
9699	{
9700	if (sgs->sum_nr_running <= sgs->group_weight)
9701	return false;
9702
9703	if ((sgs->group_capacity * 100) <
9704	(sgs->group_util * imbalance_pct))
9705	return true;
9706
9707	if ((sgs->group_capacity * imbalance_pct) <
9708	(sgs->group_runnable * 100))
9709	return true;
9710
9711	return false;
9712	}
9713
9714	static inline enum
9715	group_type group_classify(unsigned int imbalance_pct,
9716	struct sched_group *group,
9717	struct sg_lb_stats *sgs)
9718	{
9719	if (group_is_overloaded(imbalance_pct, sgs))
9720	return group_overloaded;
9721
9722	if (sg_imbalanced(group))
9723	return group_imbalanced;
9724
9725	if (sgs->group_asym_packing)
9726	return group_asym_packing;
9727
9728	if (sgs->group_smt_balance)
9729	return group_smt_balance;
9730
9731	if (sgs->group_misfit_task_load)
9732	return group_misfit_task;
9733
9734	if (!group_has_capacity(imbalance_pct, sgs))
9735	return group_fully_busy;
9736
9737	return group_has_spare;
9738	}
9739
9740	/**
9741	* sched_use_asym_prio - Check whether asym_packing priority must be used
9742	* @sd: The scheduling domain of the load balancing
9743	* @cpu: A CPU
9744	*
9745	* Always use CPU priority when balancing load between SMT siblings. When
9746	* balancing load between cores, it is not sufficient that @cpu is idle. Only
9747	* use CPU priority if the whole core is idle.
9748	*
9749	* Returns: True if the priority of @cpu must be followed. False otherwise.
9750	*/
9751	static bool sched_use_asym_prio(struct sched_domain *sd, int cpu)
9752	{
9753	if (!(sd->flags & SD_ASYM_PACKING))
9754	return false;
9755
9756	if (!sched_smt_active())
9757	return true;
9758
9759	return sd->flags & SD_SHARE_CPUCAPACITY || is_core_idle(cpu);
9760	}
9761
9762	static inline bool sched_asym(struct sched_domain *sd, int dst_cpu, int src_cpu)
9763	{
9764	/*
9765	* First check if @dst_cpu can do asym_packing load balance. Only do it
9766	* if it has higher priority than @src_cpu.
9767	*/
9768	return sched_use_asym_prio(sd, cpu: dst_cpu) &&
9769	sched_asym_prefer(a: dst_cpu, b: src_cpu);
9770	}
9771
9772	/**
9773	* sched_group_asym - Check if the destination CPU can do asym_packing balance
9774	* @env: The load balancing environment
9775	* @sgs: Load-balancing statistics of the candidate busiest group
9776	* @group: The candidate busiest group
9777	*
9778	* @env::dst_cpu can do asym_packing if it has higher priority than the
9779	* preferred CPU of @group.
9780	*
9781	* Return: true if @env::dst_cpu can do with asym_packing load balance. False
9782	* otherwise.
9783	*/
9784	static inline bool
9785	sched_group_asym(struct lb_env *env, struct sg_lb_stats *sgs, struct sched_group *group)
9786	{
9787	/*
9788	* CPU priorities do not make sense for SMT cores with more than one
9789	* busy sibling.
9790	*/
9791	if ((group->flags & SD_SHARE_CPUCAPACITY) &&
9792	(sgs->group_weight - sgs->idle_cpus != 1))
9793	return false;
9794
9795	return sched_asym(sd: env->sd, dst_cpu: env->dst_cpu, src_cpu: group->asym_prefer_cpu);
9796	}
9797
9798	/* One group has more than one SMT CPU while the other group does not */
9799	static inline bool smt_vs_nonsmt_groups(struct sched_group *sg1,
9800	struct sched_group *sg2)
9801	{
9802	if (!sg1 || !sg2)
9803	return false;
9804
9805	return (sg1->flags & SD_SHARE_CPUCAPACITY) !=
9806	(sg2->flags & SD_SHARE_CPUCAPACITY);
9807	}
9808
9809	static inline bool smt_balance(struct lb_env *env, struct sg_lb_stats *sgs,
9810	struct sched_group *group)
9811	{
9812	if (env->idle == CPU_NOT_IDLE)
9813	return false;
9814
9815	/*
9816	* For SMT source group, it is better to move a task
9817	* to a CPU that doesn't have multiple tasks sharing its CPU capacity.
9818	* Note that if a group has a single SMT, SD_SHARE_CPUCAPACITY
9819	* will not be on.
9820	*/
9821	if (group->flags & SD_SHARE_CPUCAPACITY &&
9822	sgs->sum_h_nr_running > 1)
9823	return true;
9824
9825	return false;
9826	}
9827
9828	static inline long sibling_imbalance(struct lb_env *env,
9829	struct sd_lb_stats *sds,
9830	struct sg_lb_stats *busiest,
9831	struct sg_lb_stats *local)
9832	{
9833	int ncores_busiest, ncores_local;
9834	long imbalance;
9835
9836	if (env->idle == CPU_NOT_IDLE || !busiest->sum_nr_running)
9837	return 0;
9838
9839	ncores_busiest = sds->busiest->cores;
9840	ncores_local = sds->local->cores;
9841
9842	if (ncores_busiest == ncores_local) {
9843	imbalance = busiest->sum_nr_running;
9844	lsub_positive(&imbalance, local->sum_nr_running);
9845	return imbalance;
9846	}
9847
9848	/* Balance such that nr_running/ncores ratio are same on both groups */
9849	imbalance = ncores_local * busiest->sum_nr_running;
9850	lsub_positive(&imbalance, ncores_busiest * local->sum_nr_running);
9851	/* Normalize imbalance and do rounding on normalization */
9852	imbalance = 2 * imbalance + ncores_local + ncores_busiest;
9853	imbalance /= ncores_local + ncores_busiest;
9854
9855	/* Take advantage of resource in an empty sched group */
9856	if (imbalance <= 1 && local->sum_nr_running == 0 &&
9857	busiest->sum_nr_running > 1)
9858	imbalance = 2;
9859
9860	return imbalance;
9861	}
9862
9863	static inline bool
9864	sched_reduced_capacity(struct rq *rq, struct sched_domain *sd)
9865	{
9866	/*
9867	* When there is more than 1 task, the group_overloaded case already
9868	* takes care of cpu with reduced capacity
9869	*/
9870	if (rq->cfs.h_nr_running != 1)
9871	return false;
9872
9873	return check_cpu_capacity(rq, sd);
9874	}
9875
9876	/**
9877	* update_sg_lb_stats - Update sched_group's statistics for load balancing.
9878	* @env: The load balancing environment.
9879	* @sds: Load-balancing data with statistics of the local group.
9880	* @group: sched_group whose statistics are to be updated.
9881	* @sgs: variable to hold the statistics for this group.
9882	* @sg_status: Holds flag indicating the status of the sched_group
9883	*/
9884	static inline void update_sg_lb_stats(struct lb_env *env,
9885	struct sd_lb_stats *sds,
9886	struct sched_group *group,
9887	struct sg_lb_stats *sgs,
9888	int *sg_status)
9889	{
9890	int i, nr_running, local_group;
9891
9892	memset(sgs, 0, sizeof(*sgs));
9893
9894	local_group = group == sds->local;
9895
9896	for_each_cpu_and(i, sched_group_span(group), env->cpus) {
9897	struct rq *rq = cpu_rq(i);
9898	unsigned long load = cpu_load(rq);
9899
9900	sgs->group_load += load;
9901	sgs->group_util += cpu_util_cfs(cpu: i);
9902	sgs->group_runnable += cpu_runnable(rq);
9903	sgs->sum_h_nr_running += rq->cfs.h_nr_running;
9904
9905	nr_running = rq->nr_running;
9906	sgs->sum_nr_running += nr_running;
9907
9908	if (nr_running > 1)
9909	*sg_status |= SG_OVERLOAD;
9910
9911	if (cpu_overutilized(cpu: i))
9912	*sg_status |= SG_OVERUTILIZED;
9913
9914	#ifdef CONFIG_NUMA_BALANCING
9915	sgs->nr_numa_running += rq->nr_numa_running;
9916	sgs->nr_preferred_running += rq->nr_preferred_running;
9917	#endif
9918	/*
9919	* No need to call idle_cpu() if nr_running is not 0
9920	*/
9921	if (!nr_running && idle_cpu(cpu: i)) {
9922	sgs->idle_cpus++;
9923	/* Idle cpu can't have misfit task */
9924	continue;
9925	}
9926
9927	if (local_group)
9928	continue;
9929
9930	if (env->sd->flags & SD_ASYM_CPUCAPACITY) {
9931	/* Check for a misfit task on the cpu */
9932	if (sgs->group_misfit_task_load < rq->misfit_task_load) {
9933	sgs->group_misfit_task_load = rq->misfit_task_load;
9934	*sg_status |= SG_OVERLOAD;
9935	}
9936	} else if ((env->idle != CPU_NOT_IDLE) &&
9937	sched_reduced_capacity(rq, sd: env->sd)) {
9938	/* Check for a task running on a CPU with reduced capacity */
9939	if (sgs->group_misfit_task_load < load)
9940	sgs->group_misfit_task_load = load;
9941	}
9942	}
9943
9944	sgs->group_capacity = group->sgc->capacity;
9945
9946	sgs->group_weight = group->group_weight;
9947
9948	/* Check if dst CPU is idle and preferred to this group */
9949	if (!local_group && env->idle != CPU_NOT_IDLE && sgs->sum_h_nr_running &&
9950	sched_group_asym(env, sgs, group))
9951	sgs->group_asym_packing = 1;
9952
9953	/* Check for loaded SMT group to be balanced to dst CPU */
9954	if (!local_group && smt_balance(env, sgs, group))
9955	sgs->group_smt_balance = 1;
9956
9957	sgs->group_type = group_classify(imbalance_pct: env->sd->imbalance_pct, group, sgs);
9958
9959	/* Computing avg_load makes sense only when group is overloaded */
9960	if (sgs->group_type == group_overloaded)
9961	sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
9962	sgs->group_capacity;
9963	}
9964
9965	/**
9966	* update_sd_pick_busiest - return 1 on busiest group
9967	* @env: The load balancing environment.
9968	* @sds: sched_domain statistics
9969	* @sg: sched_group candidate to be checked for being the busiest
9970	* @sgs: sched_group statistics
9971	*
9972	* Determine if @sg is a busier group than the previously selected
9973	* busiest group.
9974	*
9975	* Return: %true if @sg is a busier group than the previously selected
9976	* busiest group. %false otherwise.
9977	*/
9978	static bool update_sd_pick_busiest(struct lb_env *env,
9979	struct sd_lb_stats *sds,
9980	struct sched_group *sg,
9981	struct sg_lb_stats *sgs)
9982	{
9983	struct sg_lb_stats *busiest = &sds->busiest_stat;
9984
9985	/* Make sure that there is at least one task to pull */
9986	if (!sgs->sum_h_nr_running)
9987	return false;
9988
9989	/*
9990	* Don't try to pull misfit tasks we can't help.
9991	* We can use max_capacity here as reduction in capacity on some
9992	* CPUs in the group should either be possible to resolve
9993	* internally or be covered by avg_load imbalance (eventually).
9994	*/
9995	if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
9996	(sgs->group_type == group_misfit_task) &&
9997	(!capacity_greater(capacity_of(env->dst_cpu), sg->sgc->max_capacity) ||
9998	sds->local_stat.group_type != group_has_spare))
9999	return false;
10000
10001	if (sgs->group_type > busiest->group_type)
10002	return true;
10003
10004	if (sgs->group_type < busiest->group_type)
10005	return false;
10006
10007	/*
10008	* The candidate and the current busiest group are the same type of
10009	* group. Let check which one is the busiest according to the type.
10010	*/
10011
10012	switch (sgs->group_type) {
10013	case group_overloaded:
10014	/* Select the overloaded group with highest avg_load. */
10015	return sgs->avg_load > busiest->avg_load;
10016
10017	case group_imbalanced:
10018	/*
10019	* Select the 1st imbalanced group as we don't have any way to
10020	* choose one more than another.
10021	*/
10022	return false;
10023
10024	case group_asym_packing:
10025	/* Prefer to move from lowest priority CPU's work */
10026	return sched_asym_prefer(a: sds->busiest->asym_prefer_cpu, b: sg->asym_prefer_cpu);
10027
10028	case group_misfit_task:
10029	/*
10030	* If we have more than one misfit sg go with the biggest
10031	* misfit.
10032	*/
10033	return sgs->group_misfit_task_load > busiest->group_misfit_task_load;
10034
10035	case group_smt_balance:
10036	/*
10037	* Check if we have spare CPUs on either SMT group to
10038	* choose has spare or fully busy handling.
10039	*/
10040	if (sgs->idle_cpus != 0 || busiest->idle_cpus != 0)
10041	goto has_spare;
10042
10043	fallthrough;
10044
10045	case group_fully_busy:
10046	/*
10047	* Select the fully busy group with highest avg_load. In
10048	* theory, there is no need to pull task from such kind of
10049	* group because tasks have all compute capacity that they need
10050	* but we can still improve the overall throughput by reducing
10051	* contention when accessing shared HW resources.
10052	*
10053	* XXX for now avg_load is not computed and always 0 so we
10054	* select the 1st one, except if @sg is composed of SMT
10055	* siblings.
10056	*/
10057
10058	if (sgs->avg_load < busiest->avg_load)
10059	return false;
10060
10061	if (sgs->avg_load == busiest->avg_load) {
10062	/*
10063	* SMT sched groups need more help than non-SMT groups.
10064	* If @sg happens to also be SMT, either choice is good.
10065	*/
10066	if (sds->busiest->flags & SD_SHARE_CPUCAPACITY)
10067	return false;
10068	}
10069
10070	break;
10071
10072	case group_has_spare:
10073	/*
10074	* Do not pick sg with SMT CPUs over sg with pure CPUs,
10075	* as we do not want to pull task off SMT core with one task
10076	* and make the core idle.
10077	*/
10078	if (smt_vs_nonsmt_groups(sg1: sds->busiest, sg2: sg)) {
10079	if (sg->flags & SD_SHARE_CPUCAPACITY && sgs->sum_h_nr_running <= 1)
10080	return false;
10081	else
10082	return true;
10083	}
10084	has_spare:
10085
10086	/*
10087	* Select not overloaded group with lowest number of idle cpus
10088	* and highest number of running tasks. We could also compare
10089	* the spare capacity which is more stable but it can end up
10090	* that the group has less spare capacity but finally more idle
10091	* CPUs which means less opportunity to pull tasks.
10092	*/
10093	if (sgs->idle_cpus > busiest->idle_cpus)
10094	return false;
10095	else if ((sgs->idle_cpus == busiest->idle_cpus) &&
10096	(sgs->sum_nr_running <= busiest->sum_nr_running))
10097	return false;
10098
10099	break;
10100	}
10101
10102	/*
10103	* Candidate sg has no more than one task per CPU and has higher
10104	* per-CPU capacity. Migrating tasks to less capable CPUs may harm
10105	* throughput. Maximize throughput, power/energy consequences are not
10106	* considered.
10107	*/
10108	if ((env->sd->flags & SD_ASYM_CPUCAPACITY) &&
10109	(sgs->group_type <= group_fully_busy) &&
10110	(capacity_greater(sg->sgc->min_capacity, capacity_of(env->dst_cpu))))
10111	return false;
10112
10113	return true;
10114	}
10115
10116	#ifdef CONFIG_NUMA_BALANCING
10117	static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
10118	{
10119	if (sgs->sum_h_nr_running > sgs->nr_numa_running)
10120	return regular;
10121	if (sgs->sum_h_nr_running > sgs->nr_preferred_running)
10122	return remote;
10123	return all;
10124	}
10125
10126	static inline enum fbq_type fbq_classify_rq(struct rq *rq)
10127	{
10128	if (rq->nr_running > rq->nr_numa_running)
10129	return regular;
10130	if (rq->nr_running > rq->nr_preferred_running)
10131	return remote;
10132	return all;
10133	}
10134	#else
10135	static inline enum fbq_type fbq_classify_group(struct sg_lb_stats *sgs)
10136	{
10137	return all;
10138	}
10139
10140	static inline enum fbq_type fbq_classify_rq(struct rq *rq)
10141	{
10142	return regular;
10143	}
10144	#endif /* CONFIG_NUMA_BALANCING */
10145
10146
10147	struct sg_lb_stats;
10148
10149	/*
10150	* task_running_on_cpu - return 1 if @p is running on @cpu.
10151	*/
10152
10153	static unsigned int task_running_on_cpu(int cpu, struct task_struct *p)
10154	{
10155	/* Task has no contribution or is new */
10156	if (cpu != task_cpu(p) || !READ_ONCE(p->se.avg.last_update_time))
10157	return 0;
10158
10159	if (task_on_rq_queued(p))
10160	return 1;
10161
10162	return 0;
10163	}
10164
10165	/**
10166	* idle_cpu_without - would a given CPU be idle without p ?
10167	* @cpu: the processor on which idleness is tested.
10168	* @p: task which should be ignored.
10169	*
10170	* Return: 1 if the CPU would be idle. 0 otherwise.
10171	*/
10172	static int idle_cpu_without(int cpu, struct task_struct *p)
10173	{
10174	struct rq *rq = cpu_rq(cpu);
10175
10176	if (rq->curr != rq->idle && rq->curr != p)
10177	return 0;
10178
10179	/*
10180	* rq->nr_running can't be used but an updated version without the
10181	* impact of p on cpu must be used instead. The updated nr_running
10182	* be computed and tested before calling idle_cpu_without().
10183	*/
10184
10185	if (rq->ttwu_pending)
10186	return 0;
10187
10188	return 1;
10189	}
10190
10191	/*
10192	* update_sg_wakeup_stats - Update sched_group's statistics for wakeup.
10193	* @sd: The sched_domain level to look for idlest group.
10194	* @group: sched_group whose statistics are to be updated.
10195	* @sgs: variable to hold the statistics for this group.
10196	* @p: The task for which we look for the idlest group/CPU.
10197	*/
10198	static inline void update_sg_wakeup_stats(struct sched_domain *sd,
10199	struct sched_group *group,
10200	struct sg_lb_stats *sgs,
10201	struct task_struct *p)
10202	{
10203	int i, nr_running;
10204
10205	memset(sgs, 0, sizeof(*sgs));
10206
10207	/* Assume that task can't fit any CPU of the group */
10208	if (sd->flags & SD_ASYM_CPUCAPACITY)
10209	sgs->group_misfit_task_load = 1;
10210
10211	for_each_cpu(i, sched_group_span(group)) {
10212	struct rq *rq = cpu_rq(i);
10213	unsigned int local;
10214
10215	sgs->group_load += cpu_load_without(rq, p);
10216	sgs->group_util += cpu_util_without(cpu: i, p);
10217	sgs->group_runnable += cpu_runnable_without(rq, p);
10218	local = task_running_on_cpu(cpu: i, p);
10219	sgs->sum_h_nr_running += rq->cfs.h_nr_running - local;
10220
10221	nr_running = rq->nr_running - local;
10222	sgs->sum_nr_running += nr_running;
10223
10224	/*
10225	* No need to call idle_cpu_without() if nr_running is not 0
10226	*/
10227	if (!nr_running && idle_cpu_without(cpu: i, p))
10228	sgs->idle_cpus++;
10229
10230	/* Check if task fits in the CPU */
10231	if (sd->flags & SD_ASYM_CPUCAPACITY &&
10232	sgs->group_misfit_task_load &&
10233	task_fits_cpu(p, cpu: i))
10234	sgs->group_misfit_task_load = 0;
10235
10236	}
10237
10238	sgs->group_capacity = group->sgc->capacity;
10239
10240	sgs->group_weight = group->group_weight;
10241
10242	sgs->group_type = group_classify(imbalance_pct: sd->imbalance_pct, group, sgs);
10243
10244	/*
10245	* Computing avg_load makes sense only when group is fully busy or
10246	* overloaded
10247	*/
10248	if (sgs->group_type == group_fully_busy ||
10249	sgs->group_type == group_overloaded)
10250	sgs->avg_load = (sgs->group_load * SCHED_CAPACITY_SCALE) /
10251	sgs->group_capacity;
10252	}
10253
10254	static bool update_pick_idlest(struct sched_group *idlest,
10255	struct sg_lb_stats *idlest_sgs,
10256	struct sched_group *group,
10257	struct sg_lb_stats *sgs)
10258	{
10259	if (sgs->group_type < idlest_sgs->group_type)
10260	return true;
10261
10262	if (sgs->group_type > idlest_sgs->group_type)
10263	return false;
10264
10265	/*
10266	* The candidate and the current idlest group are the same type of
10267	* group. Let check which one is the idlest according to the type.
10268	*/
10269
10270	switch (sgs->group_type) {
10271	case group_overloaded:
10272	case group_fully_busy:
10273	/* Select the group with lowest avg_load. */
10274	if (idlest_sgs->avg_load <= sgs->avg_load)
10275	return false;
10276	break;
10277
10278	case group_imbalanced:
10279	case group_asym_packing:
10280	case group_smt_balance:
10281	/* Those types are not used in the slow wakeup path */
10282	return false;
10283
10284	case group_misfit_task:
10285	/* Select group with the highest max capacity */
10286	if (idlest->sgc->max_capacity >= group->sgc->max_capacity)
10287	return false;
10288	break;
10289
10290	case group_has_spare:
10291	/* Select group with most idle CPUs */
10292	if (idlest_sgs->idle_cpus > sgs->idle_cpus)
10293	return false;
10294
10295	/* Select group with lowest group_util */
10296	if (idlest_sgs->idle_cpus == sgs->idle_cpus &&
10297	idlest_sgs->group_util <= sgs->group_util)
10298	return false;
10299
10300	break;
10301	}
10302
10303	return true;
10304	}
10305
10306	/*
10307	* find_idlest_group() finds and returns the least busy CPU group within the
10308	* domain.
10309	*
10310	* Assumes p is allowed on at least one CPU in sd.
10311	*/
10312	static struct sched_group *
10313	find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu)
10314	{
10315	struct sched_group *idlest = NULL, *local = NULL, *group = sd->groups;
10316	struct sg_lb_stats local_sgs, tmp_sgs;
10317	struct sg_lb_stats *sgs;
10318	unsigned long imbalance;
10319	struct sg_lb_stats idlest_sgs = {
10320	.avg_load = UINT_MAX,
10321	.group_type = group_overloaded,
10322	};
10323
10324	do {
10325	int local_group;
10326
10327	/* Skip over this group if it has no CPUs allowed */
10328	if (!cpumask_intersects(src1p: sched_group_span(sg: group),
10329	src2p: p->cpus_ptr))
10330	continue;
10331
10332	/* Skip over this group if no cookie matched */
10333	if (!sched_group_cookie_match(cpu_rq(this_cpu), p, group))
10334	continue;
10335
10336	local_group = cpumask_test_cpu(cpu: this_cpu,
10337	cpumask: sched_group_span(sg: group));
10338
10339	if (local_group) {
10340	sgs = &local_sgs;
10341	local = group;
10342	} else {
10343	sgs = &tmp_sgs;
10344	}
10345
10346	update_sg_wakeup_stats(sd, group, sgs, p);
10347
10348	if (!local_group && update_pick_idlest(idlest, idlest_sgs: &idlest_sgs, group, sgs)) {
10349	idlest = group;
10350	idlest_sgs = *sgs;
10351	}
10352
10353	} while (group = group->next, group != sd->groups);
10354
10355
10356	/* There is no idlest group to push tasks to */
10357	if (!idlest)
10358	return NULL;
10359
10360	/* The local group has been skipped because of CPU affinity */
10361	if (!local)
10362	return idlest;
10363
10364	/*
10365	* If the local group is idler than the selected idlest group
10366	* don't try and push the task.
10367	*/
10368	if (local_sgs.group_type < idlest_sgs.group_type)
10369	return NULL;
10370
10371	/*
10372	* If the local group is busier than the selected idlest group
10373	* try and push the task.
10374	*/
10375	if (local_sgs.group_type > idlest_sgs.group_type)
10376	return idlest;
10377
10378	switch (local_sgs.group_type) {
10379	case group_overloaded:
10380	case group_fully_busy:
10381
10382	/* Calculate allowed imbalance based on load */
10383	imbalance = scale_load_down(NICE_0_LOAD) *
10384	(sd->imbalance_pct-100) / 100;
10385
10386	/*
10387	* When comparing groups across NUMA domains, it's possible for
10388	* the local domain to be very lightly loaded relative to the
10389	* remote domains but "imbalance" skews the comparison making
10390	* remote CPUs look much more favourable. When considering
10391	* cross-domain, add imbalance to the load on the remote node
10392	* and consider staying local.
10393	*/
10394
10395	if ((sd->flags & SD_NUMA) &&
10396	((idlest_sgs.avg_load + imbalance) >= local_sgs.avg_load))
10397	return NULL;
10398
10399	/*
10400	* If the local group is less loaded than the selected
10401	* idlest group don't try and push any tasks.
10402	*/
10403	if (idlest_sgs.avg_load >= (local_sgs.avg_load + imbalance))
10404	return NULL;
10405
10406	if (100 * local_sgs.avg_load <= sd->imbalance_pct * idlest_sgs.avg_load)
10407	return NULL;
10408	break;
10409
10410	case group_imbalanced:
10411	case group_asym_packing:
10412	case group_smt_balance:
10413	/* Those type are not used in the slow wakeup path */
10414	return NULL;
10415
10416	case group_misfit_task:
10417	/* Select group with the highest max capacity */
10418	if (local->sgc->max_capacity >= idlest->sgc->max_capacity)
10419	return NULL;
10420	break;
10421
10422	case group_has_spare:
10423	#ifdef CONFIG_NUMA
10424	if (sd->flags & SD_NUMA) {
10425	int imb_numa_nr = sd->imb_numa_nr;
10426	#ifdef CONFIG_NUMA_BALANCING
10427	int idlest_cpu;
10428	/*
10429	* If there is spare capacity at NUMA, try to select
10430	* the preferred node
10431	*/
10432	if (cpu_to_node(cpu: this_cpu) == p->numa_preferred_nid)
10433	return NULL;
10434
10435	idlest_cpu = cpumask_first(srcp: sched_group_span(sg: idlest));
10436	if (cpu_to_node(cpu: idlest_cpu) == p->numa_preferred_nid)
10437	return idlest;
10438	#endif /* CONFIG_NUMA_BALANCING */
10439	/*
10440	* Otherwise, keep the task close to the wakeup source
10441	* and improve locality if the number of running tasks
10442	* would remain below threshold where an imbalance is
10443	* allowed while accounting for the possibility the
10444	* task is pinned to a subset of CPUs. If there is a
10445	* real need of migration, periodic load balance will
10446	* take care of it.
10447	*/
10448	if (p->nr_cpus_allowed != NR_CPUS) {
10449	struct cpumask *cpus = this_cpu_cpumask_var_ptr(select_rq_mask);
10450
10451	cpumask_and(dstp: cpus, src1p: sched_group_span(sg: local), src2p: p->cpus_ptr);
10452	imb_numa_nr = min(cpumask_weight(cpus), sd->imb_numa_nr);
10453	}
10454
10455	imbalance = abs(local_sgs.idle_cpus - idlest_sgs.idle_cpus);
10456	if (!adjust_numa_imbalance(imbalance,
10457	dst_running: local_sgs.sum_nr_running + 1,
10458	imb_numa_nr)) {
10459	return NULL;
10460	}
10461	}
10462	#endif /* CONFIG_NUMA */
10463
10464	/*
10465	* Select group with highest number of idle CPUs. We could also
10466	* compare the utilization which is more stable but it can end
10467	* up that the group has less spare capacity but finally more
10468	* idle CPUs which means more opportunity to run task.
10469	*/
10470	if (local_sgs.idle_cpus >= idlest_sgs.idle_cpus)
10471	return NULL;
10472	break;
10473	}
10474
10475	return idlest;
10476	}
10477
10478	static void update_idle_cpu_scan(struct lb_env *env,
10479	unsigned long sum_util)
10480	{
10481	struct sched_domain_shared *sd_share;
10482	int llc_weight, pct;
10483	u64 x, y, tmp;
10484	/*
10485	* Update the number of CPUs to scan in LLC domain, which could
10486	* be used as a hint in select_idle_cpu(). The update of sd_share
10487	* could be expensive because it is within a shared cache line.
10488	* So the write of this hint only occurs during periodic load
10489	* balancing, rather than CPU_NEWLY_IDLE, because the latter
10490	* can fire way more frequently than the former.
10491	*/
10492	if (!sched_feat(SIS_UTIL) || env->idle == CPU_NEWLY_IDLE)
10493	return;
10494
10495	llc_weight = per_cpu(sd_llc_size, env->dst_cpu);
10496	if (env->sd->span_weight != llc_weight)
10497	return;
10498
10499	sd_share = rcu_dereference(per_cpu(sd_llc_shared, env->dst_cpu));
10500	if (!sd_share)
10501	return;
10502
10503	/*
10504	* The number of CPUs to search drops as sum_util increases, when
10505	* sum_util hits 85% or above, the scan stops.
10506	* The reason to choose 85% as the threshold is because this is the
10507	* imbalance_pct(117) when a LLC sched group is overloaded.
10508	*
10509	* let y = SCHED_CAPACITY_SCALE - p * x^2 [1]
10510	* and y'= y / SCHED_CAPACITY_SCALE
10511	*
10512	* x is the ratio of sum_util compared to the CPU capacity:
10513	* x = sum_util / (llc_weight * SCHED_CAPACITY_SCALE)
10514	* y' is the ratio of CPUs to be scanned in the LLC domain,
10515	* and the number of CPUs to scan is calculated by:
10516	*
10517	* nr_scan = llc_weight * y' [2]
10518	*
10519	* When x hits the threshold of overloaded, AKA, when
10520	* x = 100 / pct, y drops to 0. According to [1],
10521	* p should be SCHED_CAPACITY_SCALE * pct^2 / 10000
10522	*
10523	* Scale x by SCHED_CAPACITY_SCALE:
10524	* x' = sum_util / llc_weight; [3]
10525	*
10526	* and finally [1] becomes:
10527	* y = SCHED_CAPACITY_SCALE -
10528	* x'^2 * pct^2 / (10000 * SCHED_CAPACITY_SCALE) [4]
10529	*
10530	*/
10531	/* equation [3] */
10532	x = sum_util;
10533	do_div(x, llc_weight);
10534
10535	/* equation [4] */
10536	pct = env->sd->imbalance_pct;
10537	tmp = x * x * pct * pct;
10538	do_div(tmp, 10000 * SCHED_CAPACITY_SCALE);
10539	tmp = min_t(long, tmp, SCHED_CAPACITY_SCALE);
10540	y = SCHED_CAPACITY_SCALE - tmp;
10541
10542	/* equation [2] */
10543	y *= llc_weight;
10544	do_div(y, SCHED_CAPACITY_SCALE);
10545	if ((int)y != sd_share->nr_idle_scan)
10546	WRITE_ONCE(sd_share->nr_idle_scan, (int)y);
10547	}
10548
10549	/**
10550	* update_sd_lb_stats - Update sched_domain's statistics for load balancing.
10551	* @env: The load balancing environment.
10552	* @sds: variable to hold the statistics for this sched_domain.
10553	*/
10554
10555	static inline void update_sd_lb_stats(struct lb_env *env, struct sd_lb_stats *sds)
10556	{
10557	struct sched_group *sg = env->sd->groups;
10558	struct sg_lb_stats *local = &sds->local_stat;
10559	struct sg_lb_stats tmp_sgs;
10560	unsigned long sum_util = 0;
10561	int sg_status = 0;
10562
10563	do {
10564	struct sg_lb_stats *sgs = &tmp_sgs;
10565	int local_group;
10566
10567	local_group = cpumask_test_cpu(cpu: env->dst_cpu, cpumask: sched_group_span(sg));
10568	if (local_group) {
10569	sds->local = sg;
10570	sgs = local;
10571
10572	if (env->idle != CPU_NEWLY_IDLE ||
10573	time_after_eq(jiffies, sg->sgc->next_update))
10574	update_group_capacity(sd: env->sd, cpu: env->dst_cpu);
10575	}
10576
10577	update_sg_lb_stats(env, sds, group: sg, sgs, sg_status: &sg_status);
10578
10579	if (!local_group && update_sd_pick_busiest(env, sds, sg, sgs)) {
10580	sds->busiest = sg;
10581	sds->busiest_stat = *sgs;
10582	}
10583
10584	/* Now, start updating sd_lb_stats */
10585	sds->total_load += sgs->group_load;
10586	sds->total_capacity += sgs->group_capacity;
10587
10588	sum_util += sgs->group_util;
10589	sg = sg->next;
10590	} while (sg != env->sd->groups);
10591
10592	/*
10593	* Indicate that the child domain of the busiest group prefers tasks
10594	* go to a child's sibling domains first. NB the flags of a sched group
10595	* are those of the child domain.
10596	*/
10597	if (sds->busiest)
10598	sds->prefer_sibling = !!(sds->busiest->flags & SD_PREFER_SIBLING);
10599
10600
10601	if (env->sd->flags & SD_NUMA)
10602	env->fbq_type = fbq_classify_group(sgs: &sds->busiest_stat);
10603
10604	if (!env->sd->parent) {
10605	struct root_domain *rd = env->dst_rq->rd;
10606
10607	/* update overload indicator if we are at root domain */
10608	WRITE_ONCE(rd->overload, sg_status & SG_OVERLOAD);
10609
10610	/* Update over-utilization (tipping point, U >= 0) indicator */
10611	WRITE_ONCE(rd->overutilized, sg_status & SG_OVERUTILIZED);
10612	trace_sched_overutilized_tp(rd, overutilized: sg_status & SG_OVERUTILIZED);
10613	} else if (sg_status & SG_OVERUTILIZED) {
10614	struct root_domain *rd = env->dst_rq->rd;
10615
10616	WRITE_ONCE(rd->overutilized, SG_OVERUTILIZED);
10617	trace_sched_overutilized_tp(rd, SG_OVERUTILIZED);
10618	}
10619
10620	update_idle_cpu_scan(env, sum_util);
10621	}
10622
10623	/**
10624	* calculate_imbalance - Calculate the amount of imbalance present within the
10625	* groups of a given sched_domain during load balance.
10626	* @env: load balance environment
10627	* @sds: statistics of the sched_domain whose imbalance is to be calculated.
10628	*/
10629	static inline void calculate_imbalance(struct lb_env *env, struct sd_lb_stats *sds)
10630	{
10631	struct sg_lb_stats *local, *busiest;
10632
10633	local = &sds->local_stat;
10634	busiest = &sds->busiest_stat;
10635
10636	if (busiest->group_type == group_misfit_task) {
10637	if (env->sd->flags & SD_ASYM_CPUCAPACITY) {
10638	/* Set imbalance to allow misfit tasks to be balanced. */
10639	env->migration_type = migrate_misfit;
10640	env->imbalance = 1;
10641	} else {
10642	/*
10643	* Set load imbalance to allow moving task from cpu
10644	* with reduced capacity.
10645	*/
10646	env->migration_type = migrate_load;
10647	env->imbalance = busiest->group_misfit_task_load;
10648	}
10649	return;
10650	}
10651
10652	if (busiest->group_type == group_asym_packing) {
10653	/*
10654	* In case of asym capacity, we will try to migrate all load to
10655	* the preferred CPU.
10656	*/
10657	env->migration_type = migrate_task;
10658	env->imbalance = busiest->sum_h_nr_running;
10659	return;
10660	}
10661
10662	if (busiest->group_type == group_smt_balance) {
10663	/* Reduce number of tasks sharing CPU capacity */
10664	env->migration_type = migrate_task;
10665	env->imbalance = 1;
10666	return;
10667	}
10668
10669	if (busiest->group_type == group_imbalanced) {
10670	/*
10671	* In the group_imb case we cannot rely on group-wide averages
10672	* to ensure CPU-load equilibrium, try to move any task to fix
10673	* the imbalance. The next load balance will take care of
10674	* balancing back the system.
10675	*/
10676	env->migration_type = migrate_task;
10677	env->imbalance = 1;
10678	return;
10679	}
10680
10681	/*
10682	* Try to use spare capacity of local group without overloading it or
10683	* emptying busiest.
10684	*/
10685	if (local->group_type == group_has_spare) {
10686	if ((busiest->group_type > group_fully_busy) &&
10687	!(env->sd->flags & SD_SHARE_LLC)) {
10688	/*
10689	* If busiest is overloaded, try to fill spare
10690	* capacity. This might end up creating spare capacity
10691	* in busiest or busiest still being overloaded but
10692	* there is no simple way to directly compute the
10693	* amount of load to migrate in order to balance the
10694	* system.
10695	*/
10696	env->migration_type = migrate_util;
10697	env->imbalance = max(local->group_capacity, local->group_util) -
10698	local->group_util;
10699
10700	/*
10701	* In some cases, the group's utilization is max or even
10702	* higher than capacity because of migrations but the
10703	* local CPU is (newly) idle. There is at least one
10704	* waiting task in this overloaded busiest group. Let's
10705	* try to pull it.
10706	*/
10707	if (env->idle != CPU_NOT_IDLE && env->imbalance == 0) {
10708	env->migration_type = migrate_task;
10709	env->imbalance = 1;
10710	}
10711
10712	return;
10713	}
10714
10715	if (busiest->group_weight == 1 || sds->prefer_sibling) {
10716	/*
10717	* When prefer sibling, evenly spread running tasks on
10718	* groups.
10719	*/
10720	env->migration_type = migrate_task;
10721	env->imbalance = sibling_imbalance(env, sds, busiest, local);
10722	} else {
10723
10724	/*
10725	* If there is no overload, we just want to even the number of
10726	* idle cpus.
10727	*/
10728	env->migration_type = migrate_task;
10729	env->imbalance = max_t(long, 0,
10730	(local->idle_cpus - busiest->idle_cpus));
10731	}
10732
10733	#ifdef CONFIG_NUMA
10734	/* Consider allowing a small imbalance between NUMA groups */
10735	if (env->sd->flags & SD_NUMA) {
10736	env->imbalance = adjust_numa_imbalance(imbalance: env->imbalance,
10737	dst_running: local->sum_nr_running + 1,
10738	imb_numa_nr: env->sd->imb_numa_nr);
10739	}
10740	#endif
10741
10742	/* Number of tasks to move to restore balance */
10743	env->imbalance >>= 1;
10744
10745	return;
10746	}
10747
10748	/*
10749	* Local is fully busy but has to take more load to relieve the
10750	* busiest group
10751	*/
10752	if (local->group_type < group_overloaded) {
10753	/*
10754	* Local will become overloaded so the avg_load metrics are
10755	* finally needed.
10756	*/
10757
10758	local->avg_load = (local->group_load * SCHED_CAPACITY_SCALE) /
10759	local->group_capacity;
10760
10761	/*
10762	* If the local group is more loaded than the selected
10763	* busiest group don't try to pull any tasks.
10764	*/
10765	if (local->avg_load >= busiest->avg_load) {
10766	env->imbalance = 0;
10767	return;
10768	}
10769
10770	sds->avg_load = (sds->total_load * SCHED_CAPACITY_SCALE) /
10771	sds->total_capacity;
10772
10773	/*
10774	* If the local group is more loaded than the average system
10775	* load, don't try to pull any tasks.
10776	*/
10777	if (local->avg_load >= sds->avg_load) {
10778	env->imbalance = 0;
10779	return;
10780	}
10781
10782	}
10783
10784	/*
10785	* Both group are or will become overloaded and we're trying to get all
10786	* the CPUs to the average_load, so we don't want to push ourselves
10787	* above the average load, nor do we wish to reduce the max loaded CPU
10788	* below the average load. At the same time, we also don't want to
10789	* reduce the group load below the group capacity. Thus we look for
10790	* the minimum possible imbalance.
10791	*/
10792	env->migration_type = migrate_load;
10793	env->imbalance = min(
10794	(busiest->avg_load - sds->avg_load) * busiest->group_capacity,
10795	(sds->avg_load - local->avg_load) * local->group_capacity
10796	) / SCHED_CAPACITY_SCALE;
10797	}
10798
10799	/******* find_busiest_group() helpers end here *********************/
10800
10801	/*
10802	* Decision matrix according to the local and busiest group type:
10803	*
10804	* busiest \ local has_spare fully_busy misfit asym imbalanced overloaded
10805	* has_spare nr_idle balanced N/A N/A balanced balanced
10806	* fully_busy nr_idle nr_idle N/A N/A balanced balanced
10807	* misfit_task force N/A N/A N/A N/A N/A
10808	* asym_packing force force N/A N/A force force
10809	* imbalanced force force N/A N/A force force
10810	* overloaded force force N/A N/A force avg_load
10811	*
10812	* N/A : Not Applicable because already filtered while updating
10813	* statistics.
10814	* balanced : The system is balanced for these 2 groups.
10815	* force : Calculate the imbalance as load migration is probably needed.
10816	* avg_load : Only if imbalance is significant enough.
10817	* nr_idle : dst_cpu is not busy and the number of idle CPUs is quite
10818	* different in groups.
10819	*/
10820
10821	/**
10822	* find_busiest_group - Returns the busiest group within the sched_domain
10823	* if there is an imbalance.
10824	* @env: The load balancing environment.
10825	*
10826	* Also calculates the amount of runnable load which should be moved
10827	* to restore balance.
10828	*
10829	* Return: - The busiest group if imbalance exists.
10830	*/
10831	static struct sched_group *find_busiest_group(struct lb_env *env)
10832	{
10833	struct sg_lb_stats *local, *busiest;
10834	struct sd_lb_stats sds;
10835
10836	init_sd_lb_stats(sds: &sds);
10837
10838	/*
10839	* Compute the various statistics relevant for load balancing at
10840	* this level.
10841	*/
10842	update_sd_lb_stats(env, sds: &sds);
10843
10844	/* There is no busy sibling group to pull tasks from */
10845	if (!sds.busiest)
10846	goto out_balanced;
10847
10848	busiest = &sds.busiest_stat;
10849
10850	/* Misfit tasks should be dealt with regardless of the avg load */
10851	if (busiest->group_type == group_misfit_task)
10852	goto force_balance;
10853
10854	if (sched_energy_enabled()) {
10855	struct root_domain *rd = env->dst_rq->rd;
10856
10857	if (rcu_dereference(rd->pd) && !READ_ONCE(rd->overutilized))
10858	goto out_balanced;
10859	}
10860
10861	/* ASYM feature bypasses nice load balance check */
10862	if (busiest->group_type == group_asym_packing)
10863	goto force_balance;
10864
10865	/*
10866	* If the busiest group is imbalanced the below checks don't
10867	* work because they assume all things are equal, which typically
10868	* isn't true due to cpus_ptr constraints and the like.
10869	*/
10870	if (busiest->group_type == group_imbalanced)
10871	goto force_balance;
10872
10873	local = &sds.local_stat;
10874	/*
10875	* If the local group is busier than the selected busiest group
10876	* don't try and pull any tasks.
10877	*/
10878	if (local->group_type > busiest->group_type)
10879	goto out_balanced;
10880
10881	/*
10882	* When groups are overloaded, use the avg_load to ensure fairness
10883	* between tasks.
10884	*/
10885	if (local->group_type == group_overloaded) {
10886	/*
10887	* If the local group is more loaded than the selected
10888	* busiest group don't try to pull any tasks.
10889	*/
10890	if (local->avg_load >= busiest->avg_load)
10891	goto out_balanced;
10892
10893	/* XXX broken for overlapping NUMA groups */
10894	sds.avg_load = (sds.total_load * SCHED_CAPACITY_SCALE) /
10895	sds.total_capacity;
10896
10897	/*
10898	* Don't pull any tasks if this group is already above the
10899	* domain average load.
10900	*/
10901	if (local->avg_load >= sds.avg_load)
10902	goto out_balanced;
10903
10904	/*
10905	* If the busiest group is more loaded, use imbalance_pct to be
10906	* conservative.
10907	*/
10908	if (100 * busiest->avg_load <=
10909	env->sd->imbalance_pct * local->avg_load)
10910	goto out_balanced;
10911	}
10912
10913	/*
10914	* Try to move all excess tasks to a sibling domain of the busiest
10915	* group's child domain.
10916	*/
10917	if (sds.prefer_sibling && local->group_type == group_has_spare &&
10918	sibling_imbalance(env, sds: &sds, busiest, local) > 1)
10919	goto force_balance;
10920
10921	if (busiest->group_type != group_overloaded) {
10922	if (env->idle == CPU_NOT_IDLE) {
10923	/*
10924	* If the busiest group is not overloaded (and as a
10925	* result the local one too) but this CPU is already
10926	* busy, let another idle CPU try to pull task.
10927	*/
10928	goto out_balanced;
10929	}
10930
10931	if (busiest->group_type == group_smt_balance &&
10932	smt_vs_nonsmt_groups(sg1: sds.local, sg2: sds.busiest)) {
10933	/* Let non SMT CPU pull from SMT CPU sharing with sibling */
10934	goto force_balance;
10935	}
10936
10937	if (busiest->group_weight > 1 &&
10938	local->idle_cpus <= (busiest->idle_cpus + 1)) {
10939	/*
10940	* If the busiest group is not overloaded
10941	* and there is no imbalance between this and busiest
10942	* group wrt idle CPUs, it is balanced. The imbalance
10943	* becomes significant if the diff is greater than 1
10944	* otherwise we might end up to just move the imbalance
10945	* on another group. Of course this applies only if
10946	* there is more than 1 CPU per group.
10947	*/
10948	goto out_balanced;
10949	}
10950
10951	if (busiest->sum_h_nr_running == 1) {
10952	/*
10953	* busiest doesn't have any tasks waiting to run
10954	*/
10955	goto out_balanced;
10956	}
10957	}
10958
10959	force_balance:
10960	/* Looks like there is an imbalance. Compute it */
10961	calculate_imbalance(env, sds: &sds);
10962	return env->imbalance ? sds.busiest : NULL;
10963
10964	out_balanced:
10965	env->imbalance = 0;
10966	return NULL;
10967	}
10968
10969	/*
10970	* find_busiest_queue - find the busiest runqueue among the CPUs in the group.
10971	*/
10972	static struct rq *find_busiest_queue(struct lb_env *env,
10973	struct sched_group *group)
10974	{
10975	struct rq *busiest = NULL, *rq;
10976	unsigned long busiest_util = 0, busiest_load = 0, busiest_capacity = 1;
10977	unsigned int busiest_nr = 0;
10978	int i;
10979
10980	for_each_cpu_and(i, sched_group_span(group), env->cpus) {
10981	unsigned long capacity, load, util;
10982	unsigned int nr_running;
10983	enum fbq_type rt;
10984
10985	rq = cpu_rq(i);
10986	rt = fbq_classify_rq(rq);
10987
10988	/*
10989	* We classify groups/runqueues into three groups:
10990	* - regular: there are !numa tasks
10991	* - remote: there are numa tasks that run on the 'wrong' node
10992	* - all: there is no distinction
10993	*
10994	* In order to avoid migrating ideally placed numa tasks,
10995	* ignore those when there's better options.
10996	*
10997	* If we ignore the actual busiest queue to migrate another
10998	* task, the next balance pass can still reduce the busiest
10999	* queue by moving tasks around inside the node.
11000	*
11001	* If we cannot move enough load due to this classification
11002	* the next pass will adjust the group classification and
11003	* allow migration of more tasks.
11004	*
11005	* Both cases only affect the total convergence complexity.
11006	*/
11007	if (rt > env->fbq_type)
11008	continue;
11009
11010	nr_running = rq->cfs.h_nr_running;
11011	if (!nr_running)
11012	continue;
11013
11014	capacity = capacity_of(cpu: i);
11015
11016	/*
11017	* For ASYM_CPUCAPACITY domains, don't pick a CPU that could
11018	* eventually lead to active_balancing high->low capacity.
11019	* Higher per-CPU capacity is considered better than balancing
11020	* average load.
11021	*/
11022	if (env->sd->flags & SD_ASYM_CPUCAPACITY &&
11023	!capacity_greater(capacity_of(env->dst_cpu), capacity) &&
11024	nr_running == 1)
11025	continue;
11026
11027	/*
11028	* Make sure we only pull tasks from a CPU of lower priority
11029	* when balancing between SMT siblings.
11030	*
11031	* If balancing between cores, let lower priority CPUs help
11032	* SMT cores with more than one busy sibling.
11033	*/
11034	if (sched_asym(sd: env->sd, dst_cpu: i, src_cpu: env->dst_cpu) && nr_running == 1)
11035	continue;
11036
11037	switch (env->migration_type) {
11038	case migrate_load:
11039	/*
11040	* When comparing with load imbalance, use cpu_load()
11041	* which is not scaled with the CPU capacity.
11042	*/
11043	load = cpu_load(rq);
11044
11045	if (nr_running == 1 && load > env->imbalance &&
11046	!check_cpu_capacity(rq, sd: env->sd))
11047	break;
11048
11049	/*
11050	* For the load comparisons with the other CPUs,
11051	* consider the cpu_load() scaled with the CPU
11052	* capacity, so that the load can be moved away
11053	* from the CPU that is potentially running at a
11054	* lower capacity.
11055	*
11056	* Thus we're looking for max(load_i / capacity_i),
11057	* crosswise multiplication to rid ourselves of the
11058	* division works out to:
11059	* load_i * capacity_j > load_j * capacity_i;
11060	* where j is our previous maximum.
11061	*/
11062	if (load * busiest_capacity > busiest_load * capacity) {
11063	busiest_load = load;
11064	busiest_capacity = capacity;
11065	busiest = rq;
11066	}
11067	break;
11068
11069	case migrate_util:
11070	util = cpu_util_cfs_boost(cpu: i);
11071
11072	/*
11073	* Don't try to pull utilization from a CPU with one
11074	* running task. Whatever its utilization, we will fail
11075	* detach the task.
11076	*/
11077	if (nr_running <= 1)
11078	continue;
11079
11080	if (busiest_util < util) {
11081	busiest_util = util;
11082	busiest = rq;
11083	}
11084	break;
11085
11086	case migrate_task:
11087	if (busiest_nr < nr_running) {
11088	busiest_nr = nr_running;
11089	busiest = rq;
11090	}
11091	break;
11092
11093	case migrate_misfit:
11094	/*
11095	* For ASYM_CPUCAPACITY domains with misfit tasks we
11096	* simply seek the "biggest" misfit task.
11097	*/
11098	if (rq->misfit_task_load > busiest_load) {
11099	busiest_load = rq->misfit_task_load;
11100	busiest = rq;
11101	}
11102
11103	break;
11104
11105	}
11106	}
11107
11108	return busiest;
11109	}
11110
11111	/*
11112	* Max backoff if we encounter pinned tasks. Pretty arbitrary value, but
11113	* so long as it is large enough.
11114	*/
11115	#define MAX_PINNED_INTERVAL 512
11116
11117	static inline bool
11118	asym_active_balance(struct lb_env *env)
11119	{
11120	/*
11121	* ASYM_PACKING needs to force migrate tasks from busy but lower
11122	* priority CPUs in order to pack all tasks in the highest priority
11123	* CPUs. When done between cores, do it only if the whole core if the
11124	* whole core is idle.
11125	*
11126	* If @env::src_cpu is an SMT core with busy siblings, let
11127	* the lower priority @env::dst_cpu help it. Do not follow
11128	* CPU priority.
11129	*/
11130	return env->idle != CPU_NOT_IDLE && sched_use_asym_prio(sd: env->sd, cpu: env->dst_cpu) &&
11131	(sched_asym_prefer(a: env->dst_cpu, b: env->src_cpu) ||
11132	!sched_use_asym_prio(sd: env->sd, cpu: env->src_cpu));
11133	}
11134
11135	static inline bool
11136	imbalanced_active_balance(struct lb_env *env)
11137	{
11138	struct sched_domain *sd = env->sd;
11139
11140	/*
11141	* The imbalanced case includes the case of pinned tasks preventing a fair
11142	* distribution of the load on the system but also the even distribution of the
11143	* threads on a system with spare capacity
11144	*/
11145	if ((env->migration_type == migrate_task) &&
11146	(sd->nr_balance_failed > sd->cache_nice_tries+2))
11147	return 1;
11148
11149	return 0;
11150	}
11151
11152	static int need_active_balance(struct lb_env *env)
11153	{
11154	struct sched_domain *sd = env->sd;
11155
11156	if (asym_active_balance(env))
11157	return 1;
11158
11159	if (imbalanced_active_balance(env))
11160	return 1;
11161
11162	/*
11163	* The dst_cpu is idle and the src_cpu CPU has only 1 CFS task.
11164	* It's worth migrating the task if the src_cpu's capacity is reduced
11165	* because of other sched_class or IRQs if more capacity stays
11166	* available on dst_cpu.
11167	*/
11168	if ((env->idle != CPU_NOT_IDLE) &&
11169	(env->src_rq->cfs.h_nr_running == 1)) {
11170	if ((check_cpu_capacity(rq: env->src_rq, sd)) &&
11171	(capacity_of(cpu: env->src_cpu)*sd->imbalance_pct < capacity_of(cpu: env->dst_cpu)*100))
11172	return 1;
11173	}
11174
11175	if (env->migration_type == migrate_misfit)
11176	return 1;
11177
11178	return 0;
11179	}
11180
11181	static int active_load_balance_cpu_stop(void *data);
11182
11183	static int should_we_balance(struct lb_env *env)
11184	{
11185	struct cpumask *swb_cpus = this_cpu_cpumask_var_ptr(should_we_balance_tmpmask);
11186	struct sched_group *sg = env->sd->groups;
11187	int cpu, idle_smt = -1;
11188
11189	/*
11190	* Ensure the balancing environment is consistent; can happen
11191	* when the softirq triggers 'during' hotplug.
11192	*/
11193	if (!cpumask_test_cpu(cpu: env->dst_cpu, cpumask: env->cpus))
11194	return 0;
11195
11196	/*
11197	* In the newly idle case, we will allow all the CPUs
11198	* to do the newly idle load balance.
11199	*
11200	* However, we bail out if we already have tasks or a wakeup pending,
11201	* to optimize wakeup latency.
11202	*/
11203	if (env->idle == CPU_NEWLY_IDLE) {
11204	if (env->dst_rq->nr_running > 0 || env->dst_rq->ttwu_pending)
11205	return 0;
11206	return 1;
11207	}
11208
11209	cpumask_copy(dstp: swb_cpus, srcp: group_balance_mask(sg));
11210	/* Try to find first idle CPU */
11211	for_each_cpu_and(cpu, swb_cpus, env->cpus) {
11212	if (!idle_cpu(cpu))
11213	continue;
11214
11215	/*
11216	* Don't balance to idle SMT in busy core right away when
11217	* balancing cores, but remember the first idle SMT CPU for
11218	* later consideration. Find CPU on an idle core first.
11219	*/
11220	if (!(env->sd->flags & SD_SHARE_CPUCAPACITY) && !is_core_idle(cpu)) {
11221	if (idle_smt == -1)
11222	idle_smt = cpu;
11223	/*
11224	* If the core is not idle, and first SMT sibling which is
11225	* idle has been found, then its not needed to check other
11226	* SMT siblings for idleness:
11227	*/
11228	#ifdef CONFIG_SCHED_SMT
11229	cpumask_andnot(dstp: swb_cpus, src1p: swb_cpus, src2p: cpu_smt_mask(cpu));
11230	#endif
11231	continue;
11232	}
11233
11234	/*
11235	* Are we the first idle core in a non-SMT domain or higher,
11236	* or the first idle CPU in a SMT domain?
11237	*/
11238	return cpu == env->dst_cpu;
11239	}
11240
11241	/* Are we the first idle CPU with busy siblings? */
11242	if (idle_smt != -1)
11243	return idle_smt == env->dst_cpu;
11244
11245	/* Are we the first CPU of this group ? */
11246	return group_balance_cpu(sg) == env->dst_cpu;
11247	}
11248
11249	/*
11250	* Check this_cpu to ensure it is balanced within domain. Attempt to move
11251	* tasks if there is an imbalance.
11252	*/
11253	static int load_balance(int this_cpu, struct rq *this_rq,
11254	struct sched_domain *sd, enum cpu_idle_type idle,
11255	int *continue_balancing)
11256	{
11257	int ld_moved, cur_ld_moved, active_balance = 0;
11258	struct sched_domain *sd_parent = sd->parent;
11259	struct sched_group *group;
11260	struct rq *busiest;
11261	struct rq_flags rf;
11262	struct cpumask *cpus = this_cpu_cpumask_var_ptr(load_balance_mask);
11263	struct lb_env env = {
11264	.sd = sd,
11265	.dst_cpu = this_cpu,
11266	.dst_rq = this_rq,
11267	.dst_grpmask = group_balance_mask(sg: sd->groups),
11268	.idle = idle,
11269	.loop_break = SCHED_NR_MIGRATE_BREAK,
11270	.cpus = cpus,
11271	.fbq_type = all,
11272	.tasks = LIST_HEAD_INIT(env.tasks),
11273	};
11274
11275	cpumask_and(dstp: cpus, src1p: sched_domain_span(sd), cpu_active_mask);
11276
11277	schedstat_inc(sd->lb_count[idle]);
11278
11279	redo:
11280	if (!should_we_balance(env: &env)) {
11281	*continue_balancing = 0;
11282	goto out_balanced;
11283	}
11284
11285	group = find_busiest_group(env: &env);
11286	if (!group) {
11287	schedstat_inc(sd->lb_nobusyg[idle]);
11288	goto out_balanced;
11289	}
11290
11291	busiest = find_busiest_queue(env: &env, group);
11292	if (!busiest) {
11293	schedstat_inc(sd->lb_nobusyq[idle]);
11294	goto out_balanced;
11295	}
11296
11297	WARN_ON_ONCE(busiest == env.dst_rq);
11298
11299	schedstat_add(sd->lb_imbalance[idle], env.imbalance);
11300
11301	env.src_cpu = busiest->cpu;
11302	env.src_rq = busiest;
11303
11304	ld_moved = 0;
11305	/* Clear this flag as soon as we find a pullable task */
11306	env.flags |= LBF_ALL_PINNED;
11307	if (busiest->nr_running > 1) {
11308	/*
11309	* Attempt to move tasks. If find_busiest_group has found
11310	* an imbalance but busiest->nr_running <= 1, the group is
11311	* still unbalanced. ld_moved simply stays zero, so it is
11312	* correctly treated as an imbalance.
11313	*/
11314	env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);
11315
11316	more_balance:
11317	rq_lock_irqsave(rq: busiest, rf: &rf);
11318	update_rq_clock(rq: busiest);
11319
11320	/*
11321	* cur_ld_moved - load moved in current iteration
11322	* ld_moved - cumulative load moved across iterations
11323	*/
11324	cur_ld_moved = detach_tasks(env: &env);
11325
11326	/*
11327	* We've detached some tasks from busiest_rq. Every
11328	* task is masked "TASK_ON_RQ_MIGRATING", so we can safely
11329	* unlock busiest->lock, and we are able to be sure
11330	* that nobody can manipulate the tasks in parallel.
11331	* See task_rq_lock() family for the details.
11332	*/
11333
11334	rq_unlock(rq: busiest, rf: &rf);
11335
11336	if (cur_ld_moved) {
11337	attach_tasks(env: &env);
11338	ld_moved += cur_ld_moved;
11339	}
11340
11341	local_irq_restore(rf.flags);
11342
11343	if (env.flags & LBF_NEED_BREAK) {
11344	env.flags &= ~LBF_NEED_BREAK;
11345	/* Stop if we tried all running tasks */
11346	if (env.loop < busiest->nr_running)
11347	goto more_balance;
11348	}
11349
11350	/*
11351	* Revisit (affine) tasks on src_cpu that couldn't be moved to
11352	* us and move them to an alternate dst_cpu in our sched_group
11353	* where they can run. The upper limit on how many times we
11354	* iterate on same src_cpu is dependent on number of CPUs in our
11355	* sched_group.
11356	*
11357	* This changes load balance semantics a bit on who can move
11358	* load to a given_cpu. In addition to the given_cpu itself
11359	* (or a ilb_cpu acting on its behalf where given_cpu is
11360	* nohz-idle), we now have balance_cpu in a position to move
11361	* load to given_cpu. In rare situations, this may cause
11362	* conflicts (balance_cpu and given_cpu/ilb_cpu deciding
11363	* _independently_ and at _same_ time to move some load to
11364	* given_cpu) causing excess load to be moved to given_cpu.
11365	* This however should not happen so much in practice and
11366	* moreover subsequent load balance cycles should correct the
11367	* excess load moved.
11368	*/
11369	if ((env.flags & LBF_DST_PINNED) && env.imbalance > 0) {
11370
11371	/* Prevent to re-select dst_cpu via env's CPUs */
11372	__cpumask_clear_cpu(cpu: env.dst_cpu, dstp: env.cpus);
11373
11374	env.dst_rq = cpu_rq(env.new_dst_cpu);
11375	env.dst_cpu = env.new_dst_cpu;
11376	env.flags &= ~LBF_DST_PINNED;
11377	env.loop = 0;
11378	env.loop_break = SCHED_NR_MIGRATE_BREAK;
11379
11380	/*
11381	* Go back to "more_balance" rather than "redo" since we
11382	* need to continue with same src_cpu.
11383	*/
11384	goto more_balance;
11385	}
11386
11387	/*
11388	* We failed to reach balance because of affinity.
11389	*/
11390	if (sd_parent) {
11391	int *group_imbalance = &sd_parent->groups->sgc->imbalance;
11392
11393	if ((env.flags & LBF_SOME_PINNED) && env.imbalance > 0)
11394	*group_imbalance = 1;
11395	}
11396
11397	/* All tasks on this runqueue were pinned by CPU affinity */
11398	if (unlikely(env.flags & LBF_ALL_PINNED)) {
11399	__cpumask_clear_cpu(cpu: cpu_of(rq: busiest), dstp: cpus);
11400	/*
11401	* Attempting to continue load balancing at the current
11402	* sched_domain level only makes sense if there are
11403	* active CPUs remaining as possible busiest CPUs to
11404	* pull load from which are not contained within the
11405	* destination group that is receiving any migrated
11406	* load.
11407	*/
11408	if (!cpumask_subset(src1p: cpus, src2p: env.dst_grpmask)) {
11409	env.loop = 0;
11410	env.loop_break = SCHED_NR_MIGRATE_BREAK;
11411	goto redo;
11412	}
11413	goto out_all_pinned;
11414	}
11415	}
11416
11417	if (!ld_moved) {
11418	schedstat_inc(sd->lb_failed[idle]);
11419	/*
11420	* Increment the failure counter only on periodic balance.
11421	* We do not want newidle balance, which can be very
11422	* frequent, pollute the failure counter causing
11423	* excessive cache_hot migrations and active balances.
11424	*/
11425	if (idle != CPU_NEWLY_IDLE)
11426	sd->nr_balance_failed++;
11427
11428	if (need_active_balance(env: &env)) {
11429	unsigned long flags;
11430
11431	raw_spin_rq_lock_irqsave(busiest, flags);
11432
11433	/*
11434	* Don't kick the active_load_balance_cpu_stop,
11435	* if the curr task on busiest CPU can't be
11436	* moved to this_cpu:
11437	*/
11438	if (!cpumask_test_cpu(cpu: this_cpu, cpumask: busiest->curr->cpus_ptr)) {
11439	raw_spin_rq_unlock_irqrestore(rq: busiest, flags);
11440	goto out_one_pinned;
11441	}
11442
11443	/* Record that we found at least one task that could run on this_cpu */
11444	env.flags &= ~LBF_ALL_PINNED;
11445
11446	/*
11447	* ->active_balance synchronizes accesses to
11448	* ->active_balance_work. Once set, it's cleared
11449	* only after active load balance is finished.
11450	*/
11451	if (!busiest->active_balance) {
11452	busiest->active_balance = 1;
11453	busiest->push_cpu = this_cpu;
11454	active_balance = 1;
11455	}
11456
11457	preempt_disable();
11458	raw_spin_rq_unlock_irqrestore(rq: busiest, flags);
11459	if (active_balance) {
11460	stop_one_cpu_nowait(cpu: cpu_of(rq: busiest),
11461	fn: active_load_balance_cpu_stop, arg: busiest,
11462	work_buf: &busiest->active_balance_work);
11463	}
11464	preempt_enable();
11465	}
11466	} else {
11467	sd->nr_balance_failed = 0;
11468	}
11469
11470	if (likely(!active_balance) || need_active_balance(env: &env)) {
11471	/* We were unbalanced, so reset the balancing interval */
11472	sd->balance_interval = sd->min_interval;
11473	}
11474
11475	goto out;
11476
11477	out_balanced:
11478	/*
11479	* We reach balance although we may have faced some affinity
11480	* constraints. Clear the imbalance flag only if other tasks got
11481	* a chance to move and fix the imbalance.
11482	*/
11483	if (sd_parent && !(env.flags & LBF_ALL_PINNED)) {
11484	int *group_imbalance = &sd_parent->groups->sgc->imbalance;
11485
11486	if (*group_imbalance)
11487	*group_imbalance = 0;
11488	}
11489
11490	out_all_pinned:
11491	/*
11492	* We reach balance because all tasks are pinned at this level so
11493	* we can't migrate them. Let the imbalance flag set so parent level
11494	* can try to migrate them.
11495	*/
11496	schedstat_inc(sd->lb_balanced[idle]);
11497
11498	sd->nr_balance_failed = 0;
11499
11500	out_one_pinned:
11501	ld_moved = 0;
11502
11503	/*
11504	* newidle_balance() disregards balance intervals, so we could
11505	* repeatedly reach this code, which would lead to balance_interval
11506	* skyrocketing in a short amount of time. Skip the balance_interval
11507	* increase logic to avoid that.
11508	*/
11509	if (env.idle == CPU_NEWLY_IDLE)
11510	goto out;
11511
11512	/* tune up the balancing interval */
11513	if ((env.flags & LBF_ALL_PINNED &&
11514	sd->balance_interval < MAX_PINNED_INTERVAL) ||
11515	sd->balance_interval < sd->max_interval)
11516	sd->balance_interval *= 2;
11517	out:
11518	return ld_moved;
11519	}
11520
11521	static inline unsigned long
11522	get_sd_balance_interval(struct sched_domain *sd, int cpu_busy)
11523	{
11524	unsigned long interval = sd->balance_interval;
11525
11526	if (cpu_busy)
11527	interval *= sd->busy_factor;
11528
11529	/* scale ms to jiffies */
11530	interval = msecs_to_jiffies(m: interval);
11531
11532	/*
11533	* Reduce likelihood of busy balancing at higher domains racing with
11534	* balancing at lower domains by preventing their balancing periods
11535	* from being multiples of each other.
11536	*/
11537	if (cpu_busy)
11538	interval -= 1;
11539
11540	interval = clamp(interval, 1UL, max_load_balance_interval);
11541
11542	return interval;
11543	}
11544
11545	static inline void
11546	update_next_balance(struct sched_domain *sd, unsigned long *next_balance)
11547	{
11548	unsigned long interval, next;
11549
11550	/* used by idle balance, so cpu_busy = 0 */
11551	interval = get_sd_balance_interval(sd, cpu_busy: 0);
11552	next = sd->last_balance + interval;
11553
11554	if (time_after(*next_balance, next))
11555	*next_balance = next;
11556	}
11557
11558	/*
11559	* active_load_balance_cpu_stop is run by the CPU stopper. It pushes
11560	* running tasks off the busiest CPU onto idle CPUs. It requires at
11561	* least 1 task to be running on each physical CPU where possible, and
11562	* avoids physical / logical imbalances.
11563	*/
11564	static int active_load_balance_cpu_stop(void *data)
11565	{
11566	struct rq *busiest_rq = data;
11567	int busiest_cpu = cpu_of(rq: busiest_rq);
11568	int target_cpu = busiest_rq->push_cpu;
11569	struct rq *target_rq = cpu_rq(target_cpu);
11570	struct sched_domain *sd;
11571	struct task_struct *p = NULL;
11572	struct rq_flags rf;
11573
11574	rq_lock_irq(rq: busiest_rq, rf: &rf);
11575	/*
11576	* Between queueing the stop-work and running it is a hole in which
11577	* CPUs can become inactive. We should not move tasks from or to
11578	* inactive CPUs.
11579	*/
11580	if (!cpu_active(cpu: busiest_cpu) || !cpu_active(cpu: target_cpu))
11581	goto out_unlock;
11582
11583	/* Make sure the requested CPU hasn't gone down in the meantime: */
11584	if (unlikely(busiest_cpu != smp_processor_id() ||
11585	!busiest_rq->active_balance))
11586	goto out_unlock;
11587
11588	/* Is there any task to move? */
11589	if (busiest_rq->nr_running <= 1)
11590	goto out_unlock;
11591
11592	/*
11593	* This condition is "impossible", if it occurs
11594	* we need to fix it. Originally reported by
11595	* Bjorn Helgaas on a 128-CPU setup.
11596	*/
11597	WARN_ON_ONCE(busiest_rq == target_rq);
11598
11599	/* Search for an sd spanning us and the target CPU. */
11600	rcu_read_lock();
11601	for_each_domain(target_cpu, sd) {
11602	if (cpumask_test_cpu(cpu: busiest_cpu, cpumask: sched_domain_span(sd)))
11603	break;
11604	}
11605
11606	if (likely(sd)) {
11607	struct lb_env env = {
11608	.sd = sd,
11609	.dst_cpu = target_cpu,
11610	.dst_rq = target_rq,
11611	.src_cpu = busiest_rq->cpu,
11612	.src_rq = busiest_rq,
11613	.idle = CPU_IDLE,
11614	.flags = LBF_ACTIVE_LB,
11615	};
11616
11617	schedstat_inc(sd->alb_count);
11618	update_rq_clock(rq: busiest_rq);
11619
11620	p = detach_one_task(env: &env);
11621	if (p) {
11622	schedstat_inc(sd->alb_pushed);
11623	/* Active balancing done, reset the failure counter. */
11624	sd->nr_balance_failed = 0;
11625	} else {
11626	schedstat_inc(sd->alb_failed);
11627	}
11628	}
11629	rcu_read_unlock();
11630	out_unlock:
11631	busiest_rq->active_balance = 0;
11632	rq_unlock(rq: busiest_rq, rf: &rf);
11633
11634	if (p)
11635	attach_one_task(rq: target_rq, p);
11636
11637	local_irq_enable();
11638
11639	return 0;
11640	}
11641
11642	static DEFINE_SPINLOCK(balancing);
11643
11644	/*
11645	* Scale the max load_balance interval with the number of CPUs in the system.
11646	* This trades load-balance latency on larger machines for less cross talk.
11647	*/
11648	void update_max_interval(void)
11649	{
11650	max_load_balance_interval = HZ*num_online_cpus()/10;
11651	}
11652
11653	static inline bool update_newidle_cost(struct sched_domain *sd, u64 cost)
11654	{
11655	if (cost > sd->max_newidle_lb_cost) {
11656	/*
11657	* Track max cost of a domain to make sure to not delay the
11658	* next wakeup on the CPU.
11659	*/
11660	sd->max_newidle_lb_cost = cost;
11661	sd->last_decay_max_lb_cost = jiffies;
11662	} else if (time_after(jiffies, sd->last_decay_max_lb_cost + HZ)) {
11663	/*
11664	* Decay the newidle max times by ~1% per second to ensure that
11665	* it is not outdated and the current max cost is actually
11666	* shorter.
11667	*/
11668	sd->max_newidle_lb_cost = (sd->max_newidle_lb_cost * 253) / 256;
11669	sd->last_decay_max_lb_cost = jiffies;
11670
11671	return true;
11672	}
11673
11674	return false;
11675	}
11676
11677	/*
11678	* It checks each scheduling domain to see if it is due to be balanced,
11679	* and initiates a balancing operation if so.
11680	*
11681	* Balancing parameters are set up in init_sched_domains.
11682	*/
11683	static void rebalance_domains(struct rq *rq, enum cpu_idle_type idle)
11684	{
11685	int continue_balancing = 1;
11686	int cpu = rq->cpu;
11687	int busy = idle != CPU_IDLE && !sched_idle_cpu(cpu);
11688	unsigned long interval;
11689	struct sched_domain *sd;
11690	/* Earliest time when we have to do rebalance again */
11691	unsigned long next_balance = jiffies + 60*HZ;
11692	int update_next_balance = 0;
11693	int need_serialize, need_decay = 0;
11694	u64 max_cost = 0;
11695
11696	rcu_read_lock();
11697	for_each_domain(cpu, sd) {
11698	/*
11699	* Decay the newidle max times here because this is a regular
11700	* visit to all the domains.
11701	*/
11702	need_decay = update_newidle_cost(sd, cost: 0);
11703	max_cost += sd->max_newidle_lb_cost;
11704
11705	/*
11706	* Stop the load balance at this level. There is another
11707	* CPU in our sched group which is doing load balancing more
11708	* actively.
11709	*/
11710	if (!continue_balancing) {
11711	if (need_decay)
11712	continue;
11713	break;
11714	}
11715
11716	interval = get_sd_balance_interval(sd, cpu_busy: busy);
11717
11718	need_serialize = sd->flags & SD_SERIALIZE;
11719	if (need_serialize) {
11720	if (!spin_trylock(lock: &balancing))
11721	goto out;
11722	}
11723
11724	if (time_after_eq(jiffies, sd->last_balance + interval)) {
11725	if (load_balance(this_cpu: cpu, this_rq: rq, sd, idle, continue_balancing: &continue_balancing)) {
11726	/*
11727	* The LBF_DST_PINNED logic could have changed
11728	* env->dst_cpu, so we can't know our idle
11729	* state even if we migrated tasks. Update it.
11730	*/
11731	idle = idle_cpu(cpu) ? CPU_IDLE : CPU_NOT_IDLE;
11732	busy = idle != CPU_IDLE && !sched_idle_cpu(cpu);
11733	}
11734	sd->last_balance = jiffies;
11735	interval = get_sd_balance_interval(sd, cpu_busy: busy);
11736	}
11737	if (need_serialize)
11738	spin_unlock(lock: &balancing);
11739	out:
11740	if (time_after(next_balance, sd->last_balance + interval)) {
11741	next_balance = sd->last_balance + interval;
11742	update_next_balance = 1;
11743	}
11744	}
11745	if (need_decay) {
11746	/*
11747	* Ensure the rq-wide value also decays but keep it at a
11748	* reasonable floor to avoid funnies with rq->avg_idle.
11749	*/
11750	rq->max_idle_balance_cost =
11751	max((u64)sysctl_sched_migration_cost, max_cost);
11752	}
11753	rcu_read_unlock();
11754
11755	/*
11756	* next_balance will be updated only when there is a need.
11757	* When the cpu is attached to null domain for ex, it will not be
11758	* updated.
11759	*/
11760	if (likely(update_next_balance))
11761	rq->next_balance = next_balance;
11762
11763	}
11764
11765	static inline int on_null_domain(struct rq *rq)
11766	{
11767	return unlikely(!rcu_dereference_sched(rq->sd));
11768	}
11769
11770	#ifdef CONFIG_NO_HZ_COMMON
11771	/*
11772	* NOHZ idle load balancing (ILB) details:
11773	*
11774	* - When one of the busy CPUs notices that there may be an idle rebalancing
11775	* needed, they will kick the idle load balancer, which then does idle
11776	* load balancing for all the idle CPUs.
11777	*
11778	* - HK_TYPE_MISC CPUs are used for this task, because HK_TYPE_SCHED is not set
11779	* anywhere yet.
11780	*/
11781	static inline int find_new_ilb(void)
11782	{
11783	const struct cpumask *hk_mask;
11784	int ilb_cpu;
11785
11786	hk_mask = housekeeping_cpumask(type: HK_TYPE_MISC);
11787
11788	for_each_cpu_and(ilb_cpu, nohz.idle_cpus_mask, hk_mask) {
11789
11790	if (ilb_cpu == smp_processor_id())
11791	continue;
11792
11793	if (idle_cpu(cpu: ilb_cpu))
11794	return ilb_cpu;
11795	}
11796
11797	return -1;
11798	}
11799
11800	/*
11801	* Kick a CPU to do the NOHZ balancing, if it is time for it, via a cross-CPU
11802	* SMP function call (IPI).
11803	*
11804	* We pick the first idle CPU in the HK_TYPE_MISC housekeeping set (if there is one).
11805	*/
11806	static void kick_ilb(unsigned int flags)
11807	{
11808	int ilb_cpu;
11809
11810	/*
11811	* Increase nohz.next_balance only when if full ilb is triggered but
11812	* not if we only update stats.
11813	*/
11814	if (flags & NOHZ_BALANCE_KICK)
11815	nohz.next_balance = jiffies+1;
11816
11817	ilb_cpu = find_new_ilb();
11818	if (ilb_cpu < 0)
11819	return;
11820
11821	/*
11822	* Access to rq::nohz_csd is serialized by NOHZ_KICK_MASK; he who sets
11823	* the first flag owns it; cleared by nohz_csd_func().
11824	*/
11825	flags = atomic_fetch_or(i: flags, nohz_flags(ilb_cpu));
11826	if (flags & NOHZ_KICK_MASK)
11827	return;
11828
11829	/*
11830	* This way we generate an IPI on the target CPU which
11831	* is idle, and the softirq performing NOHZ idle load balancing
11832	* will be run before returning from the IPI.
11833	*/
11834	smp_call_function_single_async(cpu: ilb_cpu, csd: &cpu_rq(ilb_cpu)->nohz_csd);
11835	}
11836
11837	/*
11838	* Current decision point for kicking the idle load balancer in the presence
11839	* of idle CPUs in the system.
11840	*/
11841	static void nohz_balancer_kick(struct rq *rq)
11842	{
11843	unsigned long now = jiffies;
11844	struct sched_domain_shared *sds;
11845	struct sched_domain *sd;
11846	int nr_busy, i, cpu = rq->cpu;
11847	unsigned int flags = 0;
11848
11849	if (unlikely(rq->idle_balance))
11850	return;
11851
11852	/*
11853	* We may be recently in ticked or tickless idle mode. At the first
11854	* busy tick after returning from idle, we will update the busy stats.
11855	*/
11856	nohz_balance_exit_idle(rq);
11857
11858	/*
11859	* None are in tickless mode and hence no need for NOHZ idle load
11860	* balancing:
11861	*/
11862	if (likely(!atomic_read(&nohz.nr_cpus)))
11863	return;
11864
11865	if (READ_ONCE(nohz.has_blocked) &&
11866	time_after(now, READ_ONCE(nohz.next_blocked)))
11867	flags = NOHZ_STATS_KICK;
11868
11869	if (time_before(now, nohz.next_balance))
11870	goto out;
11871
11872	if (rq->nr_running >= 2) {
11873	flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
11874	goto out;
11875	}
11876
11877	rcu_read_lock();
11878
11879	sd = rcu_dereference(rq->sd);
11880	if (sd) {
11881	/*
11882	* If there's a runnable CFS task and the current CPU has reduced
11883	* capacity, kick the ILB to see if there's a better CPU to run on:
11884	*/
11885	if (rq->cfs.h_nr_running >= 1 && check_cpu_capacity(rq, sd)) {
11886	flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
11887	goto unlock;
11888	}
11889	}
11890
11891	sd = rcu_dereference(per_cpu(sd_asym_packing, cpu));
11892	if (sd) {
11893	/*
11894	* When ASYM_PACKING; see if there's a more preferred CPU
11895	* currently idle; in which case, kick the ILB to move tasks
11896	* around.
11897	*
11898	* When balancing betwen cores, all the SMT siblings of the
11899	* preferred CPU must be idle.
11900	*/
11901	for_each_cpu_and(i, sched_domain_span(sd), nohz.idle_cpus_mask) {
11902	if (sched_asym(sd, dst_cpu: i, src_cpu: cpu)) {
11903	flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
11904	goto unlock;
11905	}
11906	}
11907	}
11908
11909	sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, cpu));
11910	if (sd) {
11911	/*
11912	* When ASYM_CPUCAPACITY; see if there's a higher capacity CPU
11913	* to run the misfit task on.
11914	*/
11915	if (check_misfit_status(rq, sd)) {
11916	flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
11917	goto unlock;
11918	}
11919
11920	/*
11921	* For asymmetric systems, we do not want to nicely balance
11922	* cache use, instead we want to embrace asymmetry and only
11923	* ensure tasks have enough CPU capacity.
11924	*
11925	* Skip the LLC logic because it's not relevant in that case.
11926	*/
11927	goto unlock;
11928	}
11929
11930	sds = rcu_dereference(per_cpu(sd_llc_shared, cpu));
11931	if (sds) {
11932	/*
11933	* If there is an imbalance between LLC domains (IOW we could
11934	* increase the overall cache utilization), we need a less-loaded LLC
11935	* domain to pull some load from. Likewise, we may need to spread
11936	* load within the current LLC domain (e.g. packed SMT cores but
11937	* other CPUs are idle). We can't really know from here how busy
11938	* the others are - so just get a NOHZ balance going if it looks
11939	* like this LLC domain has tasks we could move.
11940	*/
11941	nr_busy = atomic_read(v: &sds->nr_busy_cpus);
11942	if (nr_busy > 1) {
11943	flags = NOHZ_STATS_KICK | NOHZ_BALANCE_KICK;
11944	goto unlock;
11945	}
11946	}
11947	unlock:
11948	rcu_read_unlock();
11949	out:
11950	if (READ_ONCE(nohz.needs_update))
11951	flags |= NOHZ_NEXT_KICK;
11952
11953	if (flags)
11954	kick_ilb(flags);
11955	}
11956
11957	static void set_cpu_sd_state_busy(int cpu)
11958	{
11959	struct sched_domain *sd;
11960
11961	rcu_read_lock();
11962	sd = rcu_dereference(per_cpu(sd_llc, cpu));
11963
11964	if (!sd || !sd->nohz_idle)
11965	goto unlock;
11966	sd->nohz_idle = 0;
11967
11968	atomic_inc(v: &sd->shared->nr_busy_cpus);
11969	unlock:
11970	rcu_read_unlock();
11971	}
11972
11973	void nohz_balance_exit_idle(struct rq *rq)
11974	{
11975	SCHED_WARN_ON(rq != this_rq());
11976
11977	if (likely(!rq->nohz_tick_stopped))
11978	return;
11979
11980	rq->nohz_tick_stopped = 0;
11981	cpumask_clear_cpu(cpu: rq->cpu, dstp: nohz.idle_cpus_mask);
11982	atomic_dec(v: &nohz.nr_cpus);
11983
11984	set_cpu_sd_state_busy(rq->cpu);
11985	}
11986
11987	static void set_cpu_sd_state_idle(int cpu)
11988	{
11989	struct sched_domain *sd;
11990
11991	rcu_read_lock();
11992	sd = rcu_dereference(per_cpu(sd_llc, cpu));
11993
11994	if (!sd || sd->nohz_idle)
11995	goto unlock;
11996	sd->nohz_idle = 1;
11997
11998	atomic_dec(v: &sd->shared->nr_busy_cpus);
11999	unlock:
12000	rcu_read_unlock();
12001	}
12002
12003	/*
12004	* This routine will record that the CPU is going idle with tick stopped.
12005	* This info will be used in performing idle load balancing in the future.
12006	*/
12007	void nohz_balance_enter_idle(int cpu)
12008	{
12009	struct rq *rq = cpu_rq(cpu);
12010
12011	SCHED_WARN_ON(cpu != smp_processor_id());
12012
12013	/* If this CPU is going down, then nothing needs to be done: */
12014	if (!cpu_active(cpu))
12015	return;
12016
12017	/* Spare idle load balancing on CPUs that don't want to be disturbed: */
12018	if (!housekeeping_cpu(cpu, type: HK_TYPE_SCHED))
12019	return;
12020
12021	/*
12022	* Can be set safely without rq->lock held
12023	* If a clear happens, it will have evaluated last additions because
12024	* rq->lock is held during the check and the clear
12025	*/
12026	rq->has_blocked_load = 1;
12027
12028	/*
12029	* The tick is still stopped but load could have been added in the
12030	* meantime. We set the nohz.has_blocked flag to trig a check of the
12031	* *_avg. The CPU is already part of nohz.idle_cpus_mask so the clear
12032	* of nohz.has_blocked can only happen after checking the new load
12033	*/
12034	if (rq->nohz_tick_stopped)
12035	goto out;
12036
12037	/* If we're a completely isolated CPU, we don't play: */
12038	if (on_null_domain(rq))
12039	return;
12040
12041	rq->nohz_tick_stopped = 1;
12042
12043	cpumask_set_cpu(cpu, dstp: nohz.idle_cpus_mask);
12044	atomic_inc(v: &nohz.nr_cpus);
12045
12046	/*
12047	* Ensures that if nohz_idle_balance() fails to observe our
12048	* @idle_cpus_mask store, it must observe the @has_blocked
12049	* and @needs_update stores.
12050	*/
12051	smp_mb__after_atomic();
12052
12053	set_cpu_sd_state_idle(cpu);
12054
12055	WRITE_ONCE(nohz.needs_update, 1);
12056	out:
12057	/*
12058	* Each time a cpu enter idle, we assume that it has blocked load and
12059	* enable the periodic update of the load of idle cpus
12060	*/
12061	WRITE_ONCE(nohz.has_blocked, 1);
12062	}
12063
12064	static bool update_nohz_stats(struct rq *rq)
12065	{
12066	unsigned int cpu = rq->cpu;
12067
12068	if (!rq->has_blocked_load)
12069	return false;
12070
12071	if (!cpumask_test_cpu(cpu, cpumask: nohz.idle_cpus_mask))
12072	return false;
12073
12074	if (!time_after(jiffies, READ_ONCE(rq->last_blocked_load_update_tick)))
12075	return true;
12076
12077	update_blocked_averages(cpu);
12078
12079	return rq->has_blocked_load;
12080	}
12081
12082	/*
12083	* Internal function that runs load balance for all idle cpus. The load balance
12084	* can be a simple update of blocked load or a complete load balance with
12085	* tasks movement depending of flags.
12086	*/
12087	static void _nohz_idle_balance(struct rq *this_rq, unsigned int flags)
12088	{
12089	/* Earliest time when we have to do rebalance again */
12090	unsigned long now = jiffies;
12091	unsigned long next_balance = now + 60*HZ;
12092	bool has_blocked_load = false;
12093	int update_next_balance = 0;
12094	int this_cpu = this_rq->cpu;
12095	int balance_cpu;
12096	struct rq *rq;
12097
12098	SCHED_WARN_ON((flags & NOHZ_KICK_MASK) == NOHZ_BALANCE_KICK);
12099
12100	/*
12101	* We assume there will be no idle load after this update and clear
12102	* the has_blocked flag. If a cpu enters idle in the mean time, it will
12103	* set the has_blocked flag and trigger another update of idle load.
12104	* Because a cpu that becomes idle, is added to idle_cpus_mask before
12105	* setting the flag, we are sure to not clear the state and not
12106	* check the load of an idle cpu.
12107	*
12108	* Same applies to idle_cpus_mask vs needs_update.
12109	*/
12110	if (flags & NOHZ_STATS_KICK)
12111	WRITE_ONCE(nohz.has_blocked, 0);
12112	if (flags & NOHZ_NEXT_KICK)
12113	WRITE_ONCE(nohz.needs_update, 0);
12114
12115	/*
12116	* Ensures that if we miss the CPU, we must see the has_blocked
12117	* store from nohz_balance_enter_idle().
12118	*/
12119	smp_mb();
12120
12121	/*
12122	* Start with the next CPU after this_cpu so we will end with this_cpu and let a
12123	* chance for other idle cpu to pull load.
12124	*/
12125	for_each_cpu_wrap(balance_cpu, nohz.idle_cpus_mask, this_cpu+1) {
12126	if (!idle_cpu(cpu: balance_cpu))
12127	continue;
12128
12129	/*
12130	* If this CPU gets work to do, stop the load balancing
12131	* work being done for other CPUs. Next load
12132	* balancing owner will pick it up.
12133	*/
12134	if (need_resched()) {
12135	if (flags & NOHZ_STATS_KICK)
12136	has_blocked_load = true;
12137	if (flags & NOHZ_NEXT_KICK)
12138	WRITE_ONCE(nohz.needs_update, 1);
12139	goto abort;
12140	}
12141
12142	rq = cpu_rq(balance_cpu);
12143
12144	if (flags & NOHZ_STATS_KICK)
12145	has_blocked_load |= update_nohz_stats(rq);
12146
12147	/*
12148	* If time for next balance is due,
12149	* do the balance.
12150	*/
12151	if (time_after_eq(jiffies, rq->next_balance)) {
12152	struct rq_flags rf;
12153
12154	rq_lock_irqsave(rq, rf: &rf);
12155	update_rq_clock(rq);
12156	rq_unlock_irqrestore(rq, rf: &rf);
12157
12158	if (flags & NOHZ_BALANCE_KICK)
12159	rebalance_domains(rq, idle: CPU_IDLE);
12160	}
12161
12162	if (time_after(next_balance, rq->next_balance)) {
12163	next_balance = rq->next_balance;
12164	update_next_balance = 1;
12165	}
12166	}
12167
12168	/*
12169	* next_balance will be updated only when there is a need.
12170	* When the CPU is attached to null domain for ex, it will not be
12171	* updated.
12172	*/
12173	if (likely(update_next_balance))
12174	nohz.next_balance = next_balance;
12175
12176	if (flags & NOHZ_STATS_KICK)
12177	WRITE_ONCE(nohz.next_blocked,
12178	now + msecs_to_jiffies(LOAD_AVG_PERIOD));
12179
12180	abort:
12181	/* There is still blocked load, enable periodic update */
12182	if (has_blocked_load)
12183	WRITE_ONCE(nohz.has_blocked, 1);
12184	}
12185
12186	/*
12187	* In CONFIG_NO_HZ_COMMON case, the idle balance kickee will do the
12188	* rebalancing for all the cpus for whom scheduler ticks are stopped.
12189	*/
12190	static bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
12191	{
12192	unsigned int flags = this_rq->nohz_idle_balance;
12193
12194	if (!flags)
12195	return false;
12196
12197	this_rq->nohz_idle_balance = 0;
12198
12199	if (idle != CPU_IDLE)
12200	return false;
12201
12202	_nohz_idle_balance(this_rq, flags);
12203
12204	return true;
12205	}
12206
12207	/*
12208	* Check if we need to directly run the ILB for updating blocked load before
12209	* entering idle state. Here we run ILB directly without issuing IPIs.
12210	*
12211	* Note that when this function is called, the tick may not yet be stopped on
12212	* this CPU yet. nohz.idle_cpus_mask is updated only when tick is stopped and
12213	* cleared on the next busy tick. In other words, nohz.idle_cpus_mask updates
12214	* don't align with CPUs enter/exit idle to avoid bottlenecks due to high idle
12215	* entry/exit rate (usec). So it is possible that _nohz_idle_balance() is
12216	* called from this function on (this) CPU that's not yet in the mask. That's
12217	* OK because the goal of nohz_run_idle_balance() is to run ILB only for
12218	* updating the blocked load of already idle CPUs without waking up one of
12219	* those idle CPUs and outside the preempt disable / irq off phase of the local
12220	* cpu about to enter idle, because it can take a long time.
12221	*/
12222	void nohz_run_idle_balance(int cpu)
12223	{
12224	unsigned int flags;
12225
12226	flags = atomic_fetch_andnot(NOHZ_NEWILB_KICK, nohz_flags(cpu));
12227
12228	/*
12229	* Update the blocked load only if no SCHED_SOFTIRQ is about to happen
12230	* (ie NOHZ_STATS_KICK set) and will do the same.
12231	*/
12232	if ((flags == NOHZ_NEWILB_KICK) && !need_resched())
12233	_nohz_idle_balance(cpu_rq(cpu), NOHZ_STATS_KICK);
12234	}
12235
12236	static void nohz_newidle_balance(struct rq *this_rq)
12237	{
12238	int this_cpu = this_rq->cpu;
12239
12240	/*
12241	* This CPU doesn't want to be disturbed by scheduler
12242	* housekeeping
12243	*/
12244	if (!housekeeping_cpu(cpu: this_cpu, type: HK_TYPE_SCHED))
12245	return;
12246
12247	/* Will wake up very soon. No time for doing anything else*/
12248	if (this_rq->avg_idle < sysctl_sched_migration_cost)
12249	return;
12250
12251	/* Don't need to update blocked load of idle CPUs*/
12252	if (!READ_ONCE(nohz.has_blocked) ||
12253	time_before(jiffies, READ_ONCE(nohz.next_blocked)))
12254	return;
12255
12256	/*
12257	* Set the need to trigger ILB in order to update blocked load
12258	* before entering idle state.
12259	*/
12260	atomic_or(NOHZ_NEWILB_KICK, nohz_flags(this_cpu));
12261	}
12262
12263	#else /* !CONFIG_NO_HZ_COMMON */
12264	static inline void nohz_balancer_kick(struct rq *rq) { }
12265
12266	static inline bool nohz_idle_balance(struct rq *this_rq, enum cpu_idle_type idle)
12267	{
12268	return false;
12269	}
12270
12271	static inline void nohz_newidle_balance(struct rq *this_rq) { }
12272	#endif /* CONFIG_NO_HZ_COMMON */
12273
12274	/*
12275	* newidle_balance is called by schedule() if this_cpu is about to become
12276	* idle. Attempts to pull tasks from other CPUs.
12277	*
12278	* Returns:
12279	* < 0 - we released the lock and there are !fair tasks present
12280	* 0 - failed, no new tasks
12281	* > 0 - success, new (fair) tasks present
12282	*/
12283	static int newidle_balance(struct rq *this_rq, struct rq_flags *rf)
12284	{
12285	unsigned long next_balance = jiffies + HZ;
12286	int this_cpu = this_rq->cpu;
12287	u64 t0, t1, curr_cost = 0;
12288	struct sched_domain *sd;
12289	int pulled_task = 0;
12290
12291	update_misfit_status(NULL, rq: this_rq);
12292
12293	/*
12294	* There is a task waiting to run. No need to search for one.
12295	* Return 0; the task will be enqueued when switching to idle.
12296	*/
12297	if (this_rq->ttwu_pending)
12298	return 0;
12299
12300	/*
12301	* We must set idle_stamp _before_ calling idle_balance(), such that we
12302	* measure the duration of idle_balance() as idle time.
12303	*/
12304	this_rq->idle_stamp = rq_clock(rq: this_rq);
12305
12306	/*
12307	* Do not pull tasks towards !active CPUs...
12308	*/
12309	if (!cpu_active(cpu: this_cpu))
12310	return 0;
12311
12312	/*
12313	* This is OK, because current is on_cpu, which avoids it being picked
12314	* for load-balance and preemption/IRQs are still disabled avoiding
12315	* further scheduler activity on it and we're being very careful to
12316	* re-start the picking loop.
12317	*/
12318	rq_unpin_lock(rq: this_rq, rf);
12319
12320	rcu_read_lock();
12321	sd = rcu_dereference_check_sched_domain(this_rq->sd);
12322
12323	if (!READ_ONCE(this_rq->rd->overload) ||
12324	(sd && this_rq->avg_idle < sd->max_newidle_lb_cost)) {
12325
12326	if (sd)
12327	update_next_balance(sd, next_balance: &next_balance);
12328	rcu_read_unlock();
12329
12330	goto out;
12331	}
12332	rcu_read_unlock();
12333
12334	raw_spin_rq_unlock(rq: this_rq);
12335
12336	t0 = sched_clock_cpu(cpu: this_cpu);
12337	update_blocked_averages(cpu: this_cpu);
12338
12339	rcu_read_lock();
12340	for_each_domain(this_cpu, sd) {
12341	int continue_balancing = 1;
12342	u64 domain_cost;
12343
12344	update_next_balance(sd, next_balance: &next_balance);
12345
12346	if (this_rq->avg_idle < curr_cost + sd->max_newidle_lb_cost)
12347	break;
12348
12349	if (sd->flags & SD_BALANCE_NEWIDLE) {
12350
12351	pulled_task = load_balance(this_cpu, this_rq,
12352	sd, idle: CPU_NEWLY_IDLE,
12353	continue_balancing: &continue_balancing);
12354
12355	t1 = sched_clock_cpu(cpu: this_cpu);
12356	domain_cost = t1 - t0;
12357	update_newidle_cost(sd, cost: domain_cost);
12358
12359	curr_cost += domain_cost;
12360	t0 = t1;
12361	}
12362
12363	/*
12364	* Stop searching for tasks to pull if there are
12365	* now runnable tasks on this rq.
12366	*/
12367	if (pulled_task || this_rq->nr_running > 0 ||
12368	this_rq->ttwu_pending)
12369	break;
12370	}
12371	rcu_read_unlock();
12372
12373	raw_spin_rq_lock(rq: this_rq);
12374
12375	if (curr_cost > this_rq->max_idle_balance_cost)
12376	this_rq->max_idle_balance_cost = curr_cost;
12377
12378	/*
12379	* While browsing the domains, we released the rq lock, a task could
12380	* have been enqueued in the meantime. Since we're not going idle,
12381	* pretend we pulled a task.
12382	*/
12383	if (this_rq->cfs.h_nr_running && !pulled_task)
12384	pulled_task = 1;
12385
12386	/* Is there a task of a high priority class? */
12387	if (this_rq->nr_running != this_rq->cfs.h_nr_running)
12388	pulled_task = -1;
12389
12390	out:
12391	/* Move the next balance forward */
12392	if (time_after(this_rq->next_balance, next_balance))
12393	this_rq->next_balance = next_balance;
12394
12395	if (pulled_task)
12396	this_rq->idle_stamp = 0;
12397	else
12398	nohz_newidle_balance(this_rq);
12399
12400	rq_repin_lock(rq: this_rq, rf);
12401
12402	return pulled_task;
12403	}
12404
12405	/*
12406	* run_rebalance_domains is triggered when needed from the scheduler tick.
12407	* Also triggered for nohz idle balancing (with nohz_balancing_kick set).
12408	*/
12409	static __latent_entropy void run_rebalance_domains(struct softirq_action *h)
12410	{
12411	struct rq *this_rq = this_rq();
12412	enum cpu_idle_type idle = this_rq->idle_balance ?
12413	CPU_IDLE : CPU_NOT_IDLE;
12414
12415	/*
12416	* If this CPU has a pending nohz_balance_kick, then do the
12417	* balancing on behalf of the other idle CPUs whose ticks are
12418	* stopped. Do nohz_idle_balance *before* rebalance_domains to
12419	* give the idle CPUs a chance to load balance. Else we may
12420	* load balance only within the local sched_domain hierarchy
12421	* and abort nohz_idle_balance altogether if we pull some load.
12422	*/
12423	if (nohz_idle_balance(this_rq, idle))
12424	return;
12425
12426	/* normal load balance */
12427	update_blocked_averages(cpu: this_rq->cpu);
12428	rebalance_domains(rq: this_rq, idle);
12429	}
12430
12431	/*
12432	* Trigger the SCHED_SOFTIRQ if it is time to do periodic load balancing.
12433	*/
12434	void trigger_load_balance(struct rq *rq)
12435	{
12436	/*
12437	* Don't need to rebalance while attached to NULL domain or
12438	* runqueue CPU is not active
12439	*/
12440	if (unlikely(on_null_domain(rq) || !cpu_active(cpu_of(rq))))
12441	return;
12442
12443	if (time_after_eq(jiffies, rq->next_balance))
12444	raise_softirq(nr: SCHED_SOFTIRQ);
12445
12446	nohz_balancer_kick(rq);
12447	}
12448
12449	static void rq_online_fair(struct rq *rq)
12450	{
12451	update_sysctl();
12452
12453	update_runtime_enabled(rq);
12454	}
12455
12456	static void rq_offline_fair(struct rq *rq)
12457	{
12458	update_sysctl();
12459
12460	/* Ensure any throttled groups are reachable by pick_next_task */
12461	unthrottle_offline_cfs_rqs(rq);
12462
12463	/* Ensure that we remove rq contribution to group share: */
12464	clear_tg_offline_cfs_rqs(rq);
12465	}
12466
12467	#endif /* CONFIG_SMP */
12468
12469	#ifdef CONFIG_SCHED_CORE
12470	static inline bool
12471	__entity_slice_used(struct sched_entity *se, int min_nr_tasks)
12472	{
12473	u64 rtime = se->sum_exec_runtime - se->prev_sum_exec_runtime;
12474	u64 slice = se->slice;
12475
12476	return (rtime * min_nr_tasks > slice);
12477	}
12478
12479	#define MIN_NR_TASKS_DURING_FORCEIDLE 2
12480	static inline void task_tick_core(struct rq *rq, struct task_struct *curr)
12481	{
12482	if (!sched_core_enabled(rq))
12483	return;
12484
12485	/*
12486	* If runqueue has only one task which used up its slice and
12487	* if the sibling is forced idle, then trigger schedule to
12488	* give forced idle task a chance.
12489	*
12490	* sched_slice() considers only this active rq and it gets the
12491	* whole slice. But during force idle, we have siblings acting
12492	* like a single runqueue and hence we need to consider runnable
12493	* tasks on this CPU and the forced idle CPU. Ideally, we should
12494	* go through the forced idle rq, but that would be a perf hit.
12495	* We can assume that the forced idle CPU has at least
12496	* MIN_NR_TASKS_DURING_FORCEIDLE - 1 tasks and use that to check
12497	* if we need to give up the CPU.
12498	*/
12499	if (rq->core->core_forceidle_count && rq->cfs.nr_running == 1 &&
12500	__entity_slice_used(se: &curr->se, MIN_NR_TASKS_DURING_FORCEIDLE))
12501	resched_curr(rq);
12502	}
12503
12504	/*
12505	* se_fi_update - Update the cfs_rq->min_vruntime_fi in a CFS hierarchy if needed.
12506	*/
12507	static void se_fi_update(const struct sched_entity *se, unsigned int fi_seq,
12508	bool forceidle)
12509	{
12510	for_each_sched_entity(se) {
12511	struct cfs_rq *cfs_rq = cfs_rq_of(se);
12512
12513	if (forceidle) {
12514	if (cfs_rq->forceidle_seq == fi_seq)
12515	break;
12516	cfs_rq->forceidle_seq = fi_seq;
12517	}
12518
12519	cfs_rq->min_vruntime_fi = cfs_rq->min_vruntime;
12520	}
12521	}
12522
12523	void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_fi)
12524	{
12525	struct sched_entity *se = &p->se;
12526
12527	if (p->sched_class != &fair_sched_class)
12528	return;
12529
12530	se_fi_update(se, fi_seq: rq->core->core_forceidle_seq, forceidle: in_fi);
12531	}
12532
12533	bool cfs_prio_less(const struct task_struct *a, const struct task_struct *b,
12534	bool in_fi)
12535	{
12536	struct rq *rq = task_rq(a);
12537	const struct sched_entity *sea = &a->se;
12538	const struct sched_entity *seb = &b->se;
12539	struct cfs_rq *cfs_rqa;
12540	struct cfs_rq *cfs_rqb;
12541	s64 delta;
12542
12543	SCHED_WARN_ON(task_rq(b)->core != rq->core);
12544
12545	#ifdef CONFIG_FAIR_GROUP_SCHED
12546	/*
12547	* Find an se in the hierarchy for tasks a and b, such that the se's
12548	* are immediate siblings.
12549	*/
12550	while (sea->cfs_rq->tg != seb->cfs_rq->tg) {
12551	int sea_depth = sea->depth;
12552	int seb_depth = seb->depth;
12553
12554	if (sea_depth >= seb_depth)
12555	sea = parent_entity(se: sea);
12556	if (sea_depth <= seb_depth)
12557	seb = parent_entity(se: seb);
12558	}
12559
12560	se_fi_update(se: sea, fi_seq: rq->core->core_forceidle_seq, forceidle: in_fi);
12561	se_fi_update(se: seb, fi_seq: rq->core->core_forceidle_seq, forceidle: in_fi);
12562
12563	cfs_rqa = sea->cfs_rq;
12564	cfs_rqb = seb->cfs_rq;
12565	#else
12566	cfs_rqa = &task_rq(a)->cfs;
12567	cfs_rqb = &task_rq(b)->cfs;
12568	#endif
12569
12570	/*
12571	* Find delta after normalizing se's vruntime with its cfs_rq's
12572	* min_vruntime_fi, which would have been updated in prior calls
12573	* to se_fi_update().
12574	*/
12575	delta = (s64)(sea->vruntime - seb->vruntime) +
12576	(s64)(cfs_rqb->min_vruntime_fi - cfs_rqa->min_vruntime_fi);
12577
12578	return delta > 0;
12579	}
12580
12581	static int task_is_throttled_fair(struct task_struct *p, int cpu)
12582	{
12583	struct cfs_rq *cfs_rq;
12584
12585	#ifdef CONFIG_FAIR_GROUP_SCHED
12586	cfs_rq = task_group(p)->cfs_rq[cpu];
12587	#else
12588	cfs_rq = &cpu_rq(cpu)->cfs;
12589	#endif
12590	return throttled_hierarchy(cfs_rq);
12591	}
12592	#else
12593	static inline void task_tick_core(struct rq *rq, struct task_struct *curr) {}
12594	#endif
12595
12596	/*
12597	* scheduler tick hitting a task of our scheduling class.
12598	*
12599	* NOTE: This function can be called remotely by the tick offload that
12600	* goes along full dynticks. Therefore no local assumption can be made
12601	* and everything must be accessed through the @rq and @curr passed in
12602	* parameters.
12603	*/
12604	static void task_tick_fair(struct rq *rq, struct task_struct *curr, int queued)
12605	{
12606	struct cfs_rq *cfs_rq;
12607	struct sched_entity *se = &curr->se;
12608
12609	for_each_sched_entity(se) {
12610	cfs_rq = cfs_rq_of(se);
12611	entity_tick(cfs_rq, curr: se, queued);
12612	}
12613
12614	if (static_branch_unlikely(&sched_numa_balancing))
12615	task_tick_numa(rq, curr);
12616
12617	update_misfit_status(p: curr, rq);
12618	update_overutilized_status(task_rq(curr));
12619
12620	task_tick_core(rq, curr);
12621	}
12622
12623	/*
12624	* called on fork with the child task as argument from the parent's context
12625	* - child not yet on the tasklist
12626	* - preemption disabled
12627	*/
12628	static void task_fork_fair(struct task_struct *p)
12629	{
12630	struct sched_entity *se = &p->se, *curr;
12631	struct cfs_rq *cfs_rq;
12632	struct rq *rq = this_rq();
12633	struct rq_flags rf;
12634
12635	rq_lock(rq, rf: &rf);
12636	update_rq_clock(rq);
12637
12638	cfs_rq = task_cfs_rq(current);
12639	curr = cfs_rq->curr;
12640	if (curr)
12641	update_curr(cfs_rq);
12642	place_entity(cfs_rq, se, ENQUEUE_INITIAL);
12643	rq_unlock(rq, rf: &rf);
12644	}
12645
12646	/*
12647	* Priority of the task has changed. Check to see if we preempt
12648	* the current task.
12649	*/
12650	static void
12651	prio_changed_fair(struct rq *rq, struct task_struct *p, int oldprio)
12652	{
12653	if (!task_on_rq_queued(p))
12654	return;
12655
12656	if (rq->cfs.nr_running == 1)
12657	return;
12658
12659	/*
12660	* Reschedule if we are currently running on this runqueue and
12661	* our priority decreased, or if we are not currently running on
12662	* this runqueue and our priority is higher than the current's
12663	*/
12664	if (task_current(rq, p)) {
12665	if (p->prio > oldprio)
12666	resched_curr(rq);
12667	} else
12668	wakeup_preempt(rq, p, flags: 0);
12669	}
12670
12671	#ifdef CONFIG_FAIR_GROUP_SCHED
12672	/*
12673	* Propagate the changes of the sched_entity across the tg tree to make it
12674	* visible to the root
12675	*/
12676	static void propagate_entity_cfs_rq(struct sched_entity *se)
12677	{
12678	struct cfs_rq *cfs_rq = cfs_rq_of(se);
12679
12680	if (cfs_rq_throttled(cfs_rq))
12681	return;
12682
12683	if (!throttled_hierarchy(cfs_rq))
12684	list_add_leaf_cfs_rq(cfs_rq);
12685
12686	/* Start to propagate at parent */
12687	se = se->parent;
12688
12689	for_each_sched_entity(se) {
12690	cfs_rq = cfs_rq_of(se);
12691
12692	update_load_avg(cfs_rq, se, UPDATE_TG);
12693
12694	if (cfs_rq_throttled(cfs_rq))
12695	break;
12696
12697	if (!throttled_hierarchy(cfs_rq))
12698	list_add_leaf_cfs_rq(cfs_rq);
12699	}
12700	}
12701	#else
12702	static void propagate_entity_cfs_rq(struct sched_entity *se) { }
12703	#endif
12704
12705	static void detach_entity_cfs_rq(struct sched_entity *se)
12706	{
12707	struct cfs_rq *cfs_rq = cfs_rq_of(se);
12708
12709	#ifdef CONFIG_SMP
12710	/*
12711	* In case the task sched_avg hasn't been attached:
12712	* - A forked task which hasn't been woken up by wake_up_new_task().
12713	* - A task which has been woken up by try_to_wake_up() but is
12714	* waiting for actually being woken up by sched_ttwu_pending().
12715	*/
12716	if (!se->avg.last_update_time)
12717	return;
12718	#endif
12719
12720	/* Catch up with the cfs_rq and remove our load when we leave */
12721	update_load_avg(cfs_rq, se, flags: 0);
12722	detach_entity_load_avg(cfs_rq, se);
12723	update_tg_load_avg(cfs_rq);
12724	propagate_entity_cfs_rq(se);
12725	}
12726
12727	static void attach_entity_cfs_rq(struct sched_entity *se)
12728	{
12729	struct cfs_rq *cfs_rq = cfs_rq_of(se);
12730
12731	/* Synchronize entity with its cfs_rq */
12732	update_load_avg(cfs_rq, se, sched_feat(ATTACH_AGE_LOAD) ? 0 : SKIP_AGE_LOAD);
12733	attach_entity_load_avg(cfs_rq, se);
12734	update_tg_load_avg(cfs_rq);
12735	propagate_entity_cfs_rq(se);
12736	}
12737
12738	static void detach_task_cfs_rq(struct task_struct *p)
12739	{
12740	struct sched_entity *se = &p->se;
12741
12742	detach_entity_cfs_rq(se);
12743	}
12744
12745	static void attach_task_cfs_rq(struct task_struct *p)
12746	{
12747	struct sched_entity *se = &p->se;
12748
12749	attach_entity_cfs_rq(se);
12750	}
12751
12752	static void switched_from_fair(struct rq *rq, struct task_struct *p)
12753	{
12754	detach_task_cfs_rq(p);
12755	}
12756
12757	static void switched_to_fair(struct rq *rq, struct task_struct *p)
12758	{
12759	attach_task_cfs_rq(p);
12760
12761	if (task_on_rq_queued(p)) {
12762	/*
12763	* We were most likely switched from sched_rt, so
12764	* kick off the schedule if running, otherwise just see
12765	* if we can still preempt the current task.
12766	*/
12767	if (task_current(rq, p))
12768	resched_curr(rq);
12769	else
12770	wakeup_preempt(rq, p, flags: 0);
12771	}
12772	}
12773
12774	/* Account for a task changing its policy or group.
12775	*
12776	* This routine is mostly called to set cfs_rq->curr field when a task
12777	* migrates between groups/classes.
12778	*/
12779	static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first)
12780	{
12781	struct sched_entity *se = &p->se;
12782
12783	#ifdef CONFIG_SMP
12784	if (task_on_rq_queued(p)) {
12785	/*
12786	* Move the next running task to the front of the list, so our
12787	* cfs_tasks list becomes MRU one.
12788	*/
12789	list_move(list: &se->group_node, head: &rq->cfs_tasks);
12790	}
12791	#endif
12792
12793	for_each_sched_entity(se) {
12794	struct cfs_rq *cfs_rq = cfs_rq_of(se);
12795
12796	set_next_entity(cfs_rq, se);
12797	/* ensure bandwidth has been allocated on our new cfs_rq */
12798	account_cfs_rq_runtime(cfs_rq, delta_exec: 0);
12799	}
12800	}
12801
12802	void init_cfs_rq(struct cfs_rq *cfs_rq)
12803	{
12804	cfs_rq->tasks_timeline = RB_ROOT_CACHED;
12805	u64_u32_store(cfs_rq->min_vruntime, (u64)(-(1LL << 20)));
12806	#ifdef CONFIG_SMP
12807	raw_spin_lock_init(&cfs_rq->removed.lock);
12808	#endif
12809	}
12810
12811	#ifdef CONFIG_FAIR_GROUP_SCHED
12812	static void task_change_group_fair(struct task_struct *p)
12813	{
12814	/*
12815	* We couldn't detach or attach a forked task which
12816	* hasn't been woken up by wake_up_new_task().
12817	*/
12818	if (READ_ONCE(p->__state) == TASK_NEW)
12819	return;
12820
12821	detach_task_cfs_rq(p);
12822
12823	#ifdef CONFIG_SMP
12824	/* Tell se's cfs_rq has been changed -- migrated */
12825	p->se.avg.last_update_time = 0;
12826	#endif
12827	set_task_rq(p, cpu: task_cpu(p));
12828	attach_task_cfs_rq(p);
12829	}
12830
12831	void free_fair_sched_group(struct task_group *tg)
12832	{
12833	int i;
12834
12835	for_each_possible_cpu(i) {
12836	if (tg->cfs_rq)
12837	kfree(objp: tg->cfs_rq[i]);
12838	if (tg->se)
12839	kfree(objp: tg->se[i]);
12840	}
12841
12842	kfree(objp: tg->cfs_rq);
12843	kfree(objp: tg->se);
12844	}
12845
12846	int alloc_fair_sched_group(struct task_group *tg, struct task_group *parent)
12847	{
12848	struct sched_entity *se;
12849	struct cfs_rq *cfs_rq;
12850	int i;
12851
12852	tg->cfs_rq = kcalloc(n: nr_cpu_ids, size: sizeof(cfs_rq), GFP_KERNEL);
12853	if (!tg->cfs_rq)
12854	goto err;
12855	tg->se = kcalloc(n: nr_cpu_ids, size: sizeof(se), GFP_KERNEL);
12856	if (!tg->se)
12857	goto err;
12858
12859	tg->shares = NICE_0_LOAD;
12860
12861	init_cfs_bandwidth(cfs_b: tg_cfs_bandwidth(tg), parent: tg_cfs_bandwidth(tg: parent));
12862
12863	for_each_possible_cpu(i) {
12864	cfs_rq = kzalloc_node(size: sizeof(struct cfs_rq),
12865	GFP_KERNEL, cpu_to_node(cpu: i));
12866	if (!cfs_rq)
12867	goto err;
12868
12869	se = kzalloc_node(size: sizeof(struct sched_entity_stats),
12870	GFP_KERNEL, cpu_to_node(cpu: i));
12871	if (!se)
12872	goto err_free_rq;
12873
12874	init_cfs_rq(cfs_rq);
12875	init_tg_cfs_entry(tg, cfs_rq, se, cpu: i, parent: parent->se[i]);
12876	init_entity_runnable_average(se);
12877	}
12878
12879	return 1;
12880
12881	err_free_rq:
12882	kfree(objp: cfs_rq);
12883	err:
12884	return 0;
12885	}
12886
12887	void online_fair_sched_group(struct task_group *tg)
12888	{
12889	struct sched_entity *se;
12890	struct rq_flags rf;
12891	struct rq *rq;
12892	int i;
12893
12894	for_each_possible_cpu(i) {
12895	rq = cpu_rq(i);
12896	se = tg->se[i];
12897	rq_lock_irq(rq, rf: &rf);
12898	update_rq_clock(rq);
12899	attach_entity_cfs_rq(se);
12900	sync_throttle(tg, cpu: i);
12901	rq_unlock_irq(rq, rf: &rf);
12902	}
12903	}
12904
12905	void unregister_fair_sched_group(struct task_group *tg)
12906	{
12907	unsigned long flags;
12908	struct rq *rq;
12909	int cpu;
12910
12911	destroy_cfs_bandwidth(cfs_b: tg_cfs_bandwidth(tg));
12912
12913	for_each_possible_cpu(cpu) {
12914	if (tg->se[cpu])
12915	remove_entity_load_avg(se: tg->se[cpu]);
12916
12917	/*
12918	* Only empty task groups can be destroyed; so we can speculatively
12919	* check on_list without danger of it being re-added.
12920	*/
12921	if (!tg->cfs_rq[cpu]->on_list)
12922	continue;
12923
12924	rq = cpu_rq(cpu);
12925
12926	raw_spin_rq_lock_irqsave(rq, flags);
12927	list_del_leaf_cfs_rq(cfs_rq: tg->cfs_rq[cpu]);
12928	raw_spin_rq_unlock_irqrestore(rq, flags);
12929	}
12930	}
12931
12932	void init_tg_cfs_entry(struct task_group *tg, struct cfs_rq *cfs_rq,
12933	struct sched_entity *se, int cpu,
12934	struct sched_entity *parent)
12935	{
12936	struct rq *rq = cpu_rq(cpu);
12937
12938	cfs_rq->tg = tg;
12939	cfs_rq->rq = rq;
12940	init_cfs_rq_runtime(cfs_rq);
12941
12942	tg->cfs_rq[cpu] = cfs_rq;
12943	tg->se[cpu] = se;
12944
12945	/* se could be NULL for root_task_group */
12946	if (!se)
12947	return;
12948
12949	if (!parent) {
12950	se->cfs_rq = &rq->cfs;
12951	se->depth = 0;
12952	} else {
12953	se->cfs_rq = parent->my_q;
12954	se->depth = parent->depth + 1;
12955	}
12956
12957	se->my_q = cfs_rq;
12958	/* guarantee group entities always have weight */
12959	update_load_set(lw: &se->load, NICE_0_LOAD);
12960	se->parent = parent;
12961	}
12962
12963	static DEFINE_MUTEX(shares_mutex);
12964
12965	static int __sched_group_set_shares(struct task_group *tg, unsigned long shares)
12966	{
12967	int i;
12968
12969	lockdep_assert_held(&shares_mutex);
12970
12971	/*
12972	* We can't change the weight of the root cgroup.
12973	*/
12974	if (!tg->se[0])
12975	return -EINVAL;
12976
12977	shares = clamp(shares, scale_load(MIN_SHARES), scale_load(MAX_SHARES));
12978
12979	if (tg->shares == shares)
12980	return 0;
12981
12982	tg->shares = shares;
12983	for_each_possible_cpu(i) {
12984	struct rq *rq = cpu_rq(i);
12985	struct sched_entity *se = tg->se[i];
12986	struct rq_flags rf;
12987
12988	/* Propagate contribution to hierarchy */
12989	rq_lock_irqsave(rq, rf: &rf);
12990	update_rq_clock(rq);
12991	for_each_sched_entity(se) {
12992	update_load_avg(cfs_rq: cfs_rq_of(se), se, UPDATE_TG);
12993	update_cfs_group(se);
12994	}
12995	rq_unlock_irqrestore(rq, rf: &rf);
12996	}
12997
12998	return 0;
12999	}
13000
13001	int sched_group_set_shares(struct task_group *tg, unsigned long shares)
13002	{
13003	int ret;
13004
13005	mutex_lock(&shares_mutex);
13006	if (tg_is_idle(tg))
13007	ret = -EINVAL;
13008	else
13009	ret = __sched_group_set_shares(tg, shares);
13010	mutex_unlock(lock: &shares_mutex);
13011
13012	return ret;
13013	}
13014
13015	int sched_group_set_idle(struct task_group *tg, long idle)
13016	{
13017	int i;
13018
13019	if (tg == &root_task_group)
13020	return -EINVAL;
13021
13022	if (idle < 0 || idle > 1)
13023	return -EINVAL;
13024
13025	mutex_lock(&shares_mutex);
13026
13027	if (tg->idle == idle) {
13028	mutex_unlock(lock: &shares_mutex);
13029	return 0;
13030	}
13031
13032	tg->idle = idle;
13033
13034	for_each_possible_cpu(i) {
13035	struct rq *rq = cpu_rq(i);
13036	struct sched_entity *se = tg->se[i];
13037	struct cfs_rq *parent_cfs_rq, *grp_cfs_rq = tg->cfs_rq[i];
13038	bool was_idle = cfs_rq_is_idle(cfs_rq: grp_cfs_rq);
13039	long idle_task_delta;
13040	struct rq_flags rf;
13041
13042	rq_lock_irqsave(rq, rf: &rf);
13043
13044	grp_cfs_rq->idle = idle;
13045	if (WARN_ON_ONCE(was_idle == cfs_rq_is_idle(grp_cfs_rq)))
13046	goto next_cpu;
13047
13048	if (se->on_rq) {
13049	parent_cfs_rq = cfs_rq_of(se);
13050	if (cfs_rq_is_idle(cfs_rq: grp_cfs_rq))
13051	parent_cfs_rq->idle_nr_running++;
13052	else
13053	parent_cfs_rq->idle_nr_running--;
13054	}
13055
13056	idle_task_delta = grp_cfs_rq->h_nr_running -
13057	grp_cfs_rq->idle_h_nr_running;
13058	if (!cfs_rq_is_idle(cfs_rq: grp_cfs_rq))
13059	idle_task_delta *= -1;
13060
13061	for_each_sched_entity(se) {
13062	struct cfs_rq *cfs_rq = cfs_rq_of(se);
13063
13064	if (!se->on_rq)
13065	break;
13066
13067	cfs_rq->idle_h_nr_running += idle_task_delta;
13068
13069	/* Already accounted at parent level and above. */
13070	if (cfs_rq_is_idle(cfs_rq))
13071	break;
13072	}
13073
13074	next_cpu:
13075	rq_unlock_irqrestore(rq, rf: &rf);
13076	}
13077
13078	/* Idle groups have minimum weight. */
13079	if (tg_is_idle(tg))
13080	__sched_group_set_shares(tg, scale_load(WEIGHT_IDLEPRIO));
13081	else
13082	__sched_group_set_shares(tg, NICE_0_LOAD);
13083
13084	mutex_unlock(lock: &shares_mutex);
13085	return 0;
13086	}
13087
13088	#endif /* CONFIG_FAIR_GROUP_SCHED */
13089
13090
13091	static unsigned int get_rr_interval_fair(struct rq *rq, struct task_struct *task)
13092	{
13093	struct sched_entity *se = &task->se;
13094	unsigned int rr_interval = 0;
13095
13096	/*
13097	* Time slice is 0 for SCHED_OTHER tasks that are on an otherwise
13098	* idle runqueue:
13099	*/
13100	if (rq->cfs.load.weight)
13101	rr_interval = NS_TO_JIFFIES(se->slice);
13102
13103	return rr_interval;
13104	}
13105
13106	/*
13107	* All the scheduling class methods:
13108	*/
13109	DEFINE_SCHED_CLASS(fair) = {
13110
13111	.enqueue_task = enqueue_task_fair,
13112	.dequeue_task = dequeue_task_fair,
13113	.yield_task = yield_task_fair,
13114	.yield_to_task = yield_to_task_fair,
13115
13116	.wakeup_preempt = check_preempt_wakeup_fair,
13117
13118	.pick_next_task = __pick_next_task_fair,
13119	.put_prev_task = put_prev_task_fair,
13120	.set_next_task = set_next_task_fair,
13121
13122	#ifdef CONFIG_SMP
13123	.balance = balance_fair,
13124	.pick_task = pick_task_fair,
13125	.select_task_rq = select_task_rq_fair,
13126	.migrate_task_rq = migrate_task_rq_fair,
13127
13128	.rq_online = rq_online_fair,
13129	.rq_offline = rq_offline_fair,
13130
13131	.task_dead = task_dead_fair,
13132	.set_cpus_allowed = set_cpus_allowed_common,
13133	#endif
13134
13135	.task_tick = task_tick_fair,
13136	.task_fork = task_fork_fair,
13137
13138	.prio_changed = prio_changed_fair,
13139	.switched_from = switched_from_fair,
13140	.switched_to = switched_to_fair,
13141
13142	.get_rr_interval = get_rr_interval_fair,
13143
13144	.update_curr = update_curr_fair,
13145
13146	#ifdef CONFIG_FAIR_GROUP_SCHED
13147	.task_change_group = task_change_group_fair,
13148	#endif
13149
13150	#ifdef CONFIG_SCHED_CORE
13151	.task_is_throttled = task_is_throttled_fair,
13152	#endif
13153
13154	#ifdef CONFIG_UCLAMP_TASK
13155	.uclamp_enabled = 1,
13156	#endif
13157	};
13158
13159	#ifdef CONFIG_SCHED_DEBUG
13160	void print_cfs_stats(struct seq_file *m, int cpu)
13161	{
13162	struct cfs_rq *cfs_rq, *pos;
13163
13164	rcu_read_lock();
13165	for_each_leaf_cfs_rq_safe(cpu_rq(cpu), cfs_rq, pos)
13166	print_cfs_rq(m, cpu, cfs_rq);
13167	rcu_read_unlock();
13168	}
13169
13170	#ifdef CONFIG_NUMA_BALANCING
13171	void show_numa_stats(struct task_struct *p, struct seq_file *m)
13172	{
13173	int node;
13174	unsigned long tsf = 0, tpf = 0, gsf = 0, gpf = 0;
13175	struct numa_group *ng;
13176
13177	rcu_read_lock();
13178	ng = rcu_dereference(p->numa_group);
13179	for_each_online_node(node) {
13180	if (p->numa_faults) {
13181	tsf = p->numa_faults[task_faults_idx(s: NUMA_MEM, nid: node, priv: 0)];
13182	tpf = p->numa_faults[task_faults_idx(s: NUMA_MEM, nid: node, priv: 1)];
13183	}
13184	if (ng) {
13185	gsf = ng->faults[task_faults_idx(s: NUMA_MEM, nid: node, priv: 0)],
13186	gpf = ng->faults[task_faults_idx(s: NUMA_MEM, nid: node, priv: 1)];
13187	}
13188	print_numa_stats(m, node, tsf, tpf, gsf, gpf);
13189	}
13190	rcu_read_unlock();
13191	}
13192	#endif /* CONFIG_NUMA_BALANCING */
13193	#endif /* CONFIG_SCHED_DEBUG */
13194
13195	__init void init_sched_fair_class(void)
13196	{
13197	#ifdef CONFIG_SMP
13198	int i;
13199
13200	for_each_possible_cpu(i) {
13201	zalloc_cpumask_var_node(mask: &per_cpu(load_balance_mask, i), GFP_KERNEL, cpu_to_node(cpu: i));
13202	zalloc_cpumask_var_node(mask: &per_cpu(select_rq_mask, i), GFP_KERNEL, cpu_to_node(cpu: i));
13203	zalloc_cpumask_var_node(mask: &per_cpu(should_we_balance_tmpmask, i),
13204	GFP_KERNEL, cpu_to_node(cpu: i));
13205
13206	#ifdef CONFIG_CFS_BANDWIDTH
13207	INIT_CSD(&cpu_rq(i)->cfsb_csd, __cfsb_csd_unthrottle, cpu_rq(i));
13208	INIT_LIST_HEAD(list: &cpu_rq(i)->cfsb_csd_list);
13209	#endif
13210	}
13211
13212	open_softirq(nr: SCHED_SOFTIRQ, action: run_rebalance_domains);
13213
13214	#ifdef CONFIG_NO_HZ_COMMON
13215	nohz.next_balance = jiffies;
13216	nohz.next_blocked = jiffies;
13217	zalloc_cpumask_var(mask: &nohz.idle_cpus_mask, GFP_NOWAIT);
13218	#endif
13219	#endif /* SMP */
13220
13221	}
13222