1 | /* SPDX-License-Identifier: GPL-2.0 */ |
---|---|
2 | #ifndef _LINUX_SCHED_H |
3 | #define _LINUX_SCHED_H |
4 | |
5 | /* |
6 | * Define 'struct task_struct' and provide the main scheduler |
7 | * APIs (schedule(), wakeup variants, etc.) |
8 | */ |
9 | |
10 | #include <uapi/linux/sched.h> |
11 | |
12 | #include <asm/current.h> |
13 | #include <asm/processor.h> |
14 | #include <linux/thread_info.h> |
15 | #include <linux/preempt.h> |
16 | #include <linux/cpumask_types.h> |
17 | |
18 | #include <linux/cache.h> |
19 | #include <linux/irqflags_types.h> |
20 | #include <linux/smp_types.h> |
21 | #include <linux/pid_types.h> |
22 | #include <linux/sem_types.h> |
23 | #include <linux/shm.h> |
24 | #include <linux/kmsan_types.h> |
25 | #include <linux/mutex_types.h> |
26 | #include <linux/plist_types.h> |
27 | #include <linux/hrtimer_types.h> |
28 | #include <linux/timer_types.h> |
29 | #include <linux/seccomp_types.h> |
30 | #include <linux/nodemask_types.h> |
31 | #include <linux/refcount_types.h> |
32 | #include <linux/resource.h> |
33 | #include <linux/latencytop.h> |
34 | #include <linux/sched/prio.h> |
35 | #include <linux/sched/types.h> |
36 | #include <linux/signal_types.h> |
37 | #include <linux/syscall_user_dispatch_types.h> |
38 | #include <linux/mm_types_task.h> |
39 | #include <linux/netdevice_xmit.h> |
40 | #include <linux/task_io_accounting.h> |
41 | #include <linux/posix-timers_types.h> |
42 | #include <linux/restart_block.h> |
43 | #include <uapi/linux/rseq.h> |
44 | #include <linux/seqlock_types.h> |
45 | #include <linux/kcsan.h> |
46 | #include <linux/rv.h> |
47 | #include <linux/uidgid_types.h> |
48 | #include <linux/tracepoint-defs.h> |
49 | #include <asm/kmap_size.h> |
50 | |
51 | /* task_struct member predeclarations (sorted alphabetically): */ |
52 | struct audit_context; |
53 | struct bio_list; |
54 | struct blk_plug; |
55 | struct bpf_local_storage; |
56 | struct bpf_run_ctx; |
57 | struct bpf_net_context; |
58 | struct capture_control; |
59 | struct cfs_rq; |
60 | struct fs_struct; |
61 | struct futex_pi_state; |
62 | struct io_context; |
63 | struct io_uring_task; |
64 | struct mempolicy; |
65 | struct nameidata; |
66 | struct nsproxy; |
67 | struct perf_event_context; |
68 | struct perf_ctx_data; |
69 | struct pid_namespace; |
70 | struct pipe_inode_info; |
71 | struct rcu_node; |
72 | struct reclaim_state; |
73 | struct robust_list_head; |
74 | struct root_domain; |
75 | struct rq; |
76 | struct sched_attr; |
77 | struct sched_dl_entity; |
78 | struct seq_file; |
79 | struct sighand_struct; |
80 | struct signal_struct; |
81 | struct task_delay_info; |
82 | struct task_group; |
83 | struct task_struct; |
84 | struct user_event_mm; |
85 | |
86 | #include <linux/sched/ext.h> |
87 | |
88 | /* |
89 | * Task state bitmask. NOTE! These bits are also |
90 | * encoded in fs/proc/array.c: get_task_state(). |
91 | * |
92 | * We have two separate sets of flags: task->__state |
93 | * is about runnability, while task->exit_state are |
94 | * about the task exiting. Confusing, but this way |
95 | * modifying one set can't modify the other one by |
96 | * mistake. |
97 | */ |
98 | |
99 | /* Used in tsk->__state: */ |
100 | #define TASK_RUNNING 0x00000000 |
101 | #define TASK_INTERRUPTIBLE 0x00000001 |
102 | #define TASK_UNINTERRUPTIBLE 0x00000002 |
103 | #define __TASK_STOPPED 0x00000004 |
104 | #define __TASK_TRACED 0x00000008 |
105 | /* Used in tsk->exit_state: */ |
106 | #define EXIT_DEAD 0x00000010 |
107 | #define EXIT_ZOMBIE 0x00000020 |
108 | #define EXIT_TRACE (EXIT_ZOMBIE | EXIT_DEAD) |
109 | /* Used in tsk->__state again: */ |
110 | #define TASK_PARKED 0x00000040 |
111 | #define TASK_DEAD 0x00000080 |
112 | #define TASK_WAKEKILL 0x00000100 |
113 | #define TASK_WAKING 0x00000200 |
114 | #define TASK_NOLOAD 0x00000400 |
115 | #define TASK_NEW 0x00000800 |
116 | #define TASK_RTLOCK_WAIT 0x00001000 |
117 | #define TASK_FREEZABLE 0x00002000 |
118 | #define __TASK_FREEZABLE_UNSAFE (0x00004000 * IS_ENABLED(CONFIG_LOCKDEP)) |
119 | #define TASK_FROZEN 0x00008000 |
120 | #define TASK_STATE_MAX 0x00010000 |
121 | |
122 | #define TASK_ANY (TASK_STATE_MAX-1) |
123 | |
124 | /* |
125 | * DO NOT ADD ANY NEW USERS ! |
126 | */ |
127 | #define TASK_FREEZABLE_UNSAFE (TASK_FREEZABLE | __TASK_FREEZABLE_UNSAFE) |
128 | |
129 | /* Convenience macros for the sake of set_current_state: */ |
130 | #define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE) |
131 | #define TASK_STOPPED (TASK_WAKEKILL | __TASK_STOPPED) |
132 | #define TASK_TRACED __TASK_TRACED |
133 | |
134 | #define TASK_IDLE (TASK_UNINTERRUPTIBLE | TASK_NOLOAD) |
135 | |
136 | /* Convenience macros for the sake of wake_up(): */ |
137 | #define TASK_NORMAL (TASK_INTERRUPTIBLE | TASK_UNINTERRUPTIBLE) |
138 | |
139 | /* get_task_state(): */ |
140 | #define TASK_REPORT (TASK_RUNNING | TASK_INTERRUPTIBLE | \ |
141 | TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ |
142 | __TASK_TRACED | EXIT_DEAD | EXIT_ZOMBIE | \ |
143 | TASK_PARKED) |
144 | |
145 | #define task_is_running(task) (READ_ONCE((task)->__state) == TASK_RUNNING) |
146 | |
147 | #define task_is_traced(task) ((READ_ONCE(task->jobctl) & JOBCTL_TRACED) != 0) |
148 | #define task_is_stopped(task) ((READ_ONCE(task->jobctl) & JOBCTL_STOPPED) != 0) |
149 | #define task_is_stopped_or_traced(task) ((READ_ONCE(task->jobctl) & (JOBCTL_STOPPED | JOBCTL_TRACED)) != 0) |
150 | |
151 | /* |
152 | * Special states are those that do not use the normal wait-loop pattern. See |
153 | * the comment with set_special_state(). |
154 | */ |
155 | #define is_special_task_state(state) \ |
156 | ((state) & (__TASK_STOPPED | __TASK_TRACED | TASK_PARKED | \ |
157 | TASK_DEAD | TASK_FROZEN)) |
158 | |
159 | #ifdef CONFIG_DEBUG_ATOMIC_SLEEP |
160 | # define debug_normal_state_change(state_value) \ |
161 | do { \ |
162 | WARN_ON_ONCE(is_special_task_state(state_value)); \ |
163 | current->task_state_change = _THIS_IP_; \ |
164 | } while (0) |
165 | |
166 | # define debug_special_state_change(state_value) \ |
167 | do { \ |
168 | WARN_ON_ONCE(!is_special_task_state(state_value)); \ |
169 | current->task_state_change = _THIS_IP_; \ |
170 | } while (0) |
171 | |
172 | # define debug_rtlock_wait_set_state() \ |
173 | do { \ |
174 | current->saved_state_change = current->task_state_change;\ |
175 | current->task_state_change = _THIS_IP_; \ |
176 | } while (0) |
177 | |
178 | # define debug_rtlock_wait_restore_state() \ |
179 | do { \ |
180 | current->task_state_change = current->saved_state_change;\ |
181 | } while (0) |
182 | |
183 | #else |
184 | # define debug_normal_state_change(cond) do { } while (0) |
185 | # define debug_special_state_change(cond) do { } while (0) |
186 | # define debug_rtlock_wait_set_state() do { } while (0) |
187 | # define debug_rtlock_wait_restore_state() do { } while (0) |
188 | #endif |
189 | |
190 | #define trace_set_current_state(state_value) \ |
191 | do { \ |
192 | if (tracepoint_enabled(sched_set_state_tp)) \ |
193 | __trace_set_current_state(state_value); \ |
194 | } while (0) |
195 | |
196 | /* |
197 | * set_current_state() includes a barrier so that the write of current->__state |
198 | * is correctly serialised wrt the caller's subsequent test of whether to |
199 | * actually sleep: |
200 | * |
201 | * for (;;) { |
202 | * set_current_state(TASK_UNINTERRUPTIBLE); |
203 | * if (CONDITION) |
204 | * break; |
205 | * |
206 | * schedule(); |
207 | * } |
208 | * __set_current_state(TASK_RUNNING); |
209 | * |
210 | * If the caller does not need such serialisation (because, for instance, the |
211 | * CONDITION test and condition change and wakeup are under the same lock) then |
212 | * use __set_current_state(). |
213 | * |
214 | * The above is typically ordered against the wakeup, which does: |
215 | * |
216 | * CONDITION = 1; |
217 | * wake_up_state(p, TASK_UNINTERRUPTIBLE); |
218 | * |
219 | * where wake_up_state()/try_to_wake_up() executes a full memory barrier before |
220 | * accessing p->__state. |
221 | * |
222 | * Wakeup will do: if (@state & p->__state) p->__state = TASK_RUNNING, that is, |
223 | * once it observes the TASK_UNINTERRUPTIBLE store the waking CPU can issue a |
224 | * TASK_RUNNING store which can collide with __set_current_state(TASK_RUNNING). |
225 | * |
226 | * However, with slightly different timing the wakeup TASK_RUNNING store can |
227 | * also collide with the TASK_UNINTERRUPTIBLE store. Losing that store is not |
228 | * a problem either because that will result in one extra go around the loop |
229 | * and our @cond test will save the day. |
230 | * |
231 | * Also see the comments of try_to_wake_up(). |
232 | */ |
233 | #define __set_current_state(state_value) \ |
234 | do { \ |
235 | debug_normal_state_change((state_value)); \ |
236 | trace_set_current_state(state_value); \ |
237 | WRITE_ONCE(current->__state, (state_value)); \ |
238 | } while (0) |
239 | |
240 | #define set_current_state(state_value) \ |
241 | do { \ |
242 | debug_normal_state_change((state_value)); \ |
243 | trace_set_current_state(state_value); \ |
244 | smp_store_mb(current->__state, (state_value)); \ |
245 | } while (0) |
246 | |
247 | /* |
248 | * set_special_state() should be used for those states when the blocking task |
249 | * can not use the regular condition based wait-loop. In that case we must |
250 | * serialize against wakeups such that any possible in-flight TASK_RUNNING |
251 | * stores will not collide with our state change. |
252 | */ |
253 | #define set_special_state(state_value) \ |
254 | do { \ |
255 | unsigned long flags; /* may shadow */ \ |
256 | \ |
257 | raw_spin_lock_irqsave(¤t->pi_lock, flags); \ |
258 | debug_special_state_change((state_value)); \ |
259 | trace_set_current_state(state_value); \ |
260 | WRITE_ONCE(current->__state, (state_value)); \ |
261 | raw_spin_unlock_irqrestore(¤t->pi_lock, flags); \ |
262 | } while (0) |
263 | |
264 | /* |
265 | * PREEMPT_RT specific variants for "sleeping" spin/rwlocks |
266 | * |
267 | * RT's spin/rwlock substitutions are state preserving. The state of the |
268 | * task when blocking on the lock is saved in task_struct::saved_state and |
269 | * restored after the lock has been acquired. These operations are |
270 | * serialized by task_struct::pi_lock against try_to_wake_up(). Any non RT |
271 | * lock related wakeups while the task is blocked on the lock are |
272 | * redirected to operate on task_struct::saved_state to ensure that these |
273 | * are not dropped. On restore task_struct::saved_state is set to |
274 | * TASK_RUNNING so any wakeup attempt redirected to saved_state will fail. |
275 | * |
276 | * The lock operation looks like this: |
277 | * |
278 | * current_save_and_set_rtlock_wait_state(); |
279 | * for (;;) { |
280 | * if (try_lock()) |
281 | * break; |
282 | * raw_spin_unlock_irq(&lock->wait_lock); |
283 | * schedule_rtlock(); |
284 | * raw_spin_lock_irq(&lock->wait_lock); |
285 | * set_current_state(TASK_RTLOCK_WAIT); |
286 | * } |
287 | * current_restore_rtlock_saved_state(); |
288 | */ |
289 | #define current_save_and_set_rtlock_wait_state() \ |
290 | do { \ |
291 | lockdep_assert_irqs_disabled(); \ |
292 | raw_spin_lock(¤t->pi_lock); \ |
293 | current->saved_state = current->__state; \ |
294 | debug_rtlock_wait_set_state(); \ |
295 | trace_set_current_state(TASK_RTLOCK_WAIT); \ |
296 | WRITE_ONCE(current->__state, TASK_RTLOCK_WAIT); \ |
297 | raw_spin_unlock(¤t->pi_lock); \ |
298 | } while (0); |
299 | |
300 | #define current_restore_rtlock_saved_state() \ |
301 | do { \ |
302 | lockdep_assert_irqs_disabled(); \ |
303 | raw_spin_lock(¤t->pi_lock); \ |
304 | debug_rtlock_wait_restore_state(); \ |
305 | trace_set_current_state(current->saved_state); \ |
306 | WRITE_ONCE(current->__state, current->saved_state); \ |
307 | current->saved_state = TASK_RUNNING; \ |
308 | raw_spin_unlock(¤t->pi_lock); \ |
309 | } while (0); |
310 | |
311 | #define get_current_state() READ_ONCE(current->__state) |
312 | |
313 | /* |
314 | * Define the task command name length as enum, then it can be visible to |
315 | * BPF programs. |
316 | */ |
317 | enum { |
318 | TASK_COMM_LEN = 16, |
319 | }; |
320 | |
321 | extern void sched_tick(void); |
322 | |
323 | #define MAX_SCHEDULE_TIMEOUT LONG_MAX |
324 | |
325 | extern long schedule_timeout(long timeout); |
326 | extern long schedule_timeout_interruptible(long timeout); |
327 | extern long schedule_timeout_killable(long timeout); |
328 | extern long schedule_timeout_uninterruptible(long timeout); |
329 | extern long schedule_timeout_idle(long timeout); |
330 | asmlinkage void schedule(void); |
331 | extern void schedule_preempt_disabled(void); |
332 | asmlinkage void preempt_schedule_irq(void); |
333 | #ifdef CONFIG_PREEMPT_RT |
334 | extern void schedule_rtlock(void); |
335 | #endif |
336 | |
337 | extern int __must_check io_schedule_prepare(void); |
338 | extern void io_schedule_finish(int token); |
339 | extern long io_schedule_timeout(long timeout); |
340 | extern void io_schedule(void); |
341 | |
342 | /* wrapper function to trace from this header file */ |
343 | DECLARE_TRACEPOINT(sched_set_state_tp); |
344 | extern void __trace_set_current_state(int state_value); |
345 | |
346 | /** |
347 | * struct prev_cputime - snapshot of system and user cputime |
348 | * @utime: time spent in user mode |
349 | * @stime: time spent in system mode |
350 | * @lock: protects the above two fields |
351 | * |
352 | * Stores previous user/system time values such that we can guarantee |
353 | * monotonicity. |
354 | */ |
355 | struct prev_cputime { |
356 | #ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE |
357 | u64 utime; |
358 | u64 stime; |
359 | raw_spinlock_t lock; |
360 | #endif |
361 | }; |
362 | |
363 | enum vtime_state { |
364 | /* Task is sleeping or running in a CPU with VTIME inactive: */ |
365 | VTIME_INACTIVE = 0, |
366 | /* Task is idle */ |
367 | VTIME_IDLE, |
368 | /* Task runs in kernelspace in a CPU with VTIME active: */ |
369 | VTIME_SYS, |
370 | /* Task runs in userspace in a CPU with VTIME active: */ |
371 | VTIME_USER, |
372 | /* Task runs as guests in a CPU with VTIME active: */ |
373 | VTIME_GUEST, |
374 | }; |
375 | |
376 | struct vtime { |
377 | seqcount_t seqcount; |
378 | unsigned long long starttime; |
379 | enum vtime_state state; |
380 | unsigned int cpu; |
381 | u64 utime; |
382 | u64 stime; |
383 | u64 gtime; |
384 | }; |
385 | |
386 | /* |
387 | * Utilization clamp constraints. |
388 | * @UCLAMP_MIN: Minimum utilization |
389 | * @UCLAMP_MAX: Maximum utilization |
390 | * @UCLAMP_CNT: Utilization clamp constraints count |
391 | */ |
392 | enum uclamp_id { |
393 | UCLAMP_MIN = 0, |
394 | UCLAMP_MAX, |
395 | UCLAMP_CNT |
396 | }; |
397 | |
398 | #ifdef CONFIG_SMP |
399 | extern struct root_domain def_root_domain; |
400 | extern struct mutex sched_domains_mutex; |
401 | extern void sched_domains_mutex_lock(void); |
402 | extern void sched_domains_mutex_unlock(void); |
403 | #else |
404 | static inline void sched_domains_mutex_lock(void) { } |
405 | static inline void sched_domains_mutex_unlock(void) { } |
406 | #endif |
407 | |
408 | struct sched_param { |
409 | int sched_priority; |
410 | }; |
411 | |
412 | struct sched_info { |
413 | #ifdef CONFIG_SCHED_INFO |
414 | /* Cumulative counters: */ |
415 | |
416 | /* # of times we have run on this CPU: */ |
417 | unsigned long pcount; |
418 | |
419 | /* Time spent waiting on a runqueue: */ |
420 | unsigned long long run_delay; |
421 | |
422 | /* Max time spent waiting on a runqueue: */ |
423 | unsigned long long max_run_delay; |
424 | |
425 | /* Min time spent waiting on a runqueue: */ |
426 | unsigned long long min_run_delay; |
427 | |
428 | /* Timestamps: */ |
429 | |
430 | /* When did we last run on a CPU? */ |
431 | unsigned long long last_arrival; |
432 | |
433 | /* When were we last queued to run? */ |
434 | unsigned long long last_queued; |
435 | |
436 | #endif /* CONFIG_SCHED_INFO */ |
437 | }; |
438 | |
439 | /* |
440 | * Integer metrics need fixed point arithmetic, e.g., sched/fair |
441 | * has a few: load, load_avg, util_avg, freq, and capacity. |
442 | * |
443 | * We define a basic fixed point arithmetic range, and then formalize |
444 | * all these metrics based on that basic range. |
445 | */ |
446 | # define SCHED_FIXEDPOINT_SHIFT 10 |
447 | # define SCHED_FIXEDPOINT_SCALE (1L << SCHED_FIXEDPOINT_SHIFT) |
448 | |
449 | /* Increase resolution of cpu_capacity calculations */ |
450 | # define SCHED_CAPACITY_SHIFT SCHED_FIXEDPOINT_SHIFT |
451 | # define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) |
452 | |
453 | struct load_weight { |
454 | unsigned long weight; |
455 | u32 inv_weight; |
456 | }; |
457 | |
458 | /* |
459 | * The load/runnable/util_avg accumulates an infinite geometric series |
460 | * (see __update_load_avg_cfs_rq() in kernel/sched/pelt.c). |
461 | * |
462 | * [load_avg definition] |
463 | * |
464 | * load_avg = runnable% * scale_load_down(load) |
465 | * |
466 | * [runnable_avg definition] |
467 | * |
468 | * runnable_avg = runnable% * SCHED_CAPACITY_SCALE |
469 | * |
470 | * [util_avg definition] |
471 | * |
472 | * util_avg = running% * SCHED_CAPACITY_SCALE |
473 | * |
474 | * where runnable% is the time ratio that a sched_entity is runnable and |
475 | * running% the time ratio that a sched_entity is running. |
476 | * |
477 | * For cfs_rq, they are the aggregated values of all runnable and blocked |
478 | * sched_entities. |
479 | * |
480 | * The load/runnable/util_avg doesn't directly factor frequency scaling and CPU |
481 | * capacity scaling. The scaling is done through the rq_clock_pelt that is used |
482 | * for computing those signals (see update_rq_clock_pelt()) |
483 | * |
484 | * N.B., the above ratios (runnable% and running%) themselves are in the |
485 | * range of [0, 1]. To do fixed point arithmetics, we therefore scale them |
486 | * to as large a range as necessary. This is for example reflected by |
487 | * util_avg's SCHED_CAPACITY_SCALE. |
488 | * |
489 | * [Overflow issue] |
490 | * |
491 | * The 64-bit load_sum can have 4353082796 (=2^64/47742/88761) entities |
492 | * with the highest load (=88761), always runnable on a single cfs_rq, |
493 | * and should not overflow as the number already hits PID_MAX_LIMIT. |
494 | * |
495 | * For all other cases (including 32-bit kernels), struct load_weight's |
496 | * weight will overflow first before we do, because: |
497 | * |
498 | * Max(load_avg) <= Max(load.weight) |
499 | * |
500 | * Then it is the load_weight's responsibility to consider overflow |
501 | * issues. |
502 | */ |
503 | struct sched_avg { |
504 | u64 last_update_time; |
505 | u64 load_sum; |
506 | u64 runnable_sum; |
507 | u32 util_sum; |
508 | u32 period_contrib; |
509 | unsigned long load_avg; |
510 | unsigned long runnable_avg; |
511 | unsigned long util_avg; |
512 | unsigned int util_est; |
513 | } ____cacheline_aligned; |
514 | |
515 | /* |
516 | * The UTIL_AVG_UNCHANGED flag is used to synchronize util_est with util_avg |
517 | * updates. When a task is dequeued, its util_est should not be updated if its |
518 | * util_avg has not been updated in the meantime. |
519 | * This information is mapped into the MSB bit of util_est at dequeue time. |
520 | * Since max value of util_est for a task is 1024 (PELT util_avg for a task) |
521 | * it is safe to use MSB. |
522 | */ |
523 | #define UTIL_EST_WEIGHT_SHIFT 2 |
524 | #define UTIL_AVG_UNCHANGED 0x80000000 |
525 | |
526 | struct sched_statistics { |
527 | #ifdef CONFIG_SCHEDSTATS |
528 | u64 wait_start; |
529 | u64 wait_max; |
530 | u64 wait_count; |
531 | u64 wait_sum; |
532 | u64 iowait_count; |
533 | u64 iowait_sum; |
534 | |
535 | u64 sleep_start; |
536 | u64 sleep_max; |
537 | s64 sum_sleep_runtime; |
538 | |
539 | u64 block_start; |
540 | u64 block_max; |
541 | s64 sum_block_runtime; |
542 | |
543 | s64 exec_max; |
544 | u64 slice_max; |
545 | |
546 | u64 nr_migrations_cold; |
547 | u64 nr_failed_migrations_affine; |
548 | u64 nr_failed_migrations_running; |
549 | u64 nr_failed_migrations_hot; |
550 | u64 nr_forced_migrations; |
551 | #ifdef CONFIG_NUMA_BALANCING |
552 | u64 numa_task_migrated; |
553 | u64 numa_task_swapped; |
554 | #endif |
555 | |
556 | u64 nr_wakeups; |
557 | u64 nr_wakeups_sync; |
558 | u64 nr_wakeups_migrate; |
559 | u64 nr_wakeups_local; |
560 | u64 nr_wakeups_remote; |
561 | u64 nr_wakeups_affine; |
562 | u64 nr_wakeups_affine_attempts; |
563 | u64 nr_wakeups_passive; |
564 | u64 nr_wakeups_idle; |
565 | |
566 | #ifdef CONFIG_SCHED_CORE |
567 | u64 core_forceidle_sum; |
568 | #endif |
569 | #endif /* CONFIG_SCHEDSTATS */ |
570 | } ____cacheline_aligned; |
571 | |
572 | struct sched_entity { |
573 | /* For load-balancing: */ |
574 | struct load_weight load; |
575 | struct rb_node run_node; |
576 | u64 deadline; |
577 | u64 min_vruntime; |
578 | u64 min_slice; |
579 | |
580 | struct list_head group_node; |
581 | unsigned char on_rq; |
582 | unsigned char sched_delayed; |
583 | unsigned char rel_deadline; |
584 | unsigned char custom_slice; |
585 | /* hole */ |
586 | |
587 | u64 exec_start; |
588 | u64 sum_exec_runtime; |
589 | u64 prev_sum_exec_runtime; |
590 | u64 vruntime; |
591 | s64 vlag; |
592 | u64 slice; |
593 | |
594 | u64 nr_migrations; |
595 | |
596 | #ifdef CONFIG_FAIR_GROUP_SCHED |
597 | int depth; |
598 | struct sched_entity *parent; |
599 | /* rq on which this entity is (to be) queued: */ |
600 | struct cfs_rq *cfs_rq; |
601 | /* rq "owned" by this entity/group: */ |
602 | struct cfs_rq *my_q; |
603 | /* cached value of my_q->h_nr_running */ |
604 | unsigned long runnable_weight; |
605 | #endif |
606 | |
607 | #ifdef CONFIG_SMP |
608 | /* |
609 | * Per entity load average tracking. |
610 | * |
611 | * Put into separate cache line so it does not |
612 | * collide with read-mostly values above. |
613 | */ |
614 | struct sched_avg avg; |
615 | #endif |
616 | }; |
617 | |
618 | struct sched_rt_entity { |
619 | struct list_head run_list; |
620 | unsigned long timeout; |
621 | unsigned long watchdog_stamp; |
622 | unsigned int time_slice; |
623 | unsigned short on_rq; |
624 | unsigned short on_list; |
625 | |
626 | struct sched_rt_entity *back; |
627 | #ifdef CONFIG_RT_GROUP_SCHED |
628 | struct sched_rt_entity *parent; |
629 | /* rq on which this entity is (to be) queued: */ |
630 | struct rt_rq *rt_rq; |
631 | /* rq "owned" by this entity/group: */ |
632 | struct rt_rq *my_q; |
633 | #endif |
634 | } __randomize_layout; |
635 | |
636 | typedef bool (*dl_server_has_tasks_f)(struct sched_dl_entity *); |
637 | typedef struct task_struct *(*dl_server_pick_f)(struct sched_dl_entity *); |
638 | |
639 | struct sched_dl_entity { |
640 | struct rb_node rb_node; |
641 | |
642 | /* |
643 | * Original scheduling parameters. Copied here from sched_attr |
644 | * during sched_setattr(), they will remain the same until |
645 | * the next sched_setattr(). |
646 | */ |
647 | u64 dl_runtime; /* Maximum runtime for each instance */ |
648 | u64 dl_deadline; /* Relative deadline of each instance */ |
649 | u64 dl_period; /* Separation of two instances (period) */ |
650 | u64 dl_bw; /* dl_runtime / dl_period */ |
651 | u64 dl_density; /* dl_runtime / dl_deadline */ |
652 | |
653 | /* |
654 | * Actual scheduling parameters. Initialized with the values above, |
655 | * they are continuously updated during task execution. Note that |
656 | * the remaining runtime could be < 0 in case we are in overrun. |
657 | */ |
658 | s64 runtime; /* Remaining runtime for this instance */ |
659 | u64 deadline; /* Absolute deadline for this instance */ |
660 | unsigned int flags; /* Specifying the scheduler behaviour */ |
661 | |
662 | /* |
663 | * Some bool flags: |
664 | * |
665 | * @dl_throttled tells if we exhausted the runtime. If so, the |
666 | * task has to wait for a replenishment to be performed at the |
667 | * next firing of dl_timer. |
668 | * |
669 | * @dl_yielded tells if task gave up the CPU before consuming |
670 | * all its available runtime during the last job. |
671 | * |
672 | * @dl_non_contending tells if the task is inactive while still |
673 | * contributing to the active utilization. In other words, it |
674 | * indicates if the inactive timer has been armed and its handler |
675 | * has not been executed yet. This flag is useful to avoid race |
676 | * conditions between the inactive timer handler and the wakeup |
677 | * code. |
678 | * |
679 | * @dl_overrun tells if the task asked to be informed about runtime |
680 | * overruns. |
681 | * |
682 | * @dl_server tells if this is a server entity. |
683 | * |
684 | * @dl_defer tells if this is a deferred or regular server. For |
685 | * now only defer server exists. |
686 | * |
687 | * @dl_defer_armed tells if the deferrable server is waiting |
688 | * for the replenishment timer to activate it. |
689 | * |
690 | * @dl_server_active tells if the dlserver is active(started). |
691 | * dlserver is started on first cfs enqueue on an idle runqueue |
692 | * and is stopped when a dequeue results in 0 cfs tasks on the |
693 | * runqueue. In other words, dlserver is active only when cpu's |
694 | * runqueue has atleast one cfs task. |
695 | * |
696 | * @dl_defer_running tells if the deferrable server is actually |
697 | * running, skipping the defer phase. |
698 | */ |
699 | unsigned int dl_throttled : 1; |
700 | unsigned int dl_yielded : 1; |
701 | unsigned int dl_non_contending : 1; |
702 | unsigned int dl_overrun : 1; |
703 | unsigned int dl_server : 1; |
704 | unsigned int dl_server_active : 1; |
705 | unsigned int dl_defer : 1; |
706 | unsigned int dl_defer_armed : 1; |
707 | unsigned int dl_defer_running : 1; |
708 | |
709 | /* |
710 | * Bandwidth enforcement timer. Each -deadline task has its |
711 | * own bandwidth to be enforced, thus we need one timer per task. |
712 | */ |
713 | struct hrtimer dl_timer; |
714 | |
715 | /* |
716 | * Inactive timer, responsible for decreasing the active utilization |
717 | * at the "0-lag time". When a -deadline task blocks, it contributes |
718 | * to GRUB's active utilization until the "0-lag time", hence a |
719 | * timer is needed to decrease the active utilization at the correct |
720 | * time. |
721 | */ |
722 | struct hrtimer inactive_timer; |
723 | |
724 | /* |
725 | * Bits for DL-server functionality. Also see the comment near |
726 | * dl_server_update(). |
727 | * |
728 | * @rq the runqueue this server is for |
729 | * |
730 | * @server_has_tasks() returns true if @server_pick return a |
731 | * runnable task. |
732 | */ |
733 | struct rq *rq; |
734 | dl_server_has_tasks_f server_has_tasks; |
735 | dl_server_pick_f server_pick_task; |
736 | |
737 | #ifdef CONFIG_RT_MUTEXES |
738 | /* |
739 | * Priority Inheritance. When a DEADLINE scheduling entity is boosted |
740 | * pi_se points to the donor, otherwise points to the dl_se it belongs |
741 | * to (the original one/itself). |
742 | */ |
743 | struct sched_dl_entity *pi_se; |
744 | #endif |
745 | }; |
746 | |
747 | #ifdef CONFIG_UCLAMP_TASK |
748 | /* Number of utilization clamp buckets (shorter alias) */ |
749 | #define UCLAMP_BUCKETS CONFIG_UCLAMP_BUCKETS_COUNT |
750 | |
751 | /* |
752 | * Utilization clamp for a scheduling entity |
753 | * @value: clamp value "assigned" to a se |
754 | * @bucket_id: bucket index corresponding to the "assigned" value |
755 | * @active: the se is currently refcounted in a rq's bucket |
756 | * @user_defined: the requested clamp value comes from user-space |
757 | * |
758 | * The bucket_id is the index of the clamp bucket matching the clamp value |
759 | * which is pre-computed and stored to avoid expensive integer divisions from |
760 | * the fast path. |
761 | * |
762 | * The active bit is set whenever a task has got an "effective" value assigned, |
763 | * which can be different from the clamp value "requested" from user-space. |
764 | * This allows to know a task is refcounted in the rq's bucket corresponding |
765 | * to the "effective" bucket_id. |
766 | * |
767 | * The user_defined bit is set whenever a task has got a task-specific clamp |
768 | * value requested from userspace, i.e. the system defaults apply to this task |
769 | * just as a restriction. This allows to relax default clamps when a less |
770 | * restrictive task-specific value has been requested, thus allowing to |
771 | * implement a "nice" semantic. For example, a task running with a 20% |
772 | * default boost can still drop its own boosting to 0%. |
773 | */ |
774 | struct uclamp_se { |
775 | unsigned int value : bits_per(SCHED_CAPACITY_SCALE); |
776 | unsigned int bucket_id : bits_per(UCLAMP_BUCKETS); |
777 | unsigned int active : 1; |
778 | unsigned int user_defined : 1; |
779 | }; |
780 | #endif /* CONFIG_UCLAMP_TASK */ |
781 | |
782 | union rcu_special { |
783 | struct { |
784 | u8 blocked; |
785 | u8 need_qs; |
786 | u8 exp_hint; /* Hint for performance. */ |
787 | u8 need_mb; /* Readers need smp_mb(). */ |
788 | } b; /* Bits. */ |
789 | u32 s; /* Set of bits. */ |
790 | }; |
791 | |
792 | enum perf_event_task_context { |
793 | perf_invalid_context = -1, |
794 | perf_hw_context = 0, |
795 | perf_sw_context, |
796 | perf_nr_task_contexts, |
797 | }; |
798 | |
799 | /* |
800 | * Number of contexts where an event can trigger: |
801 | * task, softirq, hardirq, nmi. |
802 | */ |
803 | #define PERF_NR_CONTEXTS 4 |
804 | |
805 | struct wake_q_node { |
806 | struct wake_q_node *next; |
807 | }; |
808 | |
809 | struct kmap_ctrl { |
810 | #ifdef CONFIG_KMAP_LOCAL |
811 | int idx; |
812 | pte_t pteval[KM_MAX_IDX]; |
813 | #endif |
814 | }; |
815 | |
816 | struct task_struct { |
817 | #ifdef CONFIG_THREAD_INFO_IN_TASK |
818 | /* |
819 | * For reasons of header soup (see current_thread_info()), this |
820 | * must be the first element of task_struct. |
821 | */ |
822 | struct thread_info thread_info; |
823 | #endif |
824 | unsigned int __state; |
825 | |
826 | /* saved state for "spinlock sleepers" */ |
827 | unsigned int saved_state; |
828 | |
829 | /* |
830 | * This begins the randomizable portion of task_struct. Only |
831 | * scheduling-critical items should be added above here. |
832 | */ |
833 | randomized_struct_fields_start |
834 | |
835 | void *stack; |
836 | refcount_t usage; |
837 | /* Per task flags (PF_*), defined further below: */ |
838 | unsigned int flags; |
839 | unsigned int ptrace; |
840 | |
841 | #ifdef CONFIG_MEM_ALLOC_PROFILING |
842 | struct alloc_tag *alloc_tag; |
843 | #endif |
844 | |
845 | #ifdef CONFIG_SMP |
846 | int on_cpu; |
847 | struct __call_single_node wake_entry; |
848 | unsigned int wakee_flips; |
849 | unsigned long wakee_flip_decay_ts; |
850 | struct task_struct *last_wakee; |
851 | |
852 | /* |
853 | * recent_used_cpu is initially set as the last CPU used by a task |
854 | * that wakes affine another task. Waker/wakee relationships can |
855 | * push tasks around a CPU where each wakeup moves to the next one. |
856 | * Tracking a recently used CPU allows a quick search for a recently |
857 | * used CPU that may be idle. |
858 | */ |
859 | int recent_used_cpu; |
860 | int wake_cpu; |
861 | #endif |
862 | int on_rq; |
863 | |
864 | int prio; |
865 | int static_prio; |
866 | int normal_prio; |
867 | unsigned int rt_priority; |
868 | |
869 | struct sched_entity se; |
870 | struct sched_rt_entity rt; |
871 | struct sched_dl_entity dl; |
872 | struct sched_dl_entity *dl_server; |
873 | #ifdef CONFIG_SCHED_CLASS_EXT |
874 | struct sched_ext_entity scx; |
875 | #endif |
876 | const struct sched_class *sched_class; |
877 | |
878 | #ifdef CONFIG_SCHED_CORE |
879 | struct rb_node core_node; |
880 | unsigned long core_cookie; |
881 | unsigned int core_occupation; |
882 | #endif |
883 | |
884 | #ifdef CONFIG_CGROUP_SCHED |
885 | struct task_group *sched_task_group; |
886 | #endif |
887 | |
888 | |
889 | #ifdef CONFIG_UCLAMP_TASK |
890 | /* |
891 | * Clamp values requested for a scheduling entity. |
892 | * Must be updated with task_rq_lock() held. |
893 | */ |
894 | struct uclamp_se uclamp_req[UCLAMP_CNT]; |
895 | /* |
896 | * Effective clamp values used for a scheduling entity. |
897 | * Must be updated with task_rq_lock() held. |
898 | */ |
899 | struct uclamp_se uclamp[UCLAMP_CNT]; |
900 | #endif |
901 | |
902 | struct sched_statistics stats; |
903 | |
904 | #ifdef CONFIG_PREEMPT_NOTIFIERS |
905 | /* List of struct preempt_notifier: */ |
906 | struct hlist_head preempt_notifiers; |
907 | #endif |
908 | |
909 | #ifdef CONFIG_BLK_DEV_IO_TRACE |
910 | unsigned int btrace_seq; |
911 | #endif |
912 | |
913 | unsigned int policy; |
914 | unsigned long max_allowed_capacity; |
915 | int nr_cpus_allowed; |
916 | const cpumask_t *cpus_ptr; |
917 | cpumask_t *user_cpus_ptr; |
918 | cpumask_t cpus_mask; |
919 | void *migration_pending; |
920 | #ifdef CONFIG_SMP |
921 | unsigned short migration_disabled; |
922 | #endif |
923 | unsigned short migration_flags; |
924 | |
925 | #ifdef CONFIG_PREEMPT_RCU |
926 | int rcu_read_lock_nesting; |
927 | union rcu_special rcu_read_unlock_special; |
928 | struct list_head rcu_node_entry; |
929 | struct rcu_node *rcu_blocked_node; |
930 | #endif /* #ifdef CONFIG_PREEMPT_RCU */ |
931 | |
932 | #ifdef CONFIG_TASKS_RCU |
933 | unsigned long rcu_tasks_nvcsw; |
934 | u8 rcu_tasks_holdout; |
935 | u8 rcu_tasks_idx; |
936 | int rcu_tasks_idle_cpu; |
937 | struct list_head rcu_tasks_holdout_list; |
938 | int rcu_tasks_exit_cpu; |
939 | struct list_head rcu_tasks_exit_list; |
940 | #endif /* #ifdef CONFIG_TASKS_RCU */ |
941 | |
942 | #ifdef CONFIG_TASKS_TRACE_RCU |
943 | int trc_reader_nesting; |
944 | int trc_ipi_to_cpu; |
945 | union rcu_special trc_reader_special; |
946 | struct list_head trc_holdout_list; |
947 | struct list_head trc_blkd_node; |
948 | int trc_blkd_cpu; |
949 | #endif /* #ifdef CONFIG_TASKS_TRACE_RCU */ |
950 | |
951 | struct sched_info sched_info; |
952 | |
953 | struct list_head tasks; |
954 | #ifdef CONFIG_SMP |
955 | struct plist_node pushable_tasks; |
956 | struct rb_node pushable_dl_tasks; |
957 | #endif |
958 | |
959 | struct mm_struct *mm; |
960 | struct mm_struct *active_mm; |
961 | struct address_space *faults_disabled_mapping; |
962 | |
963 | int exit_state; |
964 | int exit_code; |
965 | int exit_signal; |
966 | /* The signal sent when the parent dies: */ |
967 | int pdeath_signal; |
968 | /* JOBCTL_*, siglock protected: */ |
969 | unsigned long jobctl; |
970 | |
971 | /* Used for emulating ABI behavior of previous Linux versions: */ |
972 | unsigned int personality; |
973 | |
974 | /* Scheduler bits, serialized by scheduler locks: */ |
975 | unsigned sched_reset_on_fork:1; |
976 | unsigned sched_contributes_to_load:1; |
977 | unsigned sched_migrated:1; |
978 | unsigned sched_task_hot:1; |
979 | |
980 | /* Force alignment to the next boundary: */ |
981 | unsigned :0; |
982 | |
983 | /* Unserialized, strictly 'current' */ |
984 | |
985 | /* |
986 | * This field must not be in the scheduler word above due to wakelist |
987 | * queueing no longer being serialized by p->on_cpu. However: |
988 | * |
989 | * p->XXX = X; ttwu() |
990 | * schedule() if (p->on_rq && ..) // false |
991 | * smp_mb__after_spinlock(); if (smp_load_acquire(&p->on_cpu) && //true |
992 | * deactivate_task() ttwu_queue_wakelist()) |
993 | * p->on_rq = 0; p->sched_remote_wakeup = Y; |
994 | * |
995 | * guarantees all stores of 'current' are visible before |
996 | * ->sched_remote_wakeup gets used, so it can be in this word. |
997 | */ |
998 | unsigned sched_remote_wakeup:1; |
999 | #ifdef CONFIG_RT_MUTEXES |
1000 | unsigned sched_rt_mutex:1; |
1001 | #endif |
1002 | |
1003 | /* Bit to tell TOMOYO we're in execve(): */ |
1004 | unsigned in_execve:1; |
1005 | unsigned in_iowait:1; |
1006 | #ifndef TIF_RESTORE_SIGMASK |
1007 | unsigned restore_sigmask:1; |
1008 | #endif |
1009 | #ifdef CONFIG_MEMCG_V1 |
1010 | unsigned in_user_fault:1; |
1011 | #endif |
1012 | #ifdef CONFIG_LRU_GEN |
1013 | /* whether the LRU algorithm may apply to this access */ |
1014 | unsigned in_lru_fault:1; |
1015 | #endif |
1016 | #ifdef CONFIG_COMPAT_BRK |
1017 | unsigned brk_randomized:1; |
1018 | #endif |
1019 | #ifdef CONFIG_CGROUPS |
1020 | /* disallow userland-initiated cgroup migration */ |
1021 | unsigned no_cgroup_migration:1; |
1022 | /* task is frozen/stopped (used by the cgroup freezer) */ |
1023 | unsigned frozen:1; |
1024 | #endif |
1025 | #ifdef CONFIG_BLK_CGROUP |
1026 | unsigned use_memdelay:1; |
1027 | #endif |
1028 | #ifdef CONFIG_PSI |
1029 | /* Stalled due to lack of memory */ |
1030 | unsigned in_memstall:1; |
1031 | #endif |
1032 | #ifdef CONFIG_PAGE_OWNER |
1033 | /* Used by page_owner=on to detect recursion in page tracking. */ |
1034 | unsigned in_page_owner:1; |
1035 | #endif |
1036 | #ifdef CONFIG_EVENTFD |
1037 | /* Recursion prevention for eventfd_signal() */ |
1038 | unsigned in_eventfd:1; |
1039 | #endif |
1040 | #ifdef CONFIG_ARCH_HAS_CPU_PASID |
1041 | unsigned pasid_activated:1; |
1042 | #endif |
1043 | #ifdef CONFIG_X86_BUS_LOCK_DETECT |
1044 | unsigned reported_split_lock:1; |
1045 | #endif |
1046 | #ifdef CONFIG_TASK_DELAY_ACCT |
1047 | /* delay due to memory thrashing */ |
1048 | unsigned in_thrashing:1; |
1049 | #endif |
1050 | unsigned in_nf_duplicate:1; |
1051 | #ifdef CONFIG_PREEMPT_RT |
1052 | struct netdev_xmit net_xmit; |
1053 | #endif |
1054 | unsigned long atomic_flags; /* Flags requiring atomic access. */ |
1055 | |
1056 | struct restart_block restart_block; |
1057 | |
1058 | pid_t pid; |
1059 | pid_t tgid; |
1060 | |
1061 | #ifdef CONFIG_STACKPROTECTOR |
1062 | /* Canary value for the -fstack-protector GCC feature: */ |
1063 | unsigned long stack_canary; |
1064 | #endif |
1065 | /* |
1066 | * Pointers to the (original) parent process, youngest child, younger sibling, |
1067 | * older sibling, respectively. (p->father can be replaced with |
1068 | * p->real_parent->pid) |
1069 | */ |
1070 | |
1071 | /* Real parent process: */ |
1072 | struct task_struct __rcu *real_parent; |
1073 | |
1074 | /* Recipient of SIGCHLD, wait4() reports: */ |
1075 | struct task_struct __rcu *parent; |
1076 | |
1077 | /* |
1078 | * Children/sibling form the list of natural children: |
1079 | */ |
1080 | struct list_head children; |
1081 | struct list_head sibling; |
1082 | struct task_struct *group_leader; |
1083 | |
1084 | /* |
1085 | * 'ptraced' is the list of tasks this task is using ptrace() on. |
1086 | * |
1087 | * This includes both natural children and PTRACE_ATTACH targets. |
1088 | * 'ptrace_entry' is this task's link on the p->parent->ptraced list. |
1089 | */ |
1090 | struct list_head ptraced; |
1091 | struct list_head ptrace_entry; |
1092 | |
1093 | /* PID/PID hash table linkage. */ |
1094 | struct pid *thread_pid; |
1095 | struct hlist_node pid_links[PIDTYPE_MAX]; |
1096 | struct list_head thread_node; |
1097 | |
1098 | struct completion *vfork_done; |
1099 | |
1100 | /* CLONE_CHILD_SETTID: */ |
1101 | int __user *set_child_tid; |
1102 | |
1103 | /* CLONE_CHILD_CLEARTID: */ |
1104 | int __user *clear_child_tid; |
1105 | |
1106 | /* PF_KTHREAD | PF_IO_WORKER */ |
1107 | void *worker_private; |
1108 | |
1109 | u64 utime; |
1110 | u64 stime; |
1111 | #ifdef CONFIG_ARCH_HAS_SCALED_CPUTIME |
1112 | u64 utimescaled; |
1113 | u64 stimescaled; |
1114 | #endif |
1115 | u64 gtime; |
1116 | struct prev_cputime prev_cputime; |
1117 | #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN |
1118 | struct vtime vtime; |
1119 | #endif |
1120 | |
1121 | #ifdef CONFIG_NO_HZ_FULL |
1122 | atomic_t tick_dep_mask; |
1123 | #endif |
1124 | /* Context switch counts: */ |
1125 | unsigned long nvcsw; |
1126 | unsigned long nivcsw; |
1127 | |
1128 | /* Monotonic time in nsecs: */ |
1129 | u64 start_time; |
1130 | |
1131 | /* Boot based time in nsecs: */ |
1132 | u64 start_boottime; |
1133 | |
1134 | /* MM fault and swap info: this can arguably be seen as either mm-specific or thread-specific: */ |
1135 | unsigned long min_flt; |
1136 | unsigned long maj_flt; |
1137 | |
1138 | /* Empty if CONFIG_POSIX_CPUTIMERS=n */ |
1139 | struct posix_cputimers posix_cputimers; |
1140 | |
1141 | #ifdef CONFIG_POSIX_CPU_TIMERS_TASK_WORK |
1142 | struct posix_cputimers_work posix_cputimers_work; |
1143 | #endif |
1144 | |
1145 | /* Process credentials: */ |
1146 | |
1147 | /* Tracer's credentials at attach: */ |
1148 | const struct cred __rcu *ptracer_cred; |
1149 | |
1150 | /* Objective and real subjective task credentials (COW): */ |
1151 | const struct cred __rcu *real_cred; |
1152 | |
1153 | /* Effective (overridable) subjective task credentials (COW): */ |
1154 | const struct cred __rcu *cred; |
1155 | |
1156 | #ifdef CONFIG_KEYS |
1157 | /* Cached requested key. */ |
1158 | struct key *cached_requested_key; |
1159 | #endif |
1160 | |
1161 | /* |
1162 | * executable name, excluding path. |
1163 | * |
1164 | * - normally initialized begin_new_exec() |
1165 | * - set it with set_task_comm() |
1166 | * - strscpy_pad() to ensure it is always NUL-terminated and |
1167 | * zero-padded |
1168 | * - task_lock() to ensure the operation is atomic and the name is |
1169 | * fully updated. |
1170 | */ |
1171 | char comm[TASK_COMM_LEN]; |
1172 | |
1173 | struct nameidata *nameidata; |
1174 | |
1175 | #ifdef CONFIG_SYSVIPC |
1176 | struct sysv_sem sysvsem; |
1177 | struct sysv_shm sysvshm; |
1178 | #endif |
1179 | #ifdef CONFIG_DETECT_HUNG_TASK |
1180 | unsigned long last_switch_count; |
1181 | unsigned long last_switch_time; |
1182 | #endif |
1183 | /* Filesystem information: */ |
1184 | struct fs_struct *fs; |
1185 | |
1186 | /* Open file information: */ |
1187 | struct files_struct *files; |
1188 | |
1189 | #ifdef CONFIG_IO_URING |
1190 | struct io_uring_task *io_uring; |
1191 | #endif |
1192 | |
1193 | /* Namespaces: */ |
1194 | struct nsproxy *nsproxy; |
1195 | |
1196 | /* Signal handlers: */ |
1197 | struct signal_struct *signal; |
1198 | struct sighand_struct __rcu *sighand; |
1199 | sigset_t blocked; |
1200 | sigset_t real_blocked; |
1201 | /* Restored if set_restore_sigmask() was used: */ |
1202 | sigset_t saved_sigmask; |
1203 | struct sigpending pending; |
1204 | unsigned long sas_ss_sp; |
1205 | size_t sas_ss_size; |
1206 | unsigned int sas_ss_flags; |
1207 | |
1208 | struct callback_head *task_works; |
1209 | |
1210 | #ifdef CONFIG_AUDIT |
1211 | #ifdef CONFIG_AUDITSYSCALL |
1212 | struct audit_context *audit_context; |
1213 | #endif |
1214 | kuid_t loginuid; |
1215 | unsigned int sessionid; |
1216 | #endif |
1217 | struct seccomp seccomp; |
1218 | struct syscall_user_dispatch syscall_dispatch; |
1219 | |
1220 | /* Thread group tracking: */ |
1221 | u64 parent_exec_id; |
1222 | u64 self_exec_id; |
1223 | |
1224 | /* Protection against (de-)allocation: mm, files, fs, tty, keyrings, mems_allowed, mempolicy: */ |
1225 | spinlock_t alloc_lock; |
1226 | |
1227 | /* Protection of the PI data structures: */ |
1228 | raw_spinlock_t pi_lock; |
1229 | |
1230 | struct wake_q_node wake_q; |
1231 | |
1232 | #ifdef CONFIG_RT_MUTEXES |
1233 | /* PI waiters blocked on a rt_mutex held by this task: */ |
1234 | struct rb_root_cached pi_waiters; |
1235 | /* Updated under owner's pi_lock and rq lock */ |
1236 | struct task_struct *pi_top_task; |
1237 | /* Deadlock detection and priority inheritance handling: */ |
1238 | struct rt_mutex_waiter *pi_blocked_on; |
1239 | #endif |
1240 | |
1241 | #ifdef CONFIG_DEBUG_MUTEXES |
1242 | /* Mutex deadlock detection: */ |
1243 | struct mutex_waiter *blocked_on; |
1244 | #endif |
1245 | |
1246 | #ifdef CONFIG_DETECT_HUNG_TASK_BLOCKER |
1247 | /* |
1248 | * Encoded lock address causing task block (lower 2 bits = type from |
1249 | * <linux/hung_task.h>). Accessed via hung_task_*() helpers. |
1250 | */ |
1251 | unsigned long blocker; |
1252 | #endif |
1253 | |
1254 | #ifdef CONFIG_DEBUG_ATOMIC_SLEEP |
1255 | int non_block_count; |
1256 | #endif |
1257 | |
1258 | #ifdef CONFIG_TRACE_IRQFLAGS |
1259 | struct irqtrace_events irqtrace; |
1260 | unsigned int hardirq_threaded; |
1261 | u64 hardirq_chain_key; |
1262 | int softirqs_enabled; |
1263 | int softirq_context; |
1264 | int irq_config; |
1265 | #endif |
1266 | #ifdef CONFIG_PREEMPT_RT |
1267 | int softirq_disable_cnt; |
1268 | #endif |
1269 | |
1270 | #ifdef CONFIG_LOCKDEP |
1271 | # define MAX_LOCK_DEPTH 48UL |
1272 | u64 curr_chain_key; |
1273 | int lockdep_depth; |
1274 | unsigned int lockdep_recursion; |
1275 | struct held_lock held_locks[MAX_LOCK_DEPTH]; |
1276 | #endif |
1277 | |
1278 | #if defined(CONFIG_UBSAN) && !defined(CONFIG_UBSAN_TRAP) |
1279 | unsigned int in_ubsan; |
1280 | #endif |
1281 | |
1282 | /* Journalling filesystem info: */ |
1283 | void *journal_info; |
1284 | |
1285 | /* Stacked block device info: */ |
1286 | struct bio_list *bio_list; |
1287 | |
1288 | /* Stack plugging: */ |
1289 | struct blk_plug *plug; |
1290 | |
1291 | /* VM state: */ |
1292 | struct reclaim_state *reclaim_state; |
1293 | |
1294 | struct io_context *io_context; |
1295 | |
1296 | #ifdef CONFIG_COMPACTION |
1297 | struct capture_control *capture_control; |
1298 | #endif |
1299 | /* Ptrace state: */ |
1300 | unsigned long ptrace_message; |
1301 | kernel_siginfo_t *last_siginfo; |
1302 | |
1303 | struct task_io_accounting ioac; |
1304 | #ifdef CONFIG_PSI |
1305 | /* Pressure stall state */ |
1306 | unsigned int psi_flags; |
1307 | #endif |
1308 | #ifdef CONFIG_TASK_XACCT |
1309 | /* Accumulated RSS usage: */ |
1310 | u64 acct_rss_mem1; |
1311 | /* Accumulated virtual memory usage: */ |
1312 | u64 acct_vm_mem1; |
1313 | /* stime + utime since last update: */ |
1314 | u64 acct_timexpd; |
1315 | #endif |
1316 | #ifdef CONFIG_CPUSETS |
1317 | /* Protected by ->alloc_lock: */ |
1318 | nodemask_t mems_allowed; |
1319 | /* Sequence number to catch updates: */ |
1320 | seqcount_spinlock_t mems_allowed_seq; |
1321 | int cpuset_mem_spread_rotor; |
1322 | #endif |
1323 | #ifdef CONFIG_CGROUPS |
1324 | /* Control Group info protected by css_set_lock: */ |
1325 | struct css_set __rcu *cgroups; |
1326 | /* cg_list protected by css_set_lock and tsk->alloc_lock: */ |
1327 | struct list_head cg_list; |
1328 | #endif |
1329 | #ifdef CONFIG_X86_CPU_RESCTRL |
1330 | u32 closid; |
1331 | u32 rmid; |
1332 | #endif |
1333 | #ifdef CONFIG_FUTEX |
1334 | struct robust_list_head __user *robust_list; |
1335 | #ifdef CONFIG_COMPAT |
1336 | struct compat_robust_list_head __user *compat_robust_list; |
1337 | #endif |
1338 | struct list_head pi_state_list; |
1339 | struct futex_pi_state *pi_state_cache; |
1340 | struct mutex futex_exit_mutex; |
1341 | unsigned int futex_state; |
1342 | #endif |
1343 | #ifdef CONFIG_PERF_EVENTS |
1344 | u8 perf_recursion[PERF_NR_CONTEXTS]; |
1345 | struct perf_event_context *perf_event_ctxp; |
1346 | struct mutex perf_event_mutex; |
1347 | struct list_head perf_event_list; |
1348 | struct perf_ctx_data __rcu *perf_ctx_data; |
1349 | #endif |
1350 | #ifdef CONFIG_DEBUG_PREEMPT |
1351 | unsigned long preempt_disable_ip; |
1352 | #endif |
1353 | #ifdef CONFIG_NUMA |
1354 | /* Protected by alloc_lock: */ |
1355 | struct mempolicy *mempolicy; |
1356 | short il_prev; |
1357 | u8 il_weight; |
1358 | short pref_node_fork; |
1359 | #endif |
1360 | #ifdef CONFIG_NUMA_BALANCING |
1361 | int numa_scan_seq; |
1362 | unsigned int numa_scan_period; |
1363 | unsigned int numa_scan_period_max; |
1364 | int numa_preferred_nid; |
1365 | unsigned long numa_migrate_retry; |
1366 | /* Migration stamp: */ |
1367 | u64 node_stamp; |
1368 | u64 last_task_numa_placement; |
1369 | u64 last_sum_exec_runtime; |
1370 | struct callback_head numa_work; |
1371 | |
1372 | /* |
1373 | * This pointer is only modified for current in syscall and |
1374 | * pagefault context (and for tasks being destroyed), so it can be read |
1375 | * from any of the following contexts: |
1376 | * - RCU read-side critical section |
1377 | * - current->numa_group from everywhere |
1378 | * - task's runqueue locked, task not running |
1379 | */ |
1380 | struct numa_group __rcu *numa_group; |
1381 | |
1382 | /* |
1383 | * numa_faults is an array split into four regions: |
1384 | * faults_memory, faults_cpu, faults_memory_buffer, faults_cpu_buffer |
1385 | * in this precise order. |
1386 | * |
1387 | * faults_memory: Exponential decaying average of faults on a per-node |
1388 | * basis. Scheduling placement decisions are made based on these |
1389 | * counts. The values remain static for the duration of a PTE scan. |
1390 | * faults_cpu: Track the nodes the process was running on when a NUMA |
1391 | * hinting fault was incurred. |
1392 | * faults_memory_buffer and faults_cpu_buffer: Record faults per node |
1393 | * during the current scan window. When the scan completes, the counts |
1394 | * in faults_memory and faults_cpu decay and these values are copied. |
1395 | */ |
1396 | unsigned long *numa_faults; |
1397 | unsigned long total_numa_faults; |
1398 | |
1399 | /* |
1400 | * numa_faults_locality tracks if faults recorded during the last |
1401 | * scan window were remote/local or failed to migrate. The task scan |
1402 | * period is adapted based on the locality of the faults with different |
1403 | * weights depending on whether they were shared or private faults |
1404 | */ |
1405 | unsigned long numa_faults_locality[3]; |
1406 | |
1407 | unsigned long numa_pages_migrated; |
1408 | #endif /* CONFIG_NUMA_BALANCING */ |
1409 | |
1410 | #ifdef CONFIG_RSEQ |
1411 | struct rseq __user *rseq; |
1412 | u32 rseq_len; |
1413 | u32 rseq_sig; |
1414 | /* |
1415 | * RmW on rseq_event_mask must be performed atomically |
1416 | * with respect to preemption. |
1417 | */ |
1418 | unsigned long rseq_event_mask; |
1419 | # ifdef CONFIG_DEBUG_RSEQ |
1420 | /* |
1421 | * This is a place holder to save a copy of the rseq fields for |
1422 | * validation of read-only fields. The struct rseq has a |
1423 | * variable-length array at the end, so it cannot be used |
1424 | * directly. Reserve a size large enough for the known fields. |
1425 | */ |
1426 | char rseq_fields[sizeof(struct rseq)]; |
1427 | # endif |
1428 | #endif |
1429 | |
1430 | #ifdef CONFIG_SCHED_MM_CID |
1431 | int mm_cid; /* Current cid in mm */ |
1432 | int last_mm_cid; /* Most recent cid in mm */ |
1433 | int migrate_from_cpu; |
1434 | int mm_cid_active; /* Whether cid bitmap is active */ |
1435 | struct callback_head cid_work; |
1436 | #endif |
1437 | |
1438 | struct tlbflush_unmap_batch tlb_ubc; |
1439 | |
1440 | /* Cache last used pipe for splice(): */ |
1441 | struct pipe_inode_info *splice_pipe; |
1442 | |
1443 | struct page_frag task_frag; |
1444 | |
1445 | #ifdef CONFIG_TASK_DELAY_ACCT |
1446 | struct task_delay_info *delays; |
1447 | #endif |
1448 | |
1449 | #ifdef CONFIG_FAULT_INJECTION |
1450 | int make_it_fail; |
1451 | unsigned int fail_nth; |
1452 | #endif |
1453 | /* |
1454 | * When (nr_dirtied >= nr_dirtied_pause), it's time to call |
1455 | * balance_dirty_pages() for a dirty throttling pause: |
1456 | */ |
1457 | int nr_dirtied; |
1458 | int nr_dirtied_pause; |
1459 | /* Start of a write-and-pause period: */ |
1460 | unsigned long dirty_paused_when; |
1461 | |
1462 | #ifdef CONFIG_LATENCYTOP |
1463 | int latency_record_count; |
1464 | struct latency_record latency_record[LT_SAVECOUNT]; |
1465 | #endif |
1466 | /* |
1467 | * Time slack values; these are used to round up poll() and |
1468 | * select() etc timeout values. These are in nanoseconds. |
1469 | */ |
1470 | u64 timer_slack_ns; |
1471 | u64 default_timer_slack_ns; |
1472 | |
1473 | #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) |
1474 | unsigned int kasan_depth; |
1475 | #endif |
1476 | |
1477 | #ifdef CONFIG_KCSAN |
1478 | struct kcsan_ctx kcsan_ctx; |
1479 | #ifdef CONFIG_TRACE_IRQFLAGS |
1480 | struct irqtrace_events kcsan_save_irqtrace; |
1481 | #endif |
1482 | #ifdef CONFIG_KCSAN_WEAK_MEMORY |
1483 | int kcsan_stack_depth; |
1484 | #endif |
1485 | #endif |
1486 | |
1487 | #ifdef CONFIG_KMSAN |
1488 | struct kmsan_ctx kmsan_ctx; |
1489 | #endif |
1490 | |
1491 | #if IS_ENABLED(CONFIG_KUNIT) |
1492 | struct kunit *kunit_test; |
1493 | #endif |
1494 | |
1495 | #ifdef CONFIG_FUNCTION_GRAPH_TRACER |
1496 | /* Index of current stored address in ret_stack: */ |
1497 | int curr_ret_stack; |
1498 | int curr_ret_depth; |
1499 | |
1500 | /* Stack of return addresses for return function tracing: */ |
1501 | unsigned long *ret_stack; |
1502 | |
1503 | /* Timestamp for last schedule: */ |
1504 | unsigned long long ftrace_timestamp; |
1505 | unsigned long long ftrace_sleeptime; |
1506 | |
1507 | /* |
1508 | * Number of functions that haven't been traced |
1509 | * because of depth overrun: |
1510 | */ |
1511 | atomic_t trace_overrun; |
1512 | |
1513 | /* Pause tracing: */ |
1514 | atomic_t tracing_graph_pause; |
1515 | #endif |
1516 | |
1517 | #ifdef CONFIG_TRACING |
1518 | /* Bitmask and counter of trace recursion: */ |
1519 | unsigned long trace_recursion; |
1520 | #endif /* CONFIG_TRACING */ |
1521 | |
1522 | #ifdef CONFIG_KCOV |
1523 | /* See kernel/kcov.c for more details. */ |
1524 | |
1525 | /* Coverage collection mode enabled for this task (0 if disabled): */ |
1526 | unsigned int kcov_mode; |
1527 | |
1528 | /* Size of the kcov_area: */ |
1529 | unsigned int kcov_size; |
1530 | |
1531 | /* Buffer for coverage collection: */ |
1532 | void *kcov_area; |
1533 | |
1534 | /* KCOV descriptor wired with this task or NULL: */ |
1535 | struct kcov *kcov; |
1536 | |
1537 | /* KCOV common handle for remote coverage collection: */ |
1538 | u64 kcov_handle; |
1539 | |
1540 | /* KCOV sequence number: */ |
1541 | int kcov_sequence; |
1542 | |
1543 | /* Collect coverage from softirq context: */ |
1544 | unsigned int kcov_softirq; |
1545 | #endif |
1546 | |
1547 | #ifdef CONFIG_MEMCG_V1 |
1548 | struct mem_cgroup *memcg_in_oom; |
1549 | #endif |
1550 | |
1551 | #ifdef CONFIG_MEMCG |
1552 | /* Number of pages to reclaim on returning to userland: */ |
1553 | unsigned int memcg_nr_pages_over_high; |
1554 | |
1555 | /* Used by memcontrol for targeted memcg charge: */ |
1556 | struct mem_cgroup *active_memcg; |
1557 | |
1558 | /* Cache for current->cgroups->memcg->objcg lookups: */ |
1559 | struct obj_cgroup *objcg; |
1560 | #endif |
1561 | |
1562 | #ifdef CONFIG_BLK_CGROUP |
1563 | struct gendisk *throttle_disk; |
1564 | #endif |
1565 | |
1566 | #ifdef CONFIG_UPROBES |
1567 | struct uprobe_task *utask; |
1568 | #endif |
1569 | #if defined(CONFIG_BCACHE) || defined(CONFIG_BCACHE_MODULE) |
1570 | unsigned int sequential_io; |
1571 | unsigned int sequential_io_avg; |
1572 | #endif |
1573 | struct kmap_ctrl kmap_ctrl; |
1574 | #ifdef CONFIG_DEBUG_ATOMIC_SLEEP |
1575 | unsigned long task_state_change; |
1576 | # ifdef CONFIG_PREEMPT_RT |
1577 | unsigned long saved_state_change; |
1578 | # endif |
1579 | #endif |
1580 | struct rcu_head rcu; |
1581 | refcount_t rcu_users; |
1582 | int pagefault_disabled; |
1583 | #ifdef CONFIG_MMU |
1584 | struct task_struct *oom_reaper_list; |
1585 | struct timer_list oom_reaper_timer; |
1586 | #endif |
1587 | #ifdef CONFIG_VMAP_STACK |
1588 | struct vm_struct *stack_vm_area; |
1589 | #endif |
1590 | #ifdef CONFIG_THREAD_INFO_IN_TASK |
1591 | /* A live task holds one reference: */ |
1592 | refcount_t stack_refcount; |
1593 | #endif |
1594 | #ifdef CONFIG_LIVEPATCH |
1595 | int patch_state; |
1596 | #endif |
1597 | #ifdef CONFIG_SECURITY |
1598 | /* Used by LSM modules for access restriction: */ |
1599 | void *security; |
1600 | #endif |
1601 | #ifdef CONFIG_BPF_SYSCALL |
1602 | /* Used by BPF task local storage */ |
1603 | struct bpf_local_storage __rcu *bpf_storage; |
1604 | /* Used for BPF run context */ |
1605 | struct bpf_run_ctx *bpf_ctx; |
1606 | #endif |
1607 | /* Used by BPF for per-TASK xdp storage */ |
1608 | struct bpf_net_context *bpf_net_context; |
1609 | |
1610 | #ifdef CONFIG_GCC_PLUGIN_STACKLEAK |
1611 | unsigned long lowest_stack; |
1612 | unsigned long prev_lowest_stack; |
1613 | #endif |
1614 | |
1615 | #ifdef CONFIG_X86_MCE |
1616 | void __user *mce_vaddr; |
1617 | __u64 mce_kflags; |
1618 | u64 mce_addr; |
1619 | __u64 mce_ripv : 1, |
1620 | mce_whole_page : 1, |
1621 | __mce_reserved : 62; |
1622 | struct callback_head mce_kill_me; |
1623 | int mce_count; |
1624 | #endif |
1625 | |
1626 | #ifdef CONFIG_KRETPROBES |
1627 | struct llist_head kretprobe_instances; |
1628 | #endif |
1629 | #ifdef CONFIG_RETHOOK |
1630 | struct llist_head rethooks; |
1631 | #endif |
1632 | |
1633 | #ifdef CONFIG_ARCH_HAS_PARANOID_L1D_FLUSH |
1634 | /* |
1635 | * If L1D flush is supported on mm context switch |
1636 | * then we use this callback head to queue kill work |
1637 | * to kill tasks that are not running on SMT disabled |
1638 | * cores |
1639 | */ |
1640 | struct callback_head l1d_flush_kill; |
1641 | #endif |
1642 | |
1643 | #ifdef CONFIG_RV |
1644 | /* |
1645 | * Per-task RV monitor. Nowadays fixed in RV_PER_TASK_MONITORS. |
1646 | * If we find justification for more monitors, we can think |
1647 | * about adding more or developing a dynamic method. So far, |
1648 | * none of these are justified. |
1649 | */ |
1650 | union rv_task_monitor rv[RV_PER_TASK_MONITORS]; |
1651 | #endif |
1652 | |
1653 | #ifdef CONFIG_USER_EVENTS |
1654 | struct user_event_mm *user_event_mm; |
1655 | #endif |
1656 | |
1657 | /* CPU-specific state of this task: */ |
1658 | struct thread_struct thread; |
1659 | |
1660 | /* |
1661 | * New fields for task_struct should be added above here, so that |
1662 | * they are included in the randomized portion of task_struct. |
1663 | */ |
1664 | randomized_struct_fields_end |
1665 | } __attribute__ ((aligned (64))); |
1666 | |
1667 | #define TASK_REPORT_IDLE (TASK_REPORT + 1) |
1668 | #define TASK_REPORT_MAX (TASK_REPORT_IDLE << 1) |
1669 | |
1670 | static inline unsigned int __task_state_index(unsigned int tsk_state, |
1671 | unsigned int tsk_exit_state) |
1672 | { |
1673 | unsigned int state = (tsk_state | tsk_exit_state) & TASK_REPORT; |
1674 | |
1675 | BUILD_BUG_ON_NOT_POWER_OF_2(TASK_REPORT_MAX); |
1676 | |
1677 | if ((tsk_state & TASK_IDLE) == TASK_IDLE) |
1678 | state = TASK_REPORT_IDLE; |
1679 | |
1680 | /* |
1681 | * We're lying here, but rather than expose a completely new task state |
1682 | * to userspace, we can make this appear as if the task has gone through |
1683 | * a regular rt_mutex_lock() call. |
1684 | * Report frozen tasks as uninterruptible. |
1685 | */ |
1686 | if ((tsk_state & TASK_RTLOCK_WAIT) || (tsk_state & TASK_FROZEN)) |
1687 | state = TASK_UNINTERRUPTIBLE; |
1688 | |
1689 | return fls(x: state); |
1690 | } |
1691 | |
1692 | static inline unsigned int task_state_index(struct task_struct *tsk) |
1693 | { |
1694 | return __task_state_index(READ_ONCE(tsk->__state), tsk_exit_state: tsk->exit_state); |
1695 | } |
1696 | |
1697 | static inline char task_index_to_char(unsigned int state) |
1698 | { |
1699 | static const char state_char[] = "RSDTtXZPI"; |
1700 | |
1701 | BUILD_BUG_ON(TASK_REPORT_MAX * 2 != 1 << (sizeof(state_char) - 1)); |
1702 | |
1703 | return state_char[state]; |
1704 | } |
1705 | |
1706 | static inline char task_state_to_char(struct task_struct *tsk) |
1707 | { |
1708 | return task_index_to_char(state: task_state_index(tsk)); |
1709 | } |
1710 | |
1711 | extern struct pid *cad_pid; |
1712 | |
1713 | /* |
1714 | * Per process flags |
1715 | */ |
1716 | #define PF_VCPU 0x00000001 /* I'm a virtual CPU */ |
1717 | #define PF_IDLE 0x00000002 /* I am an IDLE thread */ |
1718 | #define PF_EXITING 0x00000004 /* Getting shut down */ |
1719 | #define PF_POSTCOREDUMP 0x00000008 /* Coredumps should ignore this task */ |
1720 | #define PF_IO_WORKER 0x00000010 /* Task is an IO worker */ |
1721 | #define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */ |
1722 | #define PF_FORKNOEXEC 0x00000040 /* Forked but didn't exec */ |
1723 | #define PF_MCE_PROCESS 0x00000080 /* Process policy on mce errors */ |
1724 | #define PF_SUPERPRIV 0x00000100 /* Used super-user privileges */ |
1725 | #define PF_DUMPCORE 0x00000200 /* Dumped core */ |
1726 | #define PF_SIGNALED 0x00000400 /* Killed by a signal */ |
1727 | #define PF_MEMALLOC 0x00000800 /* Allocating memory to free memory. See memalloc_noreclaim_save() */ |
1728 | #define PF_NPROC_EXCEEDED 0x00001000 /* set_user() noticed that RLIMIT_NPROC was exceeded */ |
1729 | #define PF_USED_MATH 0x00002000 /* If unset the fpu must be initialized before use */ |
1730 | #define PF_USER_WORKER 0x00004000 /* Kernel thread cloned from userspace thread */ |
1731 | #define PF_NOFREEZE 0x00008000 /* This thread should not be frozen */ |
1732 | #define PF_KCOMPACTD 0x00010000 /* I am kcompactd */ |
1733 | #define PF_KSWAPD 0x00020000 /* I am kswapd */ |
1734 | #define PF_MEMALLOC_NOFS 0x00040000 /* All allocations inherit GFP_NOFS. See memalloc_nfs_save() */ |
1735 | #define PF_MEMALLOC_NOIO 0x00080000 /* All allocations inherit GFP_NOIO. See memalloc_noio_save() */ |
1736 | #define PF_LOCAL_THROTTLE 0x00100000 /* Throttle writes only against the bdi I write to, |
1737 | * I am cleaning dirty pages from some other bdi. */ |
1738 | #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ |
1739 | #define PF_RANDOMIZE 0x00400000 /* Randomize virtual address space */ |
1740 | #define PF__HOLE__00800000 0x00800000 |
1741 | #define PF__HOLE__01000000 0x01000000 |
1742 | #define PF__HOLE__02000000 0x02000000 |
1743 | #define PF_NO_SETAFFINITY 0x04000000 /* Userland is not allowed to meddle with cpus_mask */ |
1744 | #define PF_MCE_EARLY 0x08000000 /* Early kill for mce process policy */ |
1745 | #define PF_MEMALLOC_PIN 0x10000000 /* Allocations constrained to zones which allow long term pinning. |
1746 | * See memalloc_pin_save() */ |
1747 | #define PF_BLOCK_TS 0x20000000 /* plug has ts that needs updating */ |
1748 | #define PF__HOLE__40000000 0x40000000 |
1749 | #define PF_SUSPEND_TASK 0x80000000 /* This thread called freeze_processes() and should not be frozen */ |
1750 | |
1751 | /* |
1752 | * Only the _current_ task can read/write to tsk->flags, but other |
1753 | * tasks can access tsk->flags in readonly mode for example |
1754 | * with tsk_used_math (like during threaded core dumping). |
1755 | * There is however an exception to this rule during ptrace |
1756 | * or during fork: the ptracer task is allowed to write to the |
1757 | * child->flags of its traced child (same goes for fork, the parent |
1758 | * can write to the child->flags), because we're guaranteed the |
1759 | * child is not running and in turn not changing child->flags |
1760 | * at the same time the parent does it. |
1761 | */ |
1762 | #define clear_stopped_child_used_math(child) do { (child)->flags &= ~PF_USED_MATH; } while (0) |
1763 | #define set_stopped_child_used_math(child) do { (child)->flags |= PF_USED_MATH; } while (0) |
1764 | #define clear_used_math() clear_stopped_child_used_math(current) |
1765 | #define set_used_math() set_stopped_child_used_math(current) |
1766 | |
1767 | #define conditional_stopped_child_used_math(condition, child) \ |
1768 | do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= (condition) ? PF_USED_MATH : 0; } while (0) |
1769 | |
1770 | #define conditional_used_math(condition) conditional_stopped_child_used_math(condition, current) |
1771 | |
1772 | #define copy_to_stopped_child_used_math(child) \ |
1773 | do { (child)->flags &= ~PF_USED_MATH, (child)->flags |= current->flags & PF_USED_MATH; } while (0) |
1774 | |
1775 | /* NOTE: this will return 0 or PF_USED_MATH, it will never return 1 */ |
1776 | #define tsk_used_math(p) ((p)->flags & PF_USED_MATH) |
1777 | #define used_math() tsk_used_math(current) |
1778 | |
1779 | static __always_inline bool is_percpu_thread(void) |
1780 | { |
1781 | #ifdef CONFIG_SMP |
1782 | return (current->flags & PF_NO_SETAFFINITY) && |
1783 | (current->nr_cpus_allowed == 1); |
1784 | #else |
1785 | return true; |
1786 | #endif |
1787 | } |
1788 | |
1789 | /* Per-process atomic flags. */ |
1790 | #define PFA_NO_NEW_PRIVS 0 /* May not gain new privileges. */ |
1791 | #define PFA_SPREAD_PAGE 1 /* Spread page cache over cpuset */ |
1792 | #define PFA_SPREAD_SLAB 2 /* Spread some slab caches over cpuset */ |
1793 | #define PFA_SPEC_SSB_DISABLE 3 /* Speculative Store Bypass disabled */ |
1794 | #define PFA_SPEC_SSB_FORCE_DISABLE 4 /* Speculative Store Bypass force disabled*/ |
1795 | #define PFA_SPEC_IB_DISABLE 5 /* Indirect branch speculation restricted */ |
1796 | #define PFA_SPEC_IB_FORCE_DISABLE 6 /* Indirect branch speculation permanently restricted */ |
1797 | #define PFA_SPEC_SSB_NOEXEC 7 /* Speculative Store Bypass clear on execve() */ |
1798 | |
1799 | #define TASK_PFA_TEST(name, func) \ |
1800 | static inline bool task_##func(struct task_struct *p) \ |
1801 | { return test_bit(PFA_##name, &p->atomic_flags); } |
1802 | |
1803 | #define TASK_PFA_SET(name, func) \ |
1804 | static inline void task_set_##func(struct task_struct *p) \ |
1805 | { set_bit(PFA_##name, &p->atomic_flags); } |
1806 | |
1807 | #define TASK_PFA_CLEAR(name, func) \ |
1808 | static inline void task_clear_##func(struct task_struct *p) \ |
1809 | { clear_bit(PFA_##name, &p->atomic_flags); } |
1810 | |
1811 | TASK_PFA_TEST(NO_NEW_PRIVS, no_new_privs) |
1812 | TASK_PFA_SET(NO_NEW_PRIVS, no_new_privs) |
1813 | |
1814 | TASK_PFA_TEST(SPREAD_PAGE, spread_page) |
1815 | TASK_PFA_SET(SPREAD_PAGE, spread_page) |
1816 | TASK_PFA_CLEAR(SPREAD_PAGE, spread_page) |
1817 | |
1818 | TASK_PFA_TEST(SPREAD_SLAB, spread_slab) |
1819 | TASK_PFA_SET(SPREAD_SLAB, spread_slab) |
1820 | TASK_PFA_CLEAR(SPREAD_SLAB, spread_slab) |
1821 | |
1822 | TASK_PFA_TEST(SPEC_SSB_DISABLE, spec_ssb_disable) |
1823 | TASK_PFA_SET(SPEC_SSB_DISABLE, spec_ssb_disable) |
1824 | TASK_PFA_CLEAR(SPEC_SSB_DISABLE, spec_ssb_disable) |
1825 | |
1826 | TASK_PFA_TEST(SPEC_SSB_NOEXEC, spec_ssb_noexec) |
1827 | TASK_PFA_SET(SPEC_SSB_NOEXEC, spec_ssb_noexec) |
1828 | TASK_PFA_CLEAR(SPEC_SSB_NOEXEC, spec_ssb_noexec) |
1829 | |
1830 | TASK_PFA_TEST(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable) |
1831 | TASK_PFA_SET(SPEC_SSB_FORCE_DISABLE, spec_ssb_force_disable) |
1832 | |
1833 | TASK_PFA_TEST(SPEC_IB_DISABLE, spec_ib_disable) |
1834 | TASK_PFA_SET(SPEC_IB_DISABLE, spec_ib_disable) |
1835 | TASK_PFA_CLEAR(SPEC_IB_DISABLE, spec_ib_disable) |
1836 | |
1837 | TASK_PFA_TEST(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable) |
1838 | TASK_PFA_SET(SPEC_IB_FORCE_DISABLE, spec_ib_force_disable) |
1839 | |
1840 | static inline void |
1841 | current_restore_flags(unsigned long orig_flags, unsigned long flags) |
1842 | { |
1843 | current->flags &= ~flags; |
1844 | current->flags |= orig_flags & flags; |
1845 | } |
1846 | |
1847 | extern int cpuset_cpumask_can_shrink(const struct cpumask *cur, const struct cpumask *trial); |
1848 | extern int task_can_attach(struct task_struct *p); |
1849 | extern int dl_bw_alloc(int cpu, u64 dl_bw); |
1850 | extern void dl_bw_free(int cpu, u64 dl_bw); |
1851 | #ifdef CONFIG_SMP |
1852 | |
1853 | /* do_set_cpus_allowed() - consider using set_cpus_allowed_ptr() instead */ |
1854 | extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask); |
1855 | |
1856 | /** |
1857 | * set_cpus_allowed_ptr - set CPU affinity mask of a task |
1858 | * @p: the task |
1859 | * @new_mask: CPU affinity mask |
1860 | * |
1861 | * Return: zero if successful, or a negative error code |
1862 | */ |
1863 | extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); |
1864 | extern int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node); |
1865 | extern void release_user_cpus_ptr(struct task_struct *p); |
1866 | extern int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask); |
1867 | extern void force_compatible_cpus_allowed_ptr(struct task_struct *p); |
1868 | extern void relax_compatible_cpus_allowed_ptr(struct task_struct *p); |
1869 | #else |
1870 | static inline void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) |
1871 | { |
1872 | } |
1873 | static inline int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) |
1874 | { |
1875 | /* Opencoded cpumask_test_cpu(0, new_mask) to avoid dependency on cpumask.h */ |
1876 | if ((*cpumask_bits(new_mask) & 1) == 0) |
1877 | return -EINVAL; |
1878 | return 0; |
1879 | } |
1880 | static inline int dup_user_cpus_ptr(struct task_struct *dst, struct task_struct *src, int node) |
1881 | { |
1882 | if (src->user_cpus_ptr) |
1883 | return -EINVAL; |
1884 | return 0; |
1885 | } |
1886 | static inline void release_user_cpus_ptr(struct task_struct *p) |
1887 | { |
1888 | WARN_ON(p->user_cpus_ptr); |
1889 | } |
1890 | |
1891 | static inline int dl_task_check_affinity(struct task_struct *p, const struct cpumask *mask) |
1892 | { |
1893 | return 0; |
1894 | } |
1895 | #endif |
1896 | |
1897 | extern int yield_to(struct task_struct *p, bool preempt); |
1898 | extern void set_user_nice(struct task_struct *p, long nice); |
1899 | extern int task_prio(const struct task_struct *p); |
1900 | |
1901 | /** |
1902 | * task_nice - return the nice value of a given task. |
1903 | * @p: the task in question. |
1904 | * |
1905 | * Return: The nice value [ -20 ... 0 ... 19 ]. |
1906 | */ |
1907 | static inline int task_nice(const struct task_struct *p) |
1908 | { |
1909 | return PRIO_TO_NICE((p)->static_prio); |
1910 | } |
1911 | |
1912 | extern int can_nice(const struct task_struct *p, const int nice); |
1913 | extern int task_curr(const struct task_struct *p); |
1914 | extern int idle_cpu(int cpu); |
1915 | extern int available_idle_cpu(int cpu); |
1916 | extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *); |
1917 | extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *); |
1918 | extern void sched_set_fifo(struct task_struct *p); |
1919 | extern void sched_set_fifo_low(struct task_struct *p); |
1920 | extern void sched_set_normal(struct task_struct *p, int nice); |
1921 | extern int sched_setattr(struct task_struct *, const struct sched_attr *); |
1922 | extern int sched_setattr_nocheck(struct task_struct *, const struct sched_attr *); |
1923 | extern struct task_struct *idle_task(int cpu); |
1924 | |
1925 | /** |
1926 | * is_idle_task - is the specified task an idle task? |
1927 | * @p: the task in question. |
1928 | * |
1929 | * Return: 1 if @p is an idle task. 0 otherwise. |
1930 | */ |
1931 | static __always_inline bool is_idle_task(const struct task_struct *p) |
1932 | { |
1933 | return !!(p->flags & PF_IDLE); |
1934 | } |
1935 | |
1936 | extern struct task_struct *curr_task(int cpu); |
1937 | extern void ia64_set_curr_task(int cpu, struct task_struct *p); |
1938 | |
1939 | void yield(void); |
1940 | |
1941 | union thread_union { |
1942 | struct task_struct task; |
1943 | #ifndef CONFIG_THREAD_INFO_IN_TASK |
1944 | struct thread_info thread_info; |
1945 | #endif |
1946 | unsigned long stack[THREAD_SIZE/sizeof(long)]; |
1947 | }; |
1948 | |
1949 | #ifndef CONFIG_THREAD_INFO_IN_TASK |
1950 | extern struct thread_info init_thread_info; |
1951 | #endif |
1952 | |
1953 | extern unsigned long init_stack[THREAD_SIZE / sizeof(unsigned long)]; |
1954 | |
1955 | #ifdef CONFIG_THREAD_INFO_IN_TASK |
1956 | # define task_thread_info(task) (&(task)->thread_info) |
1957 | #else |
1958 | # define task_thread_info(task) ((struct thread_info *)(task)->stack) |
1959 | #endif |
1960 | |
1961 | /* |
1962 | * find a task by one of its numerical ids |
1963 | * |
1964 | * find_task_by_pid_ns(): |
1965 | * finds a task by its pid in the specified namespace |
1966 | * find_task_by_vpid(): |
1967 | * finds a task by its virtual pid |
1968 | * |
1969 | * see also find_vpid() etc in include/linux/pid.h |
1970 | */ |
1971 | |
1972 | extern struct task_struct *find_task_by_vpid(pid_t nr); |
1973 | extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns); |
1974 | |
1975 | /* |
1976 | * find a task by its virtual pid and get the task struct |
1977 | */ |
1978 | extern struct task_struct *find_get_task_by_vpid(pid_t nr); |
1979 | |
1980 | extern int wake_up_state(struct task_struct *tsk, unsigned int state); |
1981 | extern int wake_up_process(struct task_struct *tsk); |
1982 | extern void wake_up_new_task(struct task_struct *tsk); |
1983 | |
1984 | #ifdef CONFIG_SMP |
1985 | extern void kick_process(struct task_struct *tsk); |
1986 | #else |
1987 | static inline void kick_process(struct task_struct *tsk) { } |
1988 | #endif |
1989 | |
1990 | extern void __set_task_comm(struct task_struct *tsk, const char *from, bool exec); |
1991 | #define set_task_comm(tsk, from) ({ \ |
1992 | BUILD_BUG_ON(sizeof(from) != TASK_COMM_LEN); \ |
1993 | __set_task_comm(tsk, from, false); \ |
1994 | }) |
1995 | |
1996 | /* |
1997 | * - Why not use task_lock()? |
1998 | * User space can randomly change their names anyway, so locking for readers |
1999 | * doesn't make sense. For writers, locking is probably necessary, as a race |
2000 | * condition could lead to long-term mixed results. |
2001 | * The strscpy_pad() in __set_task_comm() can ensure that the task comm is |
2002 | * always NUL-terminated and zero-padded. Therefore the race condition between |
2003 | * reader and writer is not an issue. |
2004 | * |
2005 | * - BUILD_BUG_ON() can help prevent the buf from being truncated. |
2006 | * Since the callers don't perform any return value checks, this safeguard is |
2007 | * necessary. |
2008 | */ |
2009 | #define get_task_comm(buf, tsk) ({ \ |
2010 | BUILD_BUG_ON(sizeof(buf) < TASK_COMM_LEN); \ |
2011 | strscpy_pad(buf, (tsk)->comm); \ |
2012 | buf; \ |
2013 | }) |
2014 | |
2015 | #ifdef CONFIG_SMP |
2016 | static __always_inline void scheduler_ipi(void) |
2017 | { |
2018 | /* |
2019 | * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting |
2020 | * TIF_NEED_RESCHED remotely (for the first time) will also send |
2021 | * this IPI. |
2022 | */ |
2023 | preempt_fold_need_resched(); |
2024 | } |
2025 | #else |
2026 | static inline void scheduler_ipi(void) { } |
2027 | #endif |
2028 | |
2029 | extern unsigned long wait_task_inactive(struct task_struct *, unsigned int match_state); |
2030 | |
2031 | /* |
2032 | * Set thread flags in other task's structures. |
2033 | * See asm/thread_info.h for TIF_xxxx flags available: |
2034 | */ |
2035 | static inline void set_tsk_thread_flag(struct task_struct *tsk, int flag) |
2036 | { |
2037 | set_ti_thread_flag(task_thread_info(tsk), flag); |
2038 | } |
2039 | |
2040 | static inline void clear_tsk_thread_flag(struct task_struct *tsk, int flag) |
2041 | { |
2042 | clear_ti_thread_flag(task_thread_info(tsk), flag); |
2043 | } |
2044 | |
2045 | static inline void update_tsk_thread_flag(struct task_struct *tsk, int flag, |
2046 | bool value) |
2047 | { |
2048 | update_ti_thread_flag(task_thread_info(tsk), flag, value); |
2049 | } |
2050 | |
2051 | static inline int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag) |
2052 | { |
2053 | return test_and_set_ti_thread_flag(task_thread_info(tsk), flag); |
2054 | } |
2055 | |
2056 | static inline int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag) |
2057 | { |
2058 | return test_and_clear_ti_thread_flag(task_thread_info(tsk), flag); |
2059 | } |
2060 | |
2061 | static inline int test_tsk_thread_flag(struct task_struct *tsk, int flag) |
2062 | { |
2063 | return test_ti_thread_flag(task_thread_info(tsk), flag); |
2064 | } |
2065 | |
2066 | static inline void set_tsk_need_resched(struct task_struct *tsk) |
2067 | { |
2068 | set_tsk_thread_flag(tsk,TIF_NEED_RESCHED); |
2069 | } |
2070 | |
2071 | static inline void clear_tsk_need_resched(struct task_struct *tsk) |
2072 | { |
2073 | atomic_long_andnot(_TIF_NEED_RESCHED | _TIF_NEED_RESCHED_LAZY, |
2074 | v: (atomic_long_t *)&task_thread_info(tsk)->flags); |
2075 | } |
2076 | |
2077 | static inline int test_tsk_need_resched(struct task_struct *tsk) |
2078 | { |
2079 | return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); |
2080 | } |
2081 | |
2082 | /* |
2083 | * cond_resched() and cond_resched_lock(): latency reduction via |
2084 | * explicit rescheduling in places that are safe. The return |
2085 | * value indicates whether a reschedule was done in fact. |
2086 | * cond_resched_lock() will drop the spinlock before scheduling, |
2087 | */ |
2088 | #if !defined(CONFIG_PREEMPTION) || defined(CONFIG_PREEMPT_DYNAMIC) |
2089 | extern int __cond_resched(void); |
2090 | |
2091 | #if defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_CALL) |
2092 | |
2093 | DECLARE_STATIC_CALL(cond_resched, __cond_resched); |
2094 | |
2095 | static __always_inline int _cond_resched(void) |
2096 | { |
2097 | return static_call_mod(cond_resched)(); |
2098 | } |
2099 | |
2100 | #elif defined(CONFIG_PREEMPT_DYNAMIC) && defined(CONFIG_HAVE_PREEMPT_DYNAMIC_KEY) |
2101 | |
2102 | extern int dynamic_cond_resched(void); |
2103 | |
2104 | static __always_inline int _cond_resched(void) |
2105 | { |
2106 | return dynamic_cond_resched(); |
2107 | } |
2108 | |
2109 | #else /* !CONFIG_PREEMPTION */ |
2110 | |
2111 | static inline int _cond_resched(void) |
2112 | { |
2113 | return __cond_resched(); |
2114 | } |
2115 | |
2116 | #endif /* PREEMPT_DYNAMIC && CONFIG_HAVE_PREEMPT_DYNAMIC_CALL */ |
2117 | |
2118 | #else /* CONFIG_PREEMPTION && !CONFIG_PREEMPT_DYNAMIC */ |
2119 | |
2120 | static inline int _cond_resched(void) |
2121 | { |
2122 | return 0; |
2123 | } |
2124 | |
2125 | #endif /* !CONFIG_PREEMPTION || CONFIG_PREEMPT_DYNAMIC */ |
2126 | |
2127 | #define cond_resched() ({ \ |
2128 | __might_resched(__FILE__, __LINE__, 0); \ |
2129 | _cond_resched(); \ |
2130 | }) |
2131 | |
2132 | extern int __cond_resched_lock(spinlock_t *lock); |
2133 | extern int __cond_resched_rwlock_read(rwlock_t *lock); |
2134 | extern int __cond_resched_rwlock_write(rwlock_t *lock); |
2135 | |
2136 | #define MIGHT_RESCHED_RCU_SHIFT 8 |
2137 | #define MIGHT_RESCHED_PREEMPT_MASK ((1U << MIGHT_RESCHED_RCU_SHIFT) - 1) |
2138 | |
2139 | #ifndef CONFIG_PREEMPT_RT |
2140 | /* |
2141 | * Non RT kernels have an elevated preempt count due to the held lock, |
2142 | * but are not allowed to be inside a RCU read side critical section |
2143 | */ |
2144 | # define PREEMPT_LOCK_RESCHED_OFFSETS PREEMPT_LOCK_OFFSET |
2145 | #else |
2146 | /* |
2147 | * spin/rw_lock() on RT implies rcu_read_lock(). The might_sleep() check in |
2148 | * cond_resched*lock() has to take that into account because it checks for |
2149 | * preempt_count() and rcu_preempt_depth(). |
2150 | */ |
2151 | # define PREEMPT_LOCK_RESCHED_OFFSETS \ |
2152 | (PREEMPT_LOCK_OFFSET + (1U << MIGHT_RESCHED_RCU_SHIFT)) |
2153 | #endif |
2154 | |
2155 | #define cond_resched_lock(lock) ({ \ |
2156 | __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \ |
2157 | __cond_resched_lock(lock); \ |
2158 | }) |
2159 | |
2160 | #define cond_resched_rwlock_read(lock) ({ \ |
2161 | __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \ |
2162 | __cond_resched_rwlock_read(lock); \ |
2163 | }) |
2164 | |
2165 | #define cond_resched_rwlock_write(lock) ({ \ |
2166 | __might_resched(__FILE__, __LINE__, PREEMPT_LOCK_RESCHED_OFFSETS); \ |
2167 | __cond_resched_rwlock_write(lock); \ |
2168 | }) |
2169 | |
2170 | static __always_inline bool need_resched(void) |
2171 | { |
2172 | return unlikely(tif_need_resched()); |
2173 | } |
2174 | |
2175 | /* |
2176 | * Wrappers for p->thread_info->cpu access. No-op on UP. |
2177 | */ |
2178 | #ifdef CONFIG_SMP |
2179 | |
2180 | static inline unsigned int task_cpu(const struct task_struct *p) |
2181 | { |
2182 | return READ_ONCE(task_thread_info(p)->cpu); |
2183 | } |
2184 | |
2185 | extern void set_task_cpu(struct task_struct *p, unsigned int cpu); |
2186 | |
2187 | #else |
2188 | |
2189 | static inline unsigned int task_cpu(const struct task_struct *p) |
2190 | { |
2191 | return 0; |
2192 | } |
2193 | |
2194 | static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) |
2195 | { |
2196 | } |
2197 | |
2198 | #endif /* CONFIG_SMP */ |
2199 | |
2200 | static inline bool task_is_runnable(struct task_struct *p) |
2201 | { |
2202 | return p->on_rq && !p->se.sched_delayed; |
2203 | } |
2204 | |
2205 | extern bool sched_task_on_rq(struct task_struct *p); |
2206 | extern unsigned long get_wchan(struct task_struct *p); |
2207 | extern struct task_struct *cpu_curr_snapshot(int cpu); |
2208 | |
2209 | #include <linux/spinlock.h> |
2210 | |
2211 | /* |
2212 | * In order to reduce various lock holder preemption latencies provide an |
2213 | * interface to see if a vCPU is currently running or not. |
2214 | * |
2215 | * This allows us to terminate optimistic spin loops and block, analogous to |
2216 | * the native optimistic spin heuristic of testing if the lock owner task is |
2217 | * running or not. |
2218 | */ |
2219 | #ifndef vcpu_is_preempted |
2220 | static inline bool vcpu_is_preempted(int cpu) |
2221 | { |
2222 | return false; |
2223 | } |
2224 | #endif |
2225 | |
2226 | extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); |
2227 | extern long sched_getaffinity(pid_t pid, struct cpumask *mask); |
2228 | |
2229 | #ifndef TASK_SIZE_OF |
2230 | #define TASK_SIZE_OF(tsk) TASK_SIZE |
2231 | #endif |
2232 | |
2233 | #ifdef CONFIG_SMP |
2234 | static inline bool owner_on_cpu(struct task_struct *owner) |
2235 | { |
2236 | /* |
2237 | * As lock holder preemption issue, we both skip spinning if |
2238 | * task is not on cpu or its cpu is preempted |
2239 | */ |
2240 | return READ_ONCE(owner->on_cpu) && !vcpu_is_preempted(cpu: task_cpu(p: owner)); |
2241 | } |
2242 | |
2243 | /* Returns effective CPU energy utilization, as seen by the scheduler */ |
2244 | unsigned long sched_cpu_util(int cpu); |
2245 | #endif /* CONFIG_SMP */ |
2246 | |
2247 | #ifdef CONFIG_SCHED_CORE |
2248 | extern void sched_core_free(struct task_struct *tsk); |
2249 | extern void sched_core_fork(struct task_struct *p); |
2250 | extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type, |
2251 | unsigned long uaddr); |
2252 | extern int sched_core_idle_cpu(int cpu); |
2253 | #else |
2254 | static inline void sched_core_free(struct task_struct *tsk) { } |
2255 | static inline void sched_core_fork(struct task_struct *p) { } |
2256 | static inline int sched_core_idle_cpu(int cpu) { return idle_cpu(cpu); } |
2257 | #endif |
2258 | |
2259 | extern void sched_set_stop_task(int cpu, struct task_struct *stop); |
2260 | |
2261 | #ifdef CONFIG_MEM_ALLOC_PROFILING |
2262 | static __always_inline struct alloc_tag *alloc_tag_save(struct alloc_tag *tag) |
2263 | { |
2264 | swap(current->alloc_tag, tag); |
2265 | return tag; |
2266 | } |
2267 | |
2268 | static __always_inline void alloc_tag_restore(struct alloc_tag *tag, struct alloc_tag *old) |
2269 | { |
2270 | #ifdef CONFIG_MEM_ALLOC_PROFILING_DEBUG |
2271 | WARN(current->alloc_tag != tag, "current->alloc_tag was changed:\n"); |
2272 | #endif |
2273 | current->alloc_tag = old; |
2274 | } |
2275 | #else |
2276 | #define alloc_tag_save(_tag) NULL |
2277 | #define alloc_tag_restore(_tag, _old) do {} while (0) |
2278 | #endif |
2279 | |
2280 | #endif |
2281 |
Definitions
- prev_cputime
- vtime_state
- vtime
- uclamp_id
- sched_param
- sched_info
- load_weight
- sched_avg
- sched_statistics
- sched_entity
- sched_rt_entity
- sched_dl_entity
- uclamp_se
- rcu_special
- perf_event_task_context
- wake_q_node
- kmap_ctrl
- task_struct
- __task_state_index
- task_state_index
- task_index_to_char
- task_state_to_char
- is_percpu_thread
- current_restore_flags
- task_nice
- is_idle_task
- thread_union
- scheduler_ipi
- set_tsk_thread_flag
- clear_tsk_thread_flag
- update_tsk_thread_flag
- test_and_set_tsk_thread_flag
- test_and_clear_tsk_thread_flag
- test_tsk_thread_flag
- set_tsk_need_resched
- clear_tsk_need_resched
- test_tsk_need_resched
- _cond_resched
- need_resched
- task_cpu
- task_is_runnable
Improve your Profiling and Debugging skills
Find out more