1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* sched.c - SPU scheduler. |
3 | * |
4 | * Copyright (C) IBM 2005 |
5 | * Author: Mark Nutter <mnutter@us.ibm.com> |
6 | * |
7 | * 2006-03-31 NUMA domains added. |
8 | */ |
9 | |
10 | #undef DEBUG |
11 | |
12 | #include <linux/errno.h> |
13 | #include <linux/sched/signal.h> |
14 | #include <linux/sched/loadavg.h> |
15 | #include <linux/sched/rt.h> |
16 | #include <linux/kernel.h> |
17 | #include <linux/mm.h> |
18 | #include <linux/slab.h> |
19 | #include <linux/completion.h> |
20 | #include <linux/vmalloc.h> |
21 | #include <linux/smp.h> |
22 | #include <linux/stddef.h> |
23 | #include <linux/unistd.h> |
24 | #include <linux/numa.h> |
25 | #include <linux/mutex.h> |
26 | #include <linux/notifier.h> |
27 | #include <linux/kthread.h> |
28 | #include <linux/pid_namespace.h> |
29 | #include <linux/proc_fs.h> |
30 | #include <linux/seq_file.h> |
31 | |
32 | #include <asm/io.h> |
33 | #include <asm/mmu_context.h> |
34 | #include <asm/spu.h> |
35 | #include <asm/spu_csa.h> |
36 | #include <asm/spu_priv1.h> |
37 | #include "spufs.h" |
38 | #define CREATE_TRACE_POINTS |
39 | #include "sputrace.h" |
40 | |
41 | struct spu_prio_array { |
42 | DECLARE_BITMAP(bitmap, MAX_PRIO); |
43 | struct list_head runq[MAX_PRIO]; |
44 | spinlock_t runq_lock; |
45 | int nr_waiting; |
46 | }; |
47 | |
48 | static unsigned long spu_avenrun[3]; |
49 | static struct spu_prio_array *spu_prio; |
50 | static struct task_struct *spusched_task; |
51 | static struct timer_list spusched_timer; |
52 | static struct timer_list spuloadavg_timer; |
53 | |
54 | /* |
55 | * Priority of a normal, non-rt, non-niced'd process (aka nice level 0). |
56 | */ |
57 | #define NORMAL_PRIO 120 |
58 | |
59 | /* |
60 | * Frequency of the spu scheduler tick. By default we do one SPU scheduler |
61 | * tick for every 10 CPU scheduler ticks. |
62 | */ |
63 | #define SPUSCHED_TICK (10) |
64 | |
65 | /* |
66 | * These are the 'tuning knobs' of the scheduler: |
67 | * |
68 | * Minimum timeslice is 5 msecs (or 1 spu scheduler tick, whichever is |
69 | * larger), default timeslice is 100 msecs, maximum timeslice is 800 msecs. |
70 | */ |
71 | #define MIN_SPU_TIMESLICE max(5 * HZ / (1000 * SPUSCHED_TICK), 1) |
72 | #define DEF_SPU_TIMESLICE (100 * HZ / (1000 * SPUSCHED_TICK)) |
73 | |
74 | #define SCALE_PRIO(x, prio) \ |
75 | max(x * (MAX_PRIO - prio) / (NICE_WIDTH / 2), MIN_SPU_TIMESLICE) |
76 | |
77 | /* |
78 | * scale user-nice values [ -20 ... 0 ... 19 ] to time slice values: |
79 | * [800ms ... 100ms ... 5ms] |
80 | * |
81 | * The higher a thread's priority, the bigger timeslices |
82 | * it gets during one round of execution. But even the lowest |
83 | * priority thread gets MIN_TIMESLICE worth of execution time. |
84 | */ |
85 | void spu_set_timeslice(struct spu_context *ctx) |
86 | { |
87 | if (ctx->prio < NORMAL_PRIO) |
88 | ctx->time_slice = SCALE_PRIO(DEF_SPU_TIMESLICE * 4, ctx->prio); |
89 | else |
90 | ctx->time_slice = SCALE_PRIO(DEF_SPU_TIMESLICE, ctx->prio); |
91 | } |
92 | |
93 | /* |
94 | * Update scheduling information from the owning thread. |
95 | */ |
96 | void __spu_update_sched_info(struct spu_context *ctx) |
97 | { |
98 | /* |
99 | * assert that the context is not on the runqueue, so it is safe |
100 | * to change its scheduling parameters. |
101 | */ |
102 | BUG_ON(!list_empty(&ctx->rq)); |
103 | |
104 | /* |
105 | * 32-Bit assignments are atomic on powerpc, and we don't care about |
106 | * memory ordering here because retrieving the controlling thread is |
107 | * per definition racy. |
108 | */ |
109 | ctx->tid = current->pid; |
110 | |
111 | /* |
112 | * We do our own priority calculations, so we normally want |
113 | * ->static_prio to start with. Unfortunately this field |
114 | * contains junk for threads with a realtime scheduling |
115 | * policy so we have to look at ->prio in this case. |
116 | */ |
117 | if (rt_prio(current->prio)) |
118 | ctx->prio = current->prio; |
119 | else |
120 | ctx->prio = current->static_prio; |
121 | ctx->policy = current->policy; |
122 | |
123 | /* |
124 | * TO DO: the context may be loaded, so we may need to activate |
125 | * it again on a different node. But it shouldn't hurt anything |
126 | * to update its parameters, because we know that the scheduler |
127 | * is not actively looking at this field, since it is not on the |
128 | * runqueue. The context will be rescheduled on the proper node |
129 | * if it is timesliced or preempted. |
130 | */ |
131 | cpumask_copy(dstp: &ctx->cpus_allowed, current->cpus_ptr); |
132 | |
133 | /* Save the current cpu id for spu interrupt routing. */ |
134 | ctx->last_ran = raw_smp_processor_id(); |
135 | } |
136 | |
137 | void spu_update_sched_info(struct spu_context *ctx) |
138 | { |
139 | int node; |
140 | |
141 | if (ctx->state == SPU_STATE_RUNNABLE) { |
142 | node = ctx->spu->node; |
143 | |
144 | /* |
145 | * Take list_mutex to sync with find_victim(). |
146 | */ |
147 | mutex_lock(&cbe_spu_info[node].list_mutex); |
148 | __spu_update_sched_info(ctx); |
149 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
150 | } else { |
151 | __spu_update_sched_info(ctx); |
152 | } |
153 | } |
154 | |
155 | static int __node_allowed(struct spu_context *ctx, int node) |
156 | { |
157 | if (nr_cpus_node(node)) { |
158 | const struct cpumask *mask = cpumask_of_node(node); |
159 | |
160 | if (cpumask_intersects(src1p: mask, src2p: &ctx->cpus_allowed)) |
161 | return 1; |
162 | } |
163 | |
164 | return 0; |
165 | } |
166 | |
167 | static int node_allowed(struct spu_context *ctx, int node) |
168 | { |
169 | int rval; |
170 | |
171 | spin_lock(lock: &spu_prio->runq_lock); |
172 | rval = __node_allowed(ctx, node); |
173 | spin_unlock(lock: &spu_prio->runq_lock); |
174 | |
175 | return rval; |
176 | } |
177 | |
178 | void do_notify_spus_active(void) |
179 | { |
180 | int node; |
181 | |
182 | /* |
183 | * Wake up the active spu_contexts. |
184 | */ |
185 | for_each_online_node(node) { |
186 | struct spu *spu; |
187 | |
188 | mutex_lock(&cbe_spu_info[node].list_mutex); |
189 | list_for_each_entry(spu, &cbe_spu_info[node].spus, cbe_list) { |
190 | if (spu->alloc_state != SPU_FREE) { |
191 | struct spu_context *ctx = spu->ctx; |
192 | set_bit(SPU_SCHED_NOTIFY_ACTIVE, |
193 | &ctx->sched_flags); |
194 | mb(); |
195 | wake_up_all(&ctx->stop_wq); |
196 | } |
197 | } |
198 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
199 | } |
200 | } |
201 | |
202 | /** |
203 | * spu_bind_context - bind spu context to physical spu |
204 | * @spu: physical spu to bind to |
205 | * @ctx: context to bind |
206 | */ |
207 | static void spu_bind_context(struct spu *spu, struct spu_context *ctx) |
208 | { |
209 | spu_context_trace(spu_bind_context__enter, ctx, spu); |
210 | |
211 | spuctx_switch_state(ctx, new_state: SPU_UTIL_SYSTEM); |
212 | |
213 | if (ctx->flags & SPU_CREATE_NOSCHED) |
214 | atomic_inc(v: &cbe_spu_info[spu->node].reserved_spus); |
215 | |
216 | ctx->stats.slb_flt_base = spu->stats.slb_flt; |
217 | ctx->stats.class2_intr_base = spu->stats.class2_intr; |
218 | |
219 | spu_associate_mm(spu, ctx->owner); |
220 | |
221 | spin_lock_irq(lock: &spu->register_lock); |
222 | spu->ctx = ctx; |
223 | spu->flags = 0; |
224 | ctx->spu = spu; |
225 | ctx->ops = &spu_hw_ops; |
226 | spu->pid = current->pid; |
227 | spu->tgid = current->tgid; |
228 | spu->ibox_callback = spufs_ibox_callback; |
229 | spu->wbox_callback = spufs_wbox_callback; |
230 | spu->stop_callback = spufs_stop_callback; |
231 | spu->mfc_callback = spufs_mfc_callback; |
232 | spin_unlock_irq(lock: &spu->register_lock); |
233 | |
234 | spu_unmap_mappings(ctx); |
235 | |
236 | spu_switch_log_notify(spu, ctx, type: SWITCH_LOG_START, val: 0); |
237 | spu_restore(new: &ctx->csa, spu); |
238 | spu->timestamp = jiffies; |
239 | ctx->state = SPU_STATE_RUNNABLE; |
240 | |
241 | spuctx_switch_state(ctx, new_state: SPU_UTIL_USER); |
242 | } |
243 | |
244 | /* |
245 | * Must be used with the list_mutex held. |
246 | */ |
247 | static inline int sched_spu(struct spu *spu) |
248 | { |
249 | BUG_ON(!mutex_is_locked(&cbe_spu_info[spu->node].list_mutex)); |
250 | |
251 | return (!spu->ctx || !(spu->ctx->flags & SPU_CREATE_NOSCHED)); |
252 | } |
253 | |
254 | static void aff_merge_remaining_ctxs(struct spu_gang *gang) |
255 | { |
256 | struct spu_context *ctx; |
257 | |
258 | list_for_each_entry(ctx, &gang->aff_list_head, aff_list) { |
259 | if (list_empty(head: &ctx->aff_list)) |
260 | list_add(new: &ctx->aff_list, head: &gang->aff_list_head); |
261 | } |
262 | gang->aff_flags |= AFF_MERGED; |
263 | } |
264 | |
265 | static void aff_set_offsets(struct spu_gang *gang) |
266 | { |
267 | struct spu_context *ctx; |
268 | int offset; |
269 | |
270 | offset = -1; |
271 | list_for_each_entry_reverse(ctx, &gang->aff_ref_ctx->aff_list, |
272 | aff_list) { |
273 | if (&ctx->aff_list == &gang->aff_list_head) |
274 | break; |
275 | ctx->aff_offset = offset--; |
276 | } |
277 | |
278 | offset = 0; |
279 | list_for_each_entry(ctx, gang->aff_ref_ctx->aff_list.prev, aff_list) { |
280 | if (&ctx->aff_list == &gang->aff_list_head) |
281 | break; |
282 | ctx->aff_offset = offset++; |
283 | } |
284 | |
285 | gang->aff_flags |= AFF_OFFSETS_SET; |
286 | } |
287 | |
288 | static struct spu *aff_ref_location(struct spu_context *ctx, int mem_aff, |
289 | int group_size, int lowest_offset) |
290 | { |
291 | struct spu *spu; |
292 | int node, n; |
293 | |
294 | /* |
295 | * TODO: A better algorithm could be used to find a good spu to be |
296 | * used as reference location for the ctxs chain. |
297 | */ |
298 | node = cpu_to_node(raw_smp_processor_id()); |
299 | for (n = 0; n < MAX_NUMNODES; n++, node++) { |
300 | /* |
301 | * "available_spus" counts how many spus are not potentially |
302 | * going to be used by other affinity gangs whose reference |
303 | * context is already in place. Although this code seeks to |
304 | * avoid having affinity gangs with a summed amount of |
305 | * contexts bigger than the amount of spus in the node, |
306 | * this may happen sporadically. In this case, available_spus |
307 | * becomes negative, which is harmless. |
308 | */ |
309 | int available_spus; |
310 | |
311 | node = (node < MAX_NUMNODES) ? node : 0; |
312 | if (!node_allowed(ctx, node)) |
313 | continue; |
314 | |
315 | available_spus = 0; |
316 | mutex_lock(&cbe_spu_info[node].list_mutex); |
317 | list_for_each_entry(spu, &cbe_spu_info[node].spus, cbe_list) { |
318 | if (spu->ctx && spu->ctx->gang && !spu->ctx->aff_offset |
319 | && spu->ctx->gang->aff_ref_spu) |
320 | available_spus -= spu->ctx->gang->contexts; |
321 | available_spus++; |
322 | } |
323 | if (available_spus < ctx->gang->contexts) { |
324 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
325 | continue; |
326 | } |
327 | |
328 | list_for_each_entry(spu, &cbe_spu_info[node].spus, cbe_list) { |
329 | if ((!mem_aff || spu->has_mem_affinity) && |
330 | sched_spu(spu)) { |
331 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
332 | return spu; |
333 | } |
334 | } |
335 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
336 | } |
337 | return NULL; |
338 | } |
339 | |
340 | static void aff_set_ref_point_location(struct spu_gang *gang) |
341 | { |
342 | int mem_aff, gs, lowest_offset; |
343 | struct spu_context *tmp, *ctx; |
344 | |
345 | mem_aff = gang->aff_ref_ctx->flags & SPU_CREATE_AFFINITY_MEM; |
346 | lowest_offset = 0; |
347 | gs = 0; |
348 | |
349 | list_for_each_entry(tmp, &gang->aff_list_head, aff_list) |
350 | gs++; |
351 | |
352 | list_for_each_entry_reverse(ctx, &gang->aff_ref_ctx->aff_list, |
353 | aff_list) { |
354 | if (&ctx->aff_list == &gang->aff_list_head) |
355 | break; |
356 | lowest_offset = ctx->aff_offset; |
357 | } |
358 | |
359 | gang->aff_ref_spu = aff_ref_location(ctx: gang->aff_ref_ctx, mem_aff, group_size: gs, |
360 | lowest_offset); |
361 | } |
362 | |
363 | static struct spu *ctx_location(struct spu *ref, int offset, int node) |
364 | { |
365 | struct spu *spu; |
366 | |
367 | spu = NULL; |
368 | if (offset >= 0) { |
369 | list_for_each_entry(spu, ref->aff_list.prev, aff_list) { |
370 | BUG_ON(spu->node != node); |
371 | if (offset == 0) |
372 | break; |
373 | if (sched_spu(spu)) |
374 | offset--; |
375 | } |
376 | } else { |
377 | list_for_each_entry_reverse(spu, ref->aff_list.next, aff_list) { |
378 | BUG_ON(spu->node != node); |
379 | if (offset == 0) |
380 | break; |
381 | if (sched_spu(spu)) |
382 | offset++; |
383 | } |
384 | } |
385 | |
386 | return spu; |
387 | } |
388 | |
389 | /* |
390 | * affinity_check is called each time a context is going to be scheduled. |
391 | * It returns the spu ptr on which the context must run. |
392 | */ |
393 | static int has_affinity(struct spu_context *ctx) |
394 | { |
395 | struct spu_gang *gang = ctx->gang; |
396 | |
397 | if (list_empty(head: &ctx->aff_list)) |
398 | return 0; |
399 | |
400 | if (atomic_read(v: &ctx->gang->aff_sched_count) == 0) |
401 | ctx->gang->aff_ref_spu = NULL; |
402 | |
403 | if (!gang->aff_ref_spu) { |
404 | if (!(gang->aff_flags & AFF_MERGED)) |
405 | aff_merge_remaining_ctxs(gang); |
406 | if (!(gang->aff_flags & AFF_OFFSETS_SET)) |
407 | aff_set_offsets(gang); |
408 | aff_set_ref_point_location(gang); |
409 | } |
410 | |
411 | return gang->aff_ref_spu != NULL; |
412 | } |
413 | |
414 | /** |
415 | * spu_unbind_context - unbind spu context from physical spu |
416 | * @spu: physical spu to unbind from |
417 | * @ctx: context to unbind |
418 | */ |
419 | static void spu_unbind_context(struct spu *spu, struct spu_context *ctx) |
420 | { |
421 | u32 status; |
422 | |
423 | spu_context_trace(spu_unbind_context__enter, ctx, spu); |
424 | |
425 | spuctx_switch_state(ctx, new_state: SPU_UTIL_SYSTEM); |
426 | |
427 | if (spu->ctx->flags & SPU_CREATE_NOSCHED) |
428 | atomic_dec(v: &cbe_spu_info[spu->node].reserved_spus); |
429 | |
430 | if (ctx->gang) |
431 | /* |
432 | * If ctx->gang->aff_sched_count is positive, SPU affinity is |
433 | * being considered in this gang. Using atomic_dec_if_positive |
434 | * allow us to skip an explicit check for affinity in this gang |
435 | */ |
436 | atomic_dec_if_positive(v: &ctx->gang->aff_sched_count); |
437 | |
438 | spu_unmap_mappings(ctx); |
439 | spu_save(prev: &ctx->csa, spu); |
440 | spu_switch_log_notify(spu, ctx, type: SWITCH_LOG_STOP, val: 0); |
441 | |
442 | spin_lock_irq(lock: &spu->register_lock); |
443 | spu->timestamp = jiffies; |
444 | ctx->state = SPU_STATE_SAVED; |
445 | spu->ibox_callback = NULL; |
446 | spu->wbox_callback = NULL; |
447 | spu->stop_callback = NULL; |
448 | spu->mfc_callback = NULL; |
449 | spu->pid = 0; |
450 | spu->tgid = 0; |
451 | ctx->ops = &spu_backing_ops; |
452 | spu->flags = 0; |
453 | spu->ctx = NULL; |
454 | spin_unlock_irq(lock: &spu->register_lock); |
455 | |
456 | spu_associate_mm(spu, NULL); |
457 | |
458 | ctx->stats.slb_flt += |
459 | (spu->stats.slb_flt - ctx->stats.slb_flt_base); |
460 | ctx->stats.class2_intr += |
461 | (spu->stats.class2_intr - ctx->stats.class2_intr_base); |
462 | |
463 | /* This maps the underlying spu state to idle */ |
464 | spuctx_switch_state(ctx, new_state: SPU_UTIL_IDLE_LOADED); |
465 | ctx->spu = NULL; |
466 | |
467 | if (spu_stopped(ctx, stat: &status)) |
468 | wake_up_all(&ctx->stop_wq); |
469 | } |
470 | |
471 | /** |
472 | * spu_add_to_rq - add a context to the runqueue |
473 | * @ctx: context to add |
474 | */ |
475 | static void __spu_add_to_rq(struct spu_context *ctx) |
476 | { |
477 | /* |
478 | * Unfortunately this code path can be called from multiple threads |
479 | * on behalf of a single context due to the way the problem state |
480 | * mmap support works. |
481 | * |
482 | * Fortunately we need to wake up all these threads at the same time |
483 | * and can simply skip the runqueue addition for every but the first |
484 | * thread getting into this codepath. |
485 | * |
486 | * It's still quite hacky, and long-term we should proxy all other |
487 | * threads through the owner thread so that spu_run is in control |
488 | * of all the scheduling activity for a given context. |
489 | */ |
490 | if (list_empty(head: &ctx->rq)) { |
491 | list_add_tail(new: &ctx->rq, head: &spu_prio->runq[ctx->prio]); |
492 | set_bit(nr: ctx->prio, addr: spu_prio->bitmap); |
493 | if (!spu_prio->nr_waiting++) |
494 | mod_timer(timer: &spusched_timer, expires: jiffies + SPUSCHED_TICK); |
495 | } |
496 | } |
497 | |
498 | static void spu_add_to_rq(struct spu_context *ctx) |
499 | { |
500 | spin_lock(lock: &spu_prio->runq_lock); |
501 | __spu_add_to_rq(ctx); |
502 | spin_unlock(lock: &spu_prio->runq_lock); |
503 | } |
504 | |
505 | static void __spu_del_from_rq(struct spu_context *ctx) |
506 | { |
507 | int prio = ctx->prio; |
508 | |
509 | if (!list_empty(head: &ctx->rq)) { |
510 | if (!--spu_prio->nr_waiting) |
511 | del_timer(timer: &spusched_timer); |
512 | list_del_init(entry: &ctx->rq); |
513 | |
514 | if (list_empty(head: &spu_prio->runq[prio])) |
515 | clear_bit(nr: prio, addr: spu_prio->bitmap); |
516 | } |
517 | } |
518 | |
519 | void spu_del_from_rq(struct spu_context *ctx) |
520 | { |
521 | spin_lock(lock: &spu_prio->runq_lock); |
522 | __spu_del_from_rq(ctx); |
523 | spin_unlock(lock: &spu_prio->runq_lock); |
524 | } |
525 | |
526 | static void spu_prio_wait(struct spu_context *ctx) |
527 | { |
528 | DEFINE_WAIT(wait); |
529 | |
530 | /* |
531 | * The caller must explicitly wait for a context to be loaded |
532 | * if the nosched flag is set. If NOSCHED is not set, the caller |
533 | * queues the context and waits for an spu event or error. |
534 | */ |
535 | BUG_ON(!(ctx->flags & SPU_CREATE_NOSCHED)); |
536 | |
537 | spin_lock(lock: &spu_prio->runq_lock); |
538 | prepare_to_wait_exclusive(wq_head: &ctx->stop_wq, wq_entry: &wait, TASK_INTERRUPTIBLE); |
539 | if (!signal_pending(current)) { |
540 | __spu_add_to_rq(ctx); |
541 | spin_unlock(lock: &spu_prio->runq_lock); |
542 | mutex_unlock(lock: &ctx->state_mutex); |
543 | schedule(); |
544 | mutex_lock(&ctx->state_mutex); |
545 | spin_lock(lock: &spu_prio->runq_lock); |
546 | __spu_del_from_rq(ctx); |
547 | } |
548 | spin_unlock(lock: &spu_prio->runq_lock); |
549 | __set_current_state(TASK_RUNNING); |
550 | remove_wait_queue(wq_head: &ctx->stop_wq, wq_entry: &wait); |
551 | } |
552 | |
553 | static struct spu *spu_get_idle(struct spu_context *ctx) |
554 | { |
555 | struct spu *spu, *aff_ref_spu; |
556 | int node, n; |
557 | |
558 | spu_context_nospu_trace(spu_get_idle__enter, ctx); |
559 | |
560 | if (ctx->gang) { |
561 | mutex_lock(&ctx->gang->aff_mutex); |
562 | if (has_affinity(ctx)) { |
563 | aff_ref_spu = ctx->gang->aff_ref_spu; |
564 | atomic_inc(v: &ctx->gang->aff_sched_count); |
565 | mutex_unlock(lock: &ctx->gang->aff_mutex); |
566 | node = aff_ref_spu->node; |
567 | |
568 | mutex_lock(&cbe_spu_info[node].list_mutex); |
569 | spu = ctx_location(ref: aff_ref_spu, offset: ctx->aff_offset, node); |
570 | if (spu && spu->alloc_state == SPU_FREE) |
571 | goto found; |
572 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
573 | |
574 | atomic_dec(v: &ctx->gang->aff_sched_count); |
575 | goto not_found; |
576 | } |
577 | mutex_unlock(lock: &ctx->gang->aff_mutex); |
578 | } |
579 | node = cpu_to_node(raw_smp_processor_id()); |
580 | for (n = 0; n < MAX_NUMNODES; n++, node++) { |
581 | node = (node < MAX_NUMNODES) ? node : 0; |
582 | if (!node_allowed(ctx, node)) |
583 | continue; |
584 | |
585 | mutex_lock(&cbe_spu_info[node].list_mutex); |
586 | list_for_each_entry(spu, &cbe_spu_info[node].spus, cbe_list) { |
587 | if (spu->alloc_state == SPU_FREE) |
588 | goto found; |
589 | } |
590 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
591 | } |
592 | |
593 | not_found: |
594 | spu_context_nospu_trace(spu_get_idle__not_found, ctx); |
595 | return NULL; |
596 | |
597 | found: |
598 | spu->alloc_state = SPU_USED; |
599 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
600 | spu_context_trace(spu_get_idle__found, ctx, spu); |
601 | spu_init_channels(spu); |
602 | return spu; |
603 | } |
604 | |
605 | /** |
606 | * find_victim - find a lower priority context to preempt |
607 | * @ctx: candidate context for running |
608 | * |
609 | * Returns the freed physical spu to run the new context on. |
610 | */ |
611 | static struct spu *find_victim(struct spu_context *ctx) |
612 | { |
613 | struct spu_context *victim = NULL; |
614 | struct spu *spu; |
615 | int node, n; |
616 | |
617 | spu_context_nospu_trace(spu_find_victim__enter, ctx); |
618 | |
619 | /* |
620 | * Look for a possible preemption candidate on the local node first. |
621 | * If there is no candidate look at the other nodes. This isn't |
622 | * exactly fair, but so far the whole spu scheduler tries to keep |
623 | * a strong node affinity. We might want to fine-tune this in |
624 | * the future. |
625 | */ |
626 | restart: |
627 | node = cpu_to_node(raw_smp_processor_id()); |
628 | for (n = 0; n < MAX_NUMNODES; n++, node++) { |
629 | node = (node < MAX_NUMNODES) ? node : 0; |
630 | if (!node_allowed(ctx, node)) |
631 | continue; |
632 | |
633 | mutex_lock(&cbe_spu_info[node].list_mutex); |
634 | list_for_each_entry(spu, &cbe_spu_info[node].spus, cbe_list) { |
635 | struct spu_context *tmp = spu->ctx; |
636 | |
637 | if (tmp && tmp->prio > ctx->prio && |
638 | !(tmp->flags & SPU_CREATE_NOSCHED) && |
639 | (!victim || tmp->prio > victim->prio)) { |
640 | victim = spu->ctx; |
641 | } |
642 | } |
643 | if (victim) |
644 | get_spu_context(ctx: victim); |
645 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
646 | |
647 | if (victim) { |
648 | /* |
649 | * This nests ctx->state_mutex, but we always lock |
650 | * higher priority contexts before lower priority |
651 | * ones, so this is safe until we introduce |
652 | * priority inheritance schemes. |
653 | * |
654 | * XXX if the highest priority context is locked, |
655 | * this can loop a long time. Might be better to |
656 | * look at another context or give up after X retries. |
657 | */ |
658 | if (!mutex_trylock(lock: &victim->state_mutex)) { |
659 | put_spu_context(ctx: victim); |
660 | victim = NULL; |
661 | goto restart; |
662 | } |
663 | |
664 | spu = victim->spu; |
665 | if (!spu || victim->prio <= ctx->prio) { |
666 | /* |
667 | * This race can happen because we've dropped |
668 | * the active list mutex. Not a problem, just |
669 | * restart the search. |
670 | */ |
671 | mutex_unlock(lock: &victim->state_mutex); |
672 | put_spu_context(ctx: victim); |
673 | victim = NULL; |
674 | goto restart; |
675 | } |
676 | |
677 | spu_context_trace(__spu_deactivate__unload, ctx, spu); |
678 | |
679 | mutex_lock(&cbe_spu_info[node].list_mutex); |
680 | cbe_spu_info[node].nr_active--; |
681 | spu_unbind_context(spu, ctx: victim); |
682 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
683 | |
684 | victim->stats.invol_ctx_switch++; |
685 | spu->stats.invol_ctx_switch++; |
686 | if (test_bit(SPU_SCHED_SPU_RUN, &victim->sched_flags)) |
687 | spu_add_to_rq(ctx: victim); |
688 | |
689 | mutex_unlock(lock: &victim->state_mutex); |
690 | put_spu_context(ctx: victim); |
691 | |
692 | return spu; |
693 | } |
694 | } |
695 | |
696 | return NULL; |
697 | } |
698 | |
699 | static void __spu_schedule(struct spu *spu, struct spu_context *ctx) |
700 | { |
701 | int node = spu->node; |
702 | int success = 0; |
703 | |
704 | spu_set_timeslice(ctx); |
705 | |
706 | mutex_lock(&cbe_spu_info[node].list_mutex); |
707 | if (spu->ctx == NULL) { |
708 | spu_bind_context(spu, ctx); |
709 | cbe_spu_info[node].nr_active++; |
710 | spu->alloc_state = SPU_USED; |
711 | success = 1; |
712 | } |
713 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
714 | |
715 | if (success) |
716 | wake_up_all(&ctx->run_wq); |
717 | else |
718 | spu_add_to_rq(ctx); |
719 | } |
720 | |
721 | static void spu_schedule(struct spu *spu, struct spu_context *ctx) |
722 | { |
723 | /* not a candidate for interruptible because it's called either |
724 | from the scheduler thread or from spu_deactivate */ |
725 | mutex_lock(&ctx->state_mutex); |
726 | if (ctx->state == SPU_STATE_SAVED) |
727 | __spu_schedule(spu, ctx); |
728 | spu_release(ctx); |
729 | } |
730 | |
731 | /** |
732 | * spu_unschedule - remove a context from a spu, and possibly release it. |
733 | * @spu: The SPU to unschedule from |
734 | * @ctx: The context currently scheduled on the SPU |
735 | * @free_spu Whether to free the SPU for other contexts |
736 | * |
737 | * Unbinds the context @ctx from the SPU @spu. If @free_spu is non-zero, the |
738 | * SPU is made available for other contexts (ie, may be returned by |
739 | * spu_get_idle). If this is zero, the caller is expected to schedule another |
740 | * context to this spu. |
741 | * |
742 | * Should be called with ctx->state_mutex held. |
743 | */ |
744 | static void spu_unschedule(struct spu *spu, struct spu_context *ctx, |
745 | int free_spu) |
746 | { |
747 | int node = spu->node; |
748 | |
749 | mutex_lock(&cbe_spu_info[node].list_mutex); |
750 | cbe_spu_info[node].nr_active--; |
751 | if (free_spu) |
752 | spu->alloc_state = SPU_FREE; |
753 | spu_unbind_context(spu, ctx); |
754 | ctx->stats.invol_ctx_switch++; |
755 | spu->stats.invol_ctx_switch++; |
756 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
757 | } |
758 | |
759 | /** |
760 | * spu_activate - find a free spu for a context and execute it |
761 | * @ctx: spu context to schedule |
762 | * @flags: flags (currently ignored) |
763 | * |
764 | * Tries to find a free spu to run @ctx. If no free spu is available |
765 | * add the context to the runqueue so it gets woken up once an spu |
766 | * is available. |
767 | */ |
768 | int spu_activate(struct spu_context *ctx, unsigned long flags) |
769 | { |
770 | struct spu *spu; |
771 | |
772 | /* |
773 | * If there are multiple threads waiting for a single context |
774 | * only one actually binds the context while the others will |
775 | * only be able to acquire the state_mutex once the context |
776 | * already is in runnable state. |
777 | */ |
778 | if (ctx->spu) |
779 | return 0; |
780 | |
781 | spu_activate_top: |
782 | if (signal_pending(current)) |
783 | return -ERESTARTSYS; |
784 | |
785 | spu = spu_get_idle(ctx); |
786 | /* |
787 | * If this is a realtime thread we try to get it running by |
788 | * preempting a lower priority thread. |
789 | */ |
790 | if (!spu && rt_prio(prio: ctx->prio)) |
791 | spu = find_victim(ctx); |
792 | if (spu) { |
793 | unsigned long runcntl; |
794 | |
795 | runcntl = ctx->ops->runcntl_read(ctx); |
796 | __spu_schedule(spu, ctx); |
797 | if (runcntl & SPU_RUNCNTL_RUNNABLE) |
798 | spuctx_switch_state(ctx, SPU_UTIL_USER); |
799 | |
800 | return 0; |
801 | } |
802 | |
803 | if (ctx->flags & SPU_CREATE_NOSCHED) { |
804 | spu_prio_wait(ctx); |
805 | goto spu_activate_top; |
806 | } |
807 | |
808 | spu_add_to_rq(ctx); |
809 | |
810 | return 0; |
811 | } |
812 | |
813 | /** |
814 | * grab_runnable_context - try to find a runnable context |
815 | * |
816 | * Remove the highest priority context on the runqueue and return it |
817 | * to the caller. Returns %NULL if no runnable context was found. |
818 | */ |
819 | static struct spu_context *grab_runnable_context(int prio, int node) |
820 | { |
821 | struct spu_context *ctx; |
822 | int best; |
823 | |
824 | spin_lock(lock: &spu_prio->runq_lock); |
825 | best = find_first_bit(addr: spu_prio->bitmap, size: prio); |
826 | while (best < prio) { |
827 | struct list_head *rq = &spu_prio->runq[best]; |
828 | |
829 | list_for_each_entry(ctx, rq, rq) { |
830 | /* XXX(hch): check for affinity here as well */ |
831 | if (__node_allowed(ctx, node)) { |
832 | __spu_del_from_rq(ctx); |
833 | goto found; |
834 | } |
835 | } |
836 | best++; |
837 | } |
838 | ctx = NULL; |
839 | found: |
840 | spin_unlock(lock: &spu_prio->runq_lock); |
841 | return ctx; |
842 | } |
843 | |
844 | static int __spu_deactivate(struct spu_context *ctx, int force, int max_prio) |
845 | { |
846 | struct spu *spu = ctx->spu; |
847 | struct spu_context *new = NULL; |
848 | |
849 | if (spu) { |
850 | new = grab_runnable_context(prio: max_prio, node: spu->node); |
851 | if (new || force) { |
852 | spu_unschedule(spu, ctx, free_spu: new == NULL); |
853 | if (new) { |
854 | if (new->flags & SPU_CREATE_NOSCHED) |
855 | wake_up(&new->stop_wq); |
856 | else { |
857 | spu_release(ctx); |
858 | spu_schedule(spu, ctx: new); |
859 | /* this one can't easily be made |
860 | interruptible */ |
861 | mutex_lock(&ctx->state_mutex); |
862 | } |
863 | } |
864 | } |
865 | } |
866 | |
867 | return new != NULL; |
868 | } |
869 | |
870 | /** |
871 | * spu_deactivate - unbind a context from it's physical spu |
872 | * @ctx: spu context to unbind |
873 | * |
874 | * Unbind @ctx from the physical spu it is running on and schedule |
875 | * the highest priority context to run on the freed physical spu. |
876 | */ |
877 | void spu_deactivate(struct spu_context *ctx) |
878 | { |
879 | spu_context_nospu_trace(spu_deactivate__enter, ctx); |
880 | __spu_deactivate(ctx, force: 1, MAX_PRIO); |
881 | } |
882 | |
883 | /** |
884 | * spu_yield - yield a physical spu if others are waiting |
885 | * @ctx: spu context to yield |
886 | * |
887 | * Check if there is a higher priority context waiting and if yes |
888 | * unbind @ctx from the physical spu and schedule the highest |
889 | * priority context to run on the freed physical spu instead. |
890 | */ |
891 | void spu_yield(struct spu_context *ctx) |
892 | { |
893 | spu_context_nospu_trace(spu_yield__enter, ctx); |
894 | if (!(ctx->flags & SPU_CREATE_NOSCHED)) { |
895 | mutex_lock(&ctx->state_mutex); |
896 | __spu_deactivate(ctx, force: 0, MAX_PRIO); |
897 | mutex_unlock(lock: &ctx->state_mutex); |
898 | } |
899 | } |
900 | |
901 | static noinline void spusched_tick(struct spu_context *ctx) |
902 | { |
903 | struct spu_context *new = NULL; |
904 | struct spu *spu = NULL; |
905 | |
906 | if (spu_acquire(ctx)) |
907 | BUG(); /* a kernel thread never has signals pending */ |
908 | |
909 | if (ctx->state != SPU_STATE_RUNNABLE) |
910 | goto out; |
911 | if (ctx->flags & SPU_CREATE_NOSCHED) |
912 | goto out; |
913 | if (ctx->policy == SCHED_FIFO) |
914 | goto out; |
915 | |
916 | if (--ctx->time_slice && test_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags)) |
917 | goto out; |
918 | |
919 | spu = ctx->spu; |
920 | |
921 | spu_context_trace(spusched_tick__preempt, ctx, spu); |
922 | |
923 | new = grab_runnable_context(prio: ctx->prio + 1, node: spu->node); |
924 | if (new) { |
925 | spu_unschedule(spu, ctx, free_spu: 0); |
926 | if (test_bit(SPU_SCHED_SPU_RUN, &ctx->sched_flags)) |
927 | spu_add_to_rq(ctx); |
928 | } else { |
929 | spu_context_nospu_trace(spusched_tick__newslice, ctx); |
930 | if (!ctx->time_slice) |
931 | ctx->time_slice++; |
932 | } |
933 | out: |
934 | spu_release(ctx); |
935 | |
936 | if (new) |
937 | spu_schedule(spu, ctx: new); |
938 | } |
939 | |
940 | /** |
941 | * count_active_contexts - count nr of active tasks |
942 | * |
943 | * Return the number of tasks currently running or waiting to run. |
944 | * |
945 | * Note that we don't take runq_lock / list_mutex here. Reading |
946 | * a single 32bit value is atomic on powerpc, and we don't care |
947 | * about memory ordering issues here. |
948 | */ |
949 | static unsigned long count_active_contexts(void) |
950 | { |
951 | int nr_active = 0, node; |
952 | |
953 | for (node = 0; node < MAX_NUMNODES; node++) |
954 | nr_active += cbe_spu_info[node].nr_active; |
955 | nr_active += spu_prio->nr_waiting; |
956 | |
957 | return nr_active; |
958 | } |
959 | |
960 | /** |
961 | * spu_calc_load - update the avenrun load estimates. |
962 | * |
963 | * No locking against reading these values from userspace, as for |
964 | * the CPU loadavg code. |
965 | */ |
966 | static void spu_calc_load(void) |
967 | { |
968 | unsigned long active_tasks; /* fixed-point */ |
969 | |
970 | active_tasks = count_active_contexts() * FIXED_1; |
971 | spu_avenrun[0] = calc_load(load: spu_avenrun[0], EXP_1, active: active_tasks); |
972 | spu_avenrun[1] = calc_load(load: spu_avenrun[1], EXP_5, active: active_tasks); |
973 | spu_avenrun[2] = calc_load(load: spu_avenrun[2], EXP_15, active: active_tasks); |
974 | } |
975 | |
976 | static void spusched_wake(struct timer_list *unused) |
977 | { |
978 | mod_timer(timer: &spusched_timer, expires: jiffies + SPUSCHED_TICK); |
979 | wake_up_process(tsk: spusched_task); |
980 | } |
981 | |
982 | static void spuloadavg_wake(struct timer_list *unused) |
983 | { |
984 | mod_timer(timer: &spuloadavg_timer, expires: jiffies + LOAD_FREQ); |
985 | spu_calc_load(); |
986 | } |
987 | |
988 | static int spusched_thread(void *unused) |
989 | { |
990 | struct spu *spu; |
991 | int node; |
992 | |
993 | while (!kthread_should_stop()) { |
994 | set_current_state(TASK_INTERRUPTIBLE); |
995 | schedule(); |
996 | for (node = 0; node < MAX_NUMNODES; node++) { |
997 | struct mutex *mtx = &cbe_spu_info[node].list_mutex; |
998 | |
999 | mutex_lock(mtx); |
1000 | list_for_each_entry(spu, &cbe_spu_info[node].spus, |
1001 | cbe_list) { |
1002 | struct spu_context *ctx = spu->ctx; |
1003 | |
1004 | if (ctx) { |
1005 | get_spu_context(ctx); |
1006 | mutex_unlock(mtx); |
1007 | spusched_tick(ctx); |
1008 | mutex_lock(mtx); |
1009 | put_spu_context(ctx); |
1010 | } |
1011 | } |
1012 | mutex_unlock(lock: mtx); |
1013 | } |
1014 | } |
1015 | |
1016 | return 0; |
1017 | } |
1018 | |
1019 | void spuctx_switch_state(struct spu_context *ctx, |
1020 | enum spu_utilization_state new_state) |
1021 | { |
1022 | unsigned long long curtime; |
1023 | signed long long delta; |
1024 | struct spu *spu; |
1025 | enum spu_utilization_state old_state; |
1026 | int node; |
1027 | |
1028 | curtime = ktime_get_ns(); |
1029 | delta = curtime - ctx->stats.tstamp; |
1030 | |
1031 | WARN_ON(!mutex_is_locked(&ctx->state_mutex)); |
1032 | WARN_ON(delta < 0); |
1033 | |
1034 | spu = ctx->spu; |
1035 | old_state = ctx->stats.util_state; |
1036 | ctx->stats.util_state = new_state; |
1037 | ctx->stats.tstamp = curtime; |
1038 | |
1039 | /* |
1040 | * Update the physical SPU utilization statistics. |
1041 | */ |
1042 | if (spu) { |
1043 | ctx->stats.times[old_state] += delta; |
1044 | spu->stats.times[old_state] += delta; |
1045 | spu->stats.util_state = new_state; |
1046 | spu->stats.tstamp = curtime; |
1047 | node = spu->node; |
1048 | if (old_state == SPU_UTIL_USER) |
1049 | atomic_dec(&cbe_spu_info[node].busy_spus); |
1050 | if (new_state == SPU_UTIL_USER) |
1051 | atomic_inc(&cbe_spu_info[node].busy_spus); |
1052 | } |
1053 | } |
1054 | |
1055 | #ifdef CONFIG_PROC_FS |
1056 | static int show_spu_loadavg(struct seq_file *s, void *private) |
1057 | { |
1058 | int a, b, c; |
1059 | |
1060 | a = spu_avenrun[0] + (FIXED_1/200); |
1061 | b = spu_avenrun[1] + (FIXED_1/200); |
1062 | c = spu_avenrun[2] + (FIXED_1/200); |
1063 | |
1064 | /* |
1065 | * Note that last_pid doesn't really make much sense for the |
1066 | * SPU loadavg (it even seems very odd on the CPU side...), |
1067 | * but we include it here to have a 100% compatible interface. |
1068 | */ |
1069 | seq_printf(m: s, fmt: "%d.%02d %d.%02d %d.%02d %ld/%d %d\n" , |
1070 | LOAD_INT(a), LOAD_FRAC(a), |
1071 | LOAD_INT(b), LOAD_FRAC(b), |
1072 | LOAD_INT(c), LOAD_FRAC(c), |
1073 | count_active_contexts(), |
1074 | atomic_read(v: &nr_spu_contexts), |
1075 | idr_get_cursor(idr: &task_active_pid_ns(current)->idr) - 1); |
1076 | return 0; |
1077 | } |
1078 | #endif |
1079 | |
1080 | int __init spu_sched_init(void) |
1081 | { |
1082 | struct proc_dir_entry *entry; |
1083 | int err = -ENOMEM, i; |
1084 | |
1085 | spu_prio = kzalloc(size: sizeof(struct spu_prio_array), GFP_KERNEL); |
1086 | if (!spu_prio) |
1087 | goto out; |
1088 | |
1089 | for (i = 0; i < MAX_PRIO; i++) { |
1090 | INIT_LIST_HEAD(list: &spu_prio->runq[i]); |
1091 | __clear_bit(i, spu_prio->bitmap); |
1092 | } |
1093 | spin_lock_init(&spu_prio->runq_lock); |
1094 | |
1095 | timer_setup(&spusched_timer, spusched_wake, 0); |
1096 | timer_setup(&spuloadavg_timer, spuloadavg_wake, 0); |
1097 | |
1098 | spusched_task = kthread_run(spusched_thread, NULL, "spusched" ); |
1099 | if (IS_ERR(ptr: spusched_task)) { |
1100 | err = PTR_ERR(ptr: spusched_task); |
1101 | goto out_free_spu_prio; |
1102 | } |
1103 | |
1104 | mod_timer(timer: &spuloadavg_timer, expires: 0); |
1105 | |
1106 | entry = proc_create_single("spu_loadavg" , 0, NULL, show_spu_loadavg); |
1107 | if (!entry) |
1108 | goto out_stop_kthread; |
1109 | |
1110 | pr_debug("spusched: tick: %d, min ticks: %d, default ticks: %d\n" , |
1111 | SPUSCHED_TICK, MIN_SPU_TIMESLICE, DEF_SPU_TIMESLICE); |
1112 | return 0; |
1113 | |
1114 | out_stop_kthread: |
1115 | kthread_stop(k: spusched_task); |
1116 | out_free_spu_prio: |
1117 | kfree(objp: spu_prio); |
1118 | out: |
1119 | return err; |
1120 | } |
1121 | |
1122 | void spu_sched_exit(void) |
1123 | { |
1124 | struct spu *spu; |
1125 | int node; |
1126 | |
1127 | remove_proc_entry("spu_loadavg" , NULL); |
1128 | |
1129 | del_timer_sync(timer: &spusched_timer); |
1130 | del_timer_sync(timer: &spuloadavg_timer); |
1131 | kthread_stop(k: spusched_task); |
1132 | |
1133 | for (node = 0; node < MAX_NUMNODES; node++) { |
1134 | mutex_lock(&cbe_spu_info[node].list_mutex); |
1135 | list_for_each_entry(spu, &cbe_spu_info[node].spus, cbe_list) |
1136 | if (spu->alloc_state != SPU_FREE) |
1137 | spu->alloc_state = SPU_FREE; |
1138 | mutex_unlock(&cbe_spu_info[node].list_mutex); |
1139 | } |
1140 | kfree(objp: spu_prio); |
1141 | } |
1142 | |