1 | /* SPDX-License-Identifier: GPL-2.0-only */ |
2 | /* |
3 | * arch/arm/common/mcpm_head.S -- kernel entry point for multi-cluster PM |
4 | * |
5 | * Created by: Nicolas Pitre, March 2012 |
6 | * Copyright: (C) 2012-2013 Linaro Limited |
7 | * |
8 | * Refer to Documentation/arch/arm/cluster-pm-race-avoidance.rst |
9 | * for details of the synchronisation algorithms used here. |
10 | */ |
11 | |
12 | #include <linux/linkage.h> |
13 | #include <asm/mcpm.h> |
14 | #include <asm/assembler.h> |
15 | |
16 | #include "vlock.h" |
17 | |
18 | .arch armv7-a |
19 | |
20 | .if MCPM_SYNC_CLUSTER_CPUS |
21 | .error "cpus must be the first member of struct mcpm_sync_struct" |
22 | .endif |
23 | |
24 | .macro pr_dbg string |
25 | #if defined(CONFIG_DEBUG_LL) && defined(DEBUG) |
26 | b 1901f |
27 | 1902: .asciz "CPU" |
28 | 1903: .asciz " cluster" |
29 | 1904: .asciz ": \string" |
30 | .align |
31 | 1901: adr r0, 1902b |
32 | bl printascii |
33 | mov r0, r9 |
34 | bl printhex2 |
35 | adr r0, 1903b |
36 | bl printascii |
37 | mov r0, r10 |
38 | bl printhex2 |
39 | adr r0, 1904b |
40 | bl printascii |
41 | #endif |
42 | .endm |
43 | |
44 | .arm |
45 | .align |
46 | |
47 | ENTRY(mcpm_entry_point) |
48 | |
49 | ARM_BE8(setend be) |
50 | THUMB( badr r12, 1f ) |
51 | THUMB( bx r12 ) |
52 | THUMB( .thumb ) |
53 | 1: |
54 | mrc p15, 0, r0, c0, c0, 5 @ MPIDR |
55 | ubfx r9, r0, #0, #8 @ r9 = cpu |
56 | ubfx r10, r0, #8, #8 @ r10 = cluster |
57 | mov r3, #MAX_CPUS_PER_CLUSTER |
58 | mla r4, r3, r10, r9 @ r4 = canonical CPU index |
59 | cmp r4, #(MAX_CPUS_PER_CLUSTER * MAX_NR_CLUSTERS) |
60 | blo 2f |
61 | |
62 | /* We didn't expect this CPU. Try to cheaply make it quiet. */ |
63 | 1: wfi |
64 | wfe |
65 | b 1b |
66 | |
67 | 2: pr_dbg "kernel mcpm_entry_point\n" |
68 | |
69 | /* |
70 | * MMU is off so we need to get to various variables in a |
71 | * position independent way. |
72 | */ |
73 | adr r5, 3f |
74 | ldmia r5, {r0, r6, r7, r8, r11} |
75 | add r0, r5, r0 @ r0 = mcpm_entry_early_pokes |
76 | add r6, r5, r6 @ r6 = mcpm_entry_vectors |
77 | ldr r7, [r5, r7] @ r7 = mcpm_power_up_setup_phys |
78 | add r8, r5, r8 @ r8 = mcpm_sync |
79 | add r11, r5, r11 @ r11 = first_man_locks |
80 | |
81 | @ Perform an early poke, if any |
82 | add r0, r0, r4, lsl #3 |
83 | ldmia r0, {r0, r1} |
84 | teq r0, #0 |
85 | strne r1, [r0] |
86 | |
87 | mov r0, #MCPM_SYNC_CLUSTER_SIZE |
88 | mla r8, r0, r10, r8 @ r8 = sync cluster base |
89 | |
90 | @ Signal that this CPU is coming UP: |
91 | mov r0, #CPU_COMING_UP |
92 | mov r5, #MCPM_SYNC_CPU_SIZE |
93 | mla r5, r9, r5, r8 @ r5 = sync cpu address |
94 | strb r0, [r5] |
95 | |
96 | @ At this point, the cluster cannot unexpectedly enter the GOING_DOWN |
97 | @ state, because there is at least one active CPU (this CPU). |
98 | |
99 | mov r0, #VLOCK_SIZE |
100 | mla r11, r0, r10, r11 @ r11 = cluster first man lock |
101 | mov r0, r11 |
102 | mov r1, r9 @ cpu |
103 | bl vlock_trylock @ implies DMB |
104 | |
105 | cmp r0, #0 @ failed to get the lock? |
106 | bne mcpm_setup_wait @ wait for cluster setup if so |
107 | |
108 | ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] |
109 | cmp r0, #CLUSTER_UP @ cluster already up? |
110 | bne mcpm_setup @ if not, set up the cluster |
111 | |
112 | @ Otherwise, release the first man lock and skip setup: |
113 | mov r0, r11 |
114 | bl vlock_unlock |
115 | b mcpm_setup_complete |
116 | |
117 | mcpm_setup: |
118 | @ Control dependency implies strb not observable before previous ldrb. |
119 | |
120 | @ Signal that the cluster is being brought up: |
121 | mov r0, #INBOUND_COMING_UP |
122 | strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND] |
123 | dmb |
124 | |
125 | @ Any CPU trying to take the cluster into CLUSTER_GOING_DOWN from this |
126 | @ point onwards will observe INBOUND_COMING_UP and abort. |
127 | |
128 | @ Wait for any previously-pending cluster teardown operations to abort |
129 | @ or complete: |
130 | mcpm_teardown_wait: |
131 | ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] |
132 | cmp r0, #CLUSTER_GOING_DOWN |
133 | bne first_man_setup |
134 | wfe |
135 | b mcpm_teardown_wait |
136 | |
137 | first_man_setup: |
138 | dmb |
139 | |
140 | @ If the outbound gave up before teardown started, skip cluster setup: |
141 | |
142 | cmp r0, #CLUSTER_UP |
143 | beq mcpm_setup_leave |
144 | |
145 | @ power_up_setup is now responsible for setting up the cluster: |
146 | |
147 | cmp r7, #0 |
148 | mov r0, #1 @ second (cluster) affinity level |
149 | blxne r7 @ Call power_up_setup if defined |
150 | dmb |
151 | |
152 | mov r0, #CLUSTER_UP |
153 | strb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] |
154 | dmb |
155 | |
156 | mcpm_setup_leave: |
157 | @ Leave the cluster setup critical section: |
158 | |
159 | mov r0, #INBOUND_NOT_COMING_UP |
160 | strb r0, [r8, #MCPM_SYNC_CLUSTER_INBOUND] |
161 | dsb st |
162 | sev |
163 | |
164 | mov r0, r11 |
165 | bl vlock_unlock @ implies DMB |
166 | b mcpm_setup_complete |
167 | |
168 | @ In the contended case, non-first men wait here for cluster setup |
169 | @ to complete: |
170 | mcpm_setup_wait: |
171 | ldrb r0, [r8, #MCPM_SYNC_CLUSTER_CLUSTER] |
172 | cmp r0, #CLUSTER_UP |
173 | wfene |
174 | bne mcpm_setup_wait |
175 | dmb |
176 | |
177 | mcpm_setup_complete: |
178 | @ If a platform-specific CPU setup hook is needed, it is |
179 | @ called from here. |
180 | |
181 | cmp r7, #0 |
182 | mov r0, #0 @ first (CPU) affinity level |
183 | blxne r7 @ Call power_up_setup if defined |
184 | dmb |
185 | |
186 | @ Mark the CPU as up: |
187 | |
188 | mov r0, #CPU_UP |
189 | strb r0, [r5] |
190 | |
191 | @ Observability order of CPU_UP and opening of the gate does not matter. |
192 | |
193 | mcpm_entry_gated: |
194 | ldr r5, [r6, r4, lsl #2] @ r5 = CPU entry vector |
195 | cmp r5, #0 |
196 | wfeeq |
197 | beq mcpm_entry_gated |
198 | dmb |
199 | |
200 | pr_dbg "released\n" |
201 | bx r5 |
202 | |
203 | .align 2 |
204 | |
205 | 3: .word mcpm_entry_early_pokes - . |
206 | .word mcpm_entry_vectors - 3b |
207 | .word mcpm_power_up_setup_phys - 3b |
208 | .word mcpm_sync - 3b |
209 | .word first_man_locks - 3b |
210 | |
211 | ENDPROC(mcpm_entry_point) |
212 | |
213 | .bss |
214 | |
215 | .align CACHE_WRITEBACK_ORDER |
216 | .type first_man_locks, #object |
217 | first_man_locks: |
218 | .space VLOCK_SIZE * MAX_NR_CLUSTERS |
219 | .align CACHE_WRITEBACK_ORDER |
220 | |
221 | .type mcpm_entry_vectors, #object |
222 | ENTRY(mcpm_entry_vectors) |
223 | .space 4 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER |
224 | |
225 | .type mcpm_entry_early_pokes, #object |
226 | ENTRY(mcpm_entry_early_pokes) |
227 | .space 8 * MAX_NR_CLUSTERS * MAX_CPUS_PER_CLUSTER |
228 | |
229 | .type mcpm_power_up_setup_phys, #object |
230 | ENTRY(mcpm_power_up_setup_phys) |
231 | .space 4 @ set by mcpm_sync_init() |
232 | |