1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * c 2001 PPC 64 Team, IBM Corp |
4 | */ |
5 | |
6 | #include <linux/smp.h> |
7 | #include <linux/export.h> |
8 | #include <linux/memblock.h> |
9 | #include <linux/sched/task.h> |
10 | #include <linux/numa.h> |
11 | #include <linux/pgtable.h> |
12 | |
13 | #include <asm/lppaca.h> |
14 | #include <asm/paca.h> |
15 | #include <asm/sections.h> |
16 | #include <asm/kexec.h> |
17 | #include <asm/svm.h> |
18 | #include <asm/ultravisor.h> |
19 | |
20 | #include "setup.h" |
21 | |
22 | #ifndef CONFIG_SMP |
23 | #define boot_cpuid 0 |
24 | #endif |
25 | |
26 | static void *__init alloc_paca_data(unsigned long size, unsigned long align, |
27 | unsigned long limit, int cpu) |
28 | { |
29 | void *ptr; |
30 | int nid; |
31 | |
32 | /* |
33 | * boot_cpuid paca is allocated very early before cpu_to_node is up. |
34 | * Set bottom-up mode, because the boot CPU should be on node-0, |
35 | * which will put its paca in the right place. |
36 | */ |
37 | if (cpu == boot_cpuid) { |
38 | nid = NUMA_NO_NODE; |
39 | memblock_set_bottom_up(enable: true); |
40 | } else { |
41 | nid = early_cpu_to_node(cpu); |
42 | } |
43 | |
44 | ptr = memblock_alloc_try_nid(size, align, MEMBLOCK_LOW_LIMIT, |
45 | max_addr: limit, nid); |
46 | if (!ptr) |
47 | panic(fmt: "cannot allocate paca data" ); |
48 | |
49 | if (cpu == boot_cpuid) |
50 | memblock_set_bottom_up(enable: false); |
51 | |
52 | return ptr; |
53 | } |
54 | |
55 | #ifdef CONFIG_PPC_PSERIES |
56 | |
57 | #define LPPACA_SIZE 0x400 |
58 | |
59 | static void *__init alloc_shared_lppaca(unsigned long size, unsigned long limit, |
60 | int cpu) |
61 | { |
62 | size_t shared_lppaca_total_size = PAGE_ALIGN(nr_cpu_ids * LPPACA_SIZE); |
63 | static unsigned long shared_lppaca_size; |
64 | static void *shared_lppaca; |
65 | void *ptr; |
66 | |
67 | if (!shared_lppaca) { |
68 | memblock_set_bottom_up(true); |
69 | |
70 | /* |
71 | * See Documentation/arch/powerpc/ultravisor.rst for more details. |
72 | * |
73 | * UV/HV data sharing is in PAGE_SIZE granularity. In order to |
74 | * minimize the number of pages shared, align the allocation to |
75 | * PAGE_SIZE. |
76 | */ |
77 | shared_lppaca = |
78 | memblock_alloc_try_nid(shared_lppaca_total_size, |
79 | PAGE_SIZE, MEMBLOCK_LOW_LIMIT, |
80 | limit, NUMA_NO_NODE); |
81 | if (!shared_lppaca) |
82 | panic("cannot allocate shared data" ); |
83 | |
84 | memblock_set_bottom_up(false); |
85 | uv_share_page(PHYS_PFN(__pa(shared_lppaca)), |
86 | shared_lppaca_total_size >> PAGE_SHIFT); |
87 | } |
88 | |
89 | ptr = shared_lppaca + shared_lppaca_size; |
90 | shared_lppaca_size += size; |
91 | |
92 | /* |
93 | * This is very early in boot, so no harm done if the kernel crashes at |
94 | * this point. |
95 | */ |
96 | BUG_ON(shared_lppaca_size > shared_lppaca_total_size); |
97 | |
98 | return ptr; |
99 | } |
100 | |
101 | /* |
102 | * See asm/lppaca.h for more detail. |
103 | * |
104 | * lppaca structures must must be 1kB in size, L1 cache line aligned, |
105 | * and not cross 4kB boundary. A 1kB size and 1kB alignment will satisfy |
106 | * these requirements. |
107 | */ |
108 | static inline void init_lppaca(struct lppaca *lppaca) |
109 | { |
110 | BUILD_BUG_ON(sizeof(struct lppaca) != 640); |
111 | |
112 | *lppaca = (struct lppaca) { |
113 | .desc = cpu_to_be32(0xd397d781), /* "LpPa" */ |
114 | .size = cpu_to_be16(LPPACA_SIZE), |
115 | .fpregs_in_use = 1, |
116 | .slb_count = cpu_to_be16(64), |
117 | .vmxregs_in_use = 0, |
118 | .page_ins = 0, }; |
119 | }; |
120 | |
121 | static struct lppaca * __init new_lppaca(int cpu, unsigned long limit) |
122 | { |
123 | struct lppaca *lp; |
124 | |
125 | BUILD_BUG_ON(sizeof(struct lppaca) > LPPACA_SIZE); |
126 | |
127 | if (early_cpu_has_feature(CPU_FTR_HVMODE)) |
128 | return NULL; |
129 | |
130 | if (is_secure_guest()) |
131 | lp = alloc_shared_lppaca(LPPACA_SIZE, limit, cpu); |
132 | else |
133 | lp = alloc_paca_data(LPPACA_SIZE, 0x400, limit, cpu); |
134 | |
135 | init_lppaca(lp); |
136 | |
137 | return lp; |
138 | } |
139 | #endif /* CONFIG_PPC_PSERIES */ |
140 | |
141 | #ifdef CONFIG_PPC_64S_HASH_MMU |
142 | /* |
143 | * 3 persistent SLBs are allocated here. The buffer will be zero |
144 | * initially, hence will all be invaild until we actually write them. |
145 | * |
146 | * If you make the number of persistent SLB entries dynamic, please also |
147 | * update PR KVM to flush and restore them accordingly. |
148 | */ |
149 | static struct slb_shadow * __init new_slb_shadow(int cpu, unsigned long limit) |
150 | { |
151 | struct slb_shadow *s; |
152 | |
153 | if (cpu != boot_cpuid) { |
154 | /* |
155 | * Boot CPU comes here before early_radix_enabled |
156 | * is parsed (e.g., for disable_radix). So allocate |
157 | * always and this will be fixed up in free_unused_pacas. |
158 | */ |
159 | if (early_radix_enabled()) |
160 | return NULL; |
161 | } |
162 | |
163 | s = alloc_paca_data(sizeof(*s), L1_CACHE_BYTES, limit, cpu); |
164 | |
165 | s->persistent = cpu_to_be32(SLB_NUM_BOLTED); |
166 | s->buffer_length = cpu_to_be32(sizeof(*s)); |
167 | |
168 | return s; |
169 | } |
170 | #endif /* CONFIG_PPC_64S_HASH_MMU */ |
171 | |
172 | /* The Paca is an array with one entry per processor. Each contains an |
173 | * lppaca, which contains the information shared between the |
174 | * hypervisor and Linux. |
175 | * On systems with hardware multi-threading, there are two threads |
176 | * per processor. The Paca array must contain an entry for each thread. |
177 | * The VPD Areas will give a max logical processors = 2 * max physical |
178 | * processors. The processor VPD array needs one entry per physical |
179 | * processor (not thread). |
180 | */ |
181 | struct paca_struct **paca_ptrs __read_mostly; |
182 | EXPORT_SYMBOL(paca_ptrs); |
183 | |
184 | void __init initialise_paca(struct paca_struct *new_paca, int cpu) |
185 | { |
186 | #ifdef CONFIG_PPC_PSERIES |
187 | new_paca->lppaca_ptr = NULL; |
188 | #endif |
189 | #ifdef CONFIG_PPC_BOOK3E_64 |
190 | new_paca->kernel_pgd = swapper_pg_dir; |
191 | #endif |
192 | new_paca->lock_token = 0x8000; |
193 | new_paca->paca_index = cpu; |
194 | #ifndef CONFIG_PPC_KERNEL_PCREL |
195 | new_paca->kernel_toc = kernel_toc_addr(); |
196 | #endif |
197 | new_paca->kernelbase = (unsigned long) _stext; |
198 | /* Only set MSR:IR/DR when MMU is initialized */ |
199 | new_paca->kernel_msr = MSR_KERNEL & ~(MSR_IR | MSR_DR); |
200 | new_paca->hw_cpu_id = 0xffff; |
201 | new_paca->kexec_state = KEXEC_STATE_NONE; |
202 | new_paca->__current = &init_task; |
203 | new_paca->data_offset = 0xfeeeeeeeeeeeeeeeULL; |
204 | #ifdef CONFIG_PPC_64S_HASH_MMU |
205 | new_paca->slb_shadow_ptr = NULL; |
206 | #endif |
207 | |
208 | #ifdef CONFIG_PPC_BOOK3E_64 |
209 | /* For now -- if we have threads this will be adjusted later */ |
210 | new_paca->tcd_ptr = &new_paca->tcd; |
211 | #endif |
212 | } |
213 | |
214 | /* Put the paca pointer into r13 and SPRG_PACA */ |
215 | void setup_paca(struct paca_struct *new_paca) |
216 | { |
217 | /* Setup r13 */ |
218 | local_paca = new_paca; |
219 | |
220 | #ifdef CONFIG_PPC_BOOK3E_64 |
221 | /* On Book3E, initialize the TLB miss exception frames */ |
222 | mtspr(SPRN_SPRG_TLB_EXFRAME, local_paca->extlb); |
223 | #else |
224 | /* |
225 | * In HV mode, we setup both HPACA and PACA to avoid problems |
226 | * if we do a GET_PACA() before the feature fixups have been |
227 | * applied. |
228 | * |
229 | * Normally you should test against CPU_FTR_HVMODE, but CPU features |
230 | * are not yet set up when we first reach here. |
231 | */ |
232 | if (mfmsr() & MSR_HV) |
233 | mtspr(SPRN_SPRG_HPACA, local_paca); |
234 | #endif |
235 | mtspr(SPRN_SPRG_PACA, local_paca); |
236 | |
237 | } |
238 | |
239 | static int __initdata paca_nr_cpu_ids; |
240 | static int __initdata paca_ptrs_size; |
241 | static int __initdata paca_struct_size; |
242 | |
243 | void __init allocate_paca_ptrs(void) |
244 | { |
245 | paca_nr_cpu_ids = nr_cpu_ids; |
246 | |
247 | paca_ptrs_size = sizeof(struct paca_struct *) * nr_cpu_ids; |
248 | paca_ptrs = memblock_alloc_raw(size: paca_ptrs_size, SMP_CACHE_BYTES); |
249 | if (!paca_ptrs) |
250 | panic(fmt: "Failed to allocate %d bytes for paca pointers\n" , |
251 | paca_ptrs_size); |
252 | |
253 | memset(paca_ptrs, 0x88, paca_ptrs_size); |
254 | } |
255 | |
256 | void __init allocate_paca(int cpu) |
257 | { |
258 | u64 limit; |
259 | struct paca_struct *paca; |
260 | |
261 | BUG_ON(cpu >= paca_nr_cpu_ids); |
262 | |
263 | #ifdef CONFIG_PPC_BOOK3S_64 |
264 | /* |
265 | * We access pacas in real mode, and cannot take SLB faults |
266 | * on them when in virtual mode, so allocate them accordingly. |
267 | */ |
268 | limit = min(ppc64_bolted_size(), ppc64_rma_size); |
269 | #else |
270 | limit = ppc64_rma_size; |
271 | #endif |
272 | |
273 | paca = alloc_paca_data(sizeof(struct paca_struct), L1_CACHE_BYTES, |
274 | limit, cpu); |
275 | paca_ptrs[cpu] = paca; |
276 | |
277 | initialise_paca(new_paca: paca, cpu); |
278 | #ifdef CONFIG_PPC_PSERIES |
279 | paca->lppaca_ptr = new_lppaca(cpu, limit); |
280 | #endif |
281 | #ifdef CONFIG_PPC_64S_HASH_MMU |
282 | paca->slb_shadow_ptr = new_slb_shadow(cpu, limit); |
283 | #endif |
284 | paca_struct_size += sizeof(struct paca_struct); |
285 | } |
286 | |
287 | void __init free_unused_pacas(void) |
288 | { |
289 | int new_ptrs_size; |
290 | |
291 | new_ptrs_size = sizeof(struct paca_struct *) * nr_cpu_ids; |
292 | if (new_ptrs_size < paca_ptrs_size) |
293 | memblock_phys_free(__pa(paca_ptrs) + new_ptrs_size, |
294 | size: paca_ptrs_size - new_ptrs_size); |
295 | |
296 | paca_nr_cpu_ids = nr_cpu_ids; |
297 | paca_ptrs_size = new_ptrs_size; |
298 | |
299 | #ifdef CONFIG_PPC_64S_HASH_MMU |
300 | if (early_radix_enabled()) { |
301 | /* Ugly fixup, see new_slb_shadow() */ |
302 | memblock_phys_free(__pa(paca_ptrs[boot_cpuid]->slb_shadow_ptr), |
303 | sizeof(struct slb_shadow)); |
304 | paca_ptrs[boot_cpuid]->slb_shadow_ptr = NULL; |
305 | } |
306 | #endif |
307 | |
308 | printk(KERN_DEBUG "Allocated %u bytes for %u pacas\n" , |
309 | paca_ptrs_size + paca_struct_size, nr_cpu_ids); |
310 | } |
311 | |
312 | #ifdef CONFIG_PPC_64S_HASH_MMU |
313 | void copy_mm_to_paca(struct mm_struct *mm) |
314 | { |
315 | mm_context_t *context = &mm->context; |
316 | |
317 | VM_BUG_ON(!mm_ctx_slb_addr_limit(context)); |
318 | memcpy(&get_paca()->mm_ctx_low_slices_psize, mm_ctx_low_slices(context), |
319 | LOW_SLICE_ARRAY_SZ); |
320 | memcpy(&get_paca()->mm_ctx_high_slices_psize, mm_ctx_high_slices(context), |
321 | TASK_SLICE_ARRAY_SZ(context)); |
322 | } |
323 | #endif /* CONFIG_PPC_64S_HASH_MMU */ |
324 | |