1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Based on arch/arm/mm/context.c |
4 | * |
5 | * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved. |
6 | * Copyright (C) 2012 ARM Ltd. |
7 | */ |
8 | |
9 | #include <linux/bitfield.h> |
10 | #include <linux/bitops.h> |
11 | #include <linux/sched.h> |
12 | #include <linux/slab.h> |
13 | #include <linux/mm.h> |
14 | |
15 | #include <asm/cpufeature.h> |
16 | #include <asm/mmu_context.h> |
17 | #include <asm/smp.h> |
18 | #include <asm/tlbflush.h> |
19 | |
20 | static u32 asid_bits; |
21 | static DEFINE_RAW_SPINLOCK(cpu_asid_lock); |
22 | |
23 | static atomic64_t asid_generation; |
24 | static unsigned long *asid_map; |
25 | |
26 | static DEFINE_PER_CPU(atomic64_t, active_asids); |
27 | static DEFINE_PER_CPU(u64, reserved_asids); |
28 | static cpumask_t tlb_flush_pending; |
29 | |
30 | static unsigned long max_pinned_asids; |
31 | static unsigned long nr_pinned_asids; |
32 | static unsigned long *pinned_asid_map; |
33 | |
34 | #define ASID_MASK (~GENMASK(asid_bits - 1, 0)) |
35 | #define ASID_FIRST_VERSION (1UL << asid_bits) |
36 | |
37 | #define NUM_USER_ASIDS ASID_FIRST_VERSION |
38 | #define ctxid2asid(asid) ((asid) & ~ASID_MASK) |
39 | #define asid2ctxid(asid, genid) ((asid) | (genid)) |
40 | |
41 | /* Get the ASIDBits supported by the current CPU */ |
42 | static u32 get_cpu_asid_bits(void) |
43 | { |
44 | u32 asid; |
45 | int fld = cpuid_feature_extract_unsigned_field(read_cpuid(ID_AA64MMFR0_EL1), |
46 | ID_AA64MMFR0_EL1_ASIDBITS_SHIFT); |
47 | |
48 | switch (fld) { |
49 | default: |
50 | pr_warn("CPU%d: Unknown ASID size (%d); assuming 8-bit\n" , |
51 | smp_processor_id(), fld); |
52 | fallthrough; |
53 | case ID_AA64MMFR0_EL1_ASIDBITS_8: |
54 | asid = 8; |
55 | break; |
56 | case ID_AA64MMFR0_EL1_ASIDBITS_16: |
57 | asid = 16; |
58 | } |
59 | |
60 | return asid; |
61 | } |
62 | |
63 | /* Check if the current cpu's ASIDBits is compatible with asid_bits */ |
64 | void verify_cpu_asid_bits(void) |
65 | { |
66 | u32 asid = get_cpu_asid_bits(); |
67 | |
68 | if (asid < asid_bits) { |
69 | /* |
70 | * We cannot decrease the ASID size at runtime, so panic if we support |
71 | * fewer ASID bits than the boot CPU. |
72 | */ |
73 | pr_crit("CPU%d: smaller ASID size(%u) than boot CPU (%u)\n" , |
74 | smp_processor_id(), asid, asid_bits); |
75 | cpu_panic_kernel(); |
76 | } |
77 | } |
78 | |
79 | static void set_kpti_asid_bits(unsigned long *map) |
80 | { |
81 | unsigned int len = BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(unsigned long); |
82 | /* |
83 | * In case of KPTI kernel/user ASIDs are allocated in |
84 | * pairs, the bottom bit distinguishes the two: if it |
85 | * is set, then the ASID will map only userspace. Thus |
86 | * mark even as reserved for kernel. |
87 | */ |
88 | memset(map, 0xaa, len); |
89 | } |
90 | |
91 | static void set_reserved_asid_bits(void) |
92 | { |
93 | if (pinned_asid_map) |
94 | bitmap_copy(dst: asid_map, src: pinned_asid_map, NUM_USER_ASIDS); |
95 | else if (arm64_kernel_unmapped_at_el0()) |
96 | set_kpti_asid_bits(asid_map); |
97 | else |
98 | bitmap_clear(map: asid_map, start: 0, NUM_USER_ASIDS); |
99 | } |
100 | |
101 | #define asid_gen_match(asid) \ |
102 | (!(((asid) ^ atomic64_read(&asid_generation)) >> asid_bits)) |
103 | |
104 | static void flush_context(void) |
105 | { |
106 | int i; |
107 | u64 asid; |
108 | |
109 | /* Update the list of reserved ASIDs and the ASID bitmap. */ |
110 | set_reserved_asid_bits(); |
111 | |
112 | for_each_possible_cpu(i) { |
113 | asid = atomic64_xchg_relaxed(v: &per_cpu(active_asids, i), new: 0); |
114 | /* |
115 | * If this CPU has already been through a |
116 | * rollover, but hasn't run another task in |
117 | * the meantime, we must preserve its reserved |
118 | * ASID, as this is the only trace we have of |
119 | * the process it is still running. |
120 | */ |
121 | if (asid == 0) |
122 | asid = per_cpu(reserved_asids, i); |
123 | __set_bit(ctxid2asid(asid), asid_map); |
124 | per_cpu(reserved_asids, i) = asid; |
125 | } |
126 | |
127 | /* |
128 | * Queue a TLB invalidation for each CPU to perform on next |
129 | * context-switch |
130 | */ |
131 | cpumask_setall(dstp: &tlb_flush_pending); |
132 | } |
133 | |
134 | static bool check_update_reserved_asid(u64 asid, u64 newasid) |
135 | { |
136 | int cpu; |
137 | bool hit = false; |
138 | |
139 | /* |
140 | * Iterate over the set of reserved ASIDs looking for a match. |
141 | * If we find one, then we can update our mm to use newasid |
142 | * (i.e. the same ASID in the current generation) but we can't |
143 | * exit the loop early, since we need to ensure that all copies |
144 | * of the old ASID are updated to reflect the mm. Failure to do |
145 | * so could result in us missing the reserved ASID in a future |
146 | * generation. |
147 | */ |
148 | for_each_possible_cpu(cpu) { |
149 | if (per_cpu(reserved_asids, cpu) == asid) { |
150 | hit = true; |
151 | per_cpu(reserved_asids, cpu) = newasid; |
152 | } |
153 | } |
154 | |
155 | return hit; |
156 | } |
157 | |
158 | static u64 new_context(struct mm_struct *mm) |
159 | { |
160 | static u32 cur_idx = 1; |
161 | u64 asid = atomic64_read(v: &mm->context.id); |
162 | u64 generation = atomic64_read(v: &asid_generation); |
163 | |
164 | if (asid != 0) { |
165 | u64 newasid = asid2ctxid(ctxid2asid(asid), generation); |
166 | |
167 | /* |
168 | * If our current ASID was active during a rollover, we |
169 | * can continue to use it and this was just a false alarm. |
170 | */ |
171 | if (check_update_reserved_asid(asid, newasid)) |
172 | return newasid; |
173 | |
174 | /* |
175 | * If it is pinned, we can keep using it. Note that reserved |
176 | * takes priority, because even if it is also pinned, we need to |
177 | * update the generation into the reserved_asids. |
178 | */ |
179 | if (refcount_read(r: &mm->context.pinned)) |
180 | return newasid; |
181 | |
182 | /* |
183 | * We had a valid ASID in a previous life, so try to re-use |
184 | * it if possible. |
185 | */ |
186 | if (!__test_and_set_bit(ctxid2asid(asid), asid_map)) |
187 | return newasid; |
188 | } |
189 | |
190 | /* |
191 | * Allocate a free ASID. If we can't find one, take a note of the |
192 | * currently active ASIDs and mark the TLBs as requiring flushes. We |
193 | * always count from ASID #2 (index 1), as we use ASID #0 when setting |
194 | * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd |
195 | * pairs. |
196 | */ |
197 | asid = find_next_zero_bit(addr: asid_map, NUM_USER_ASIDS, offset: cur_idx); |
198 | if (asid != NUM_USER_ASIDS) |
199 | goto set_asid; |
200 | |
201 | /* We're out of ASIDs, so increment the global generation count */ |
202 | generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION, |
203 | v: &asid_generation); |
204 | flush_context(); |
205 | |
206 | /* We have more ASIDs than CPUs, so this will always succeed */ |
207 | asid = find_next_zero_bit(addr: asid_map, NUM_USER_ASIDS, offset: 1); |
208 | |
209 | set_asid: |
210 | __set_bit(asid, asid_map); |
211 | cur_idx = asid; |
212 | return asid2ctxid(asid, generation); |
213 | } |
214 | |
215 | void check_and_switch_context(struct mm_struct *mm) |
216 | { |
217 | unsigned long flags; |
218 | unsigned int cpu; |
219 | u64 asid, old_active_asid; |
220 | |
221 | if (system_supports_cnp()) |
222 | cpu_set_reserved_ttbr0(); |
223 | |
224 | asid = atomic64_read(v: &mm->context.id); |
225 | |
226 | /* |
227 | * The memory ordering here is subtle. |
228 | * If our active_asids is non-zero and the ASID matches the current |
229 | * generation, then we update the active_asids entry with a relaxed |
230 | * cmpxchg. Racing with a concurrent rollover means that either: |
231 | * |
232 | * - We get a zero back from the cmpxchg and end up waiting on the |
233 | * lock. Taking the lock synchronises with the rollover and so |
234 | * we are forced to see the updated generation. |
235 | * |
236 | * - We get a valid ASID back from the cmpxchg, which means the |
237 | * relaxed xchg in flush_context will treat us as reserved |
238 | * because atomic RmWs are totally ordered for a given location. |
239 | */ |
240 | old_active_asid = atomic64_read(this_cpu_ptr(&active_asids)); |
241 | if (old_active_asid && asid_gen_match(asid) && |
242 | atomic64_cmpxchg_relaxed(this_cpu_ptr(&active_asids), |
243 | old: old_active_asid, new: asid)) |
244 | goto switch_mm_fastpath; |
245 | |
246 | raw_spin_lock_irqsave(&cpu_asid_lock, flags); |
247 | /* Check that our ASID belongs to the current generation. */ |
248 | asid = atomic64_read(v: &mm->context.id); |
249 | if (!asid_gen_match(asid)) { |
250 | asid = new_context(mm); |
251 | atomic64_set(v: &mm->context.id, i: asid); |
252 | } |
253 | |
254 | cpu = smp_processor_id(); |
255 | if (cpumask_test_and_clear_cpu(cpu, cpumask: &tlb_flush_pending)) |
256 | local_flush_tlb_all(); |
257 | |
258 | atomic64_set(this_cpu_ptr(&active_asids), i: asid); |
259 | raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); |
260 | |
261 | switch_mm_fastpath: |
262 | |
263 | arm64_apply_bp_hardening(); |
264 | |
265 | /* |
266 | * Defer TTBR0_EL1 setting for user threads to uaccess_enable() when |
267 | * emulating PAN. |
268 | */ |
269 | if (!system_uses_ttbr0_pan()) |
270 | cpu_switch_mm(mm->pgd, mm); |
271 | } |
272 | |
273 | unsigned long arm64_mm_context_get(struct mm_struct *mm) |
274 | { |
275 | unsigned long flags; |
276 | u64 asid; |
277 | |
278 | if (!pinned_asid_map) |
279 | return 0; |
280 | |
281 | raw_spin_lock_irqsave(&cpu_asid_lock, flags); |
282 | |
283 | asid = atomic64_read(v: &mm->context.id); |
284 | |
285 | if (refcount_inc_not_zero(r: &mm->context.pinned)) |
286 | goto out_unlock; |
287 | |
288 | if (nr_pinned_asids >= max_pinned_asids) { |
289 | asid = 0; |
290 | goto out_unlock; |
291 | } |
292 | |
293 | if (!asid_gen_match(asid)) { |
294 | /* |
295 | * We went through one or more rollover since that ASID was |
296 | * used. Ensure that it is still valid, or generate a new one. |
297 | */ |
298 | asid = new_context(mm); |
299 | atomic64_set(v: &mm->context.id, i: asid); |
300 | } |
301 | |
302 | nr_pinned_asids++; |
303 | __set_bit(ctxid2asid(asid), pinned_asid_map); |
304 | refcount_set(r: &mm->context.pinned, n: 1); |
305 | |
306 | out_unlock: |
307 | raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); |
308 | |
309 | asid = ctxid2asid(asid); |
310 | |
311 | /* Set the equivalent of USER_ASID_BIT */ |
312 | if (asid && arm64_kernel_unmapped_at_el0()) |
313 | asid |= 1; |
314 | |
315 | return asid; |
316 | } |
317 | EXPORT_SYMBOL_GPL(arm64_mm_context_get); |
318 | |
319 | void arm64_mm_context_put(struct mm_struct *mm) |
320 | { |
321 | unsigned long flags; |
322 | u64 asid = atomic64_read(v: &mm->context.id); |
323 | |
324 | if (!pinned_asid_map) |
325 | return; |
326 | |
327 | raw_spin_lock_irqsave(&cpu_asid_lock, flags); |
328 | |
329 | if (refcount_dec_and_test(r: &mm->context.pinned)) { |
330 | __clear_bit(ctxid2asid(asid), pinned_asid_map); |
331 | nr_pinned_asids--; |
332 | } |
333 | |
334 | raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); |
335 | } |
336 | EXPORT_SYMBOL_GPL(arm64_mm_context_put); |
337 | |
338 | /* Errata workaround post TTBRx_EL1 update. */ |
339 | asmlinkage void post_ttbr_update_workaround(void) |
340 | { |
341 | if (!IS_ENABLED(CONFIG_CAVIUM_ERRATUM_27456)) |
342 | return; |
343 | |
344 | asm(ALTERNATIVE("nop; nop; nop" , |
345 | "ic iallu; dsb nsh; isb" , |
346 | ARM64_WORKAROUND_CAVIUM_27456)); |
347 | } |
348 | |
349 | void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm) |
350 | { |
351 | unsigned long ttbr1 = read_sysreg(ttbr1_el1); |
352 | unsigned long asid = ASID(mm); |
353 | unsigned long ttbr0 = phys_to_ttbr(pgd_phys); |
354 | |
355 | /* Skip CNP for the reserved ASID */ |
356 | if (system_supports_cnp() && asid) |
357 | ttbr0 |= TTBR_CNP_BIT; |
358 | |
359 | /* SW PAN needs a copy of the ASID in TTBR0 for entry */ |
360 | if (IS_ENABLED(CONFIG_ARM64_SW_TTBR0_PAN)) |
361 | ttbr0 |= FIELD_PREP(TTBR_ASID_MASK, asid); |
362 | |
363 | /* Set ASID in TTBR1 since TCR.A1 is set */ |
364 | ttbr1 &= ~TTBR_ASID_MASK; |
365 | ttbr1 |= FIELD_PREP(TTBR_ASID_MASK, asid); |
366 | |
367 | cpu_set_reserved_ttbr0_nosync(); |
368 | write_sysreg(ttbr1, ttbr1_el1); |
369 | write_sysreg(ttbr0, ttbr0_el1); |
370 | isb(); |
371 | post_ttbr_update_workaround(); |
372 | } |
373 | |
374 | static int asids_update_limit(void) |
375 | { |
376 | unsigned long num_available_asids = NUM_USER_ASIDS; |
377 | |
378 | if (arm64_kernel_unmapped_at_el0()) { |
379 | num_available_asids /= 2; |
380 | if (pinned_asid_map) |
381 | set_kpti_asid_bits(pinned_asid_map); |
382 | } |
383 | /* |
384 | * Expect allocation after rollover to fail if we don't have at least |
385 | * one more ASID than CPUs. ASID #0 is reserved for init_mm. |
386 | */ |
387 | WARN_ON(num_available_asids - 1 <= num_possible_cpus()); |
388 | pr_info("ASID allocator initialised with %lu entries\n" , |
389 | num_available_asids); |
390 | |
391 | /* |
392 | * There must always be an ASID available after rollover. Ensure that, |
393 | * even if all CPUs have a reserved ASID and the maximum number of ASIDs |
394 | * are pinned, there still is at least one empty slot in the ASID map. |
395 | */ |
396 | max_pinned_asids = num_available_asids - num_possible_cpus() - 2; |
397 | return 0; |
398 | } |
399 | arch_initcall(asids_update_limit); |
400 | |
401 | static int asids_init(void) |
402 | { |
403 | asid_bits = get_cpu_asid_bits(); |
404 | atomic64_set(v: &asid_generation, ASID_FIRST_VERSION); |
405 | asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); |
406 | if (!asid_map) |
407 | panic(fmt: "Failed to allocate bitmap for %lu ASIDs\n" , |
408 | NUM_USER_ASIDS); |
409 | |
410 | pinned_asid_map = bitmap_zalloc(NUM_USER_ASIDS, GFP_KERNEL); |
411 | nr_pinned_asids = 0; |
412 | |
413 | /* |
414 | * We cannot call set_reserved_asid_bits() here because CPU |
415 | * caps are not finalized yet, so it is safer to assume KPTI |
416 | * and reserve kernel ASID's from beginning. |
417 | */ |
418 | if (IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) |
419 | set_kpti_asid_bits(asid_map); |
420 | return 0; |
421 | } |
422 | early_initcall(asids_init); |
423 | |