1// SPDX-License-Identifier: GPL-2.0
2/*
3 * This file contains core hardware tag-based KASAN code.
4 *
5 * Copyright (c) 2020 Google, Inc.
6 * Author: Andrey Konovalov <andreyknvl@google.com>
7 */
8
9#define pr_fmt(fmt) "kasan: " fmt
10
11#include <kunit/visibility.h>
12#include <linux/init.h>
13#include <linux/kasan.h>
14#include <linux/kernel.h>
15#include <linux/memory.h>
16#include <linux/mm.h>
17#include <linux/static_key.h>
18#include <linux/string.h>
19#include <linux/string_choices.h>
20#include <linux/types.h>
21#include <linux/vmalloc.h>
22
23#include "kasan.h"
24
25enum kasan_arg {
26 KASAN_ARG_DEFAULT,
27 KASAN_ARG_OFF,
28 KASAN_ARG_ON,
29};
30
31enum kasan_arg_mode {
32 KASAN_ARG_MODE_DEFAULT,
33 KASAN_ARG_MODE_SYNC,
34 KASAN_ARG_MODE_ASYNC,
35 KASAN_ARG_MODE_ASYMM,
36};
37
38enum kasan_arg_vmalloc {
39 KASAN_ARG_VMALLOC_DEFAULT,
40 KASAN_ARG_VMALLOC_OFF,
41 KASAN_ARG_VMALLOC_ON,
42};
43
44static enum kasan_arg kasan_arg __ro_after_init;
45static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
46static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
47
48/*
49 * Whether the selected mode is synchronous, asynchronous, or asymmetric.
50 * Defaults to KASAN_MODE_SYNC.
51 */
52enum kasan_mode kasan_mode __ro_after_init;
53EXPORT_SYMBOL_GPL(kasan_mode);
54
55/* Whether to enable vmalloc tagging. */
56#ifdef CONFIG_KASAN_VMALLOC
57DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
58#else
59DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc);
60#endif
61EXPORT_SYMBOL_GPL(kasan_flag_vmalloc);
62
63/* Whether to check write accesses only. */
64static bool kasan_flag_write_only = false;
65
66#define PAGE_ALLOC_SAMPLE_DEFAULT 1
67#define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3
68
69/*
70 * Sampling interval of page_alloc allocation (un)poisoning.
71 * Defaults to no sampling.
72 */
73unsigned long kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
74
75/*
76 * Minimum order of page_alloc allocations to be affected by sampling.
77 * The default value is chosen to match both
78 * PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER.
79 */
80unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
81
82DEFINE_PER_CPU(long, kasan_page_alloc_skip);
83
84/* kasan=off/on */
85static int __init early_kasan_flag(char *arg)
86{
87 if (!arg)
88 return -EINVAL;
89
90 if (!strcmp(arg, "off"))
91 kasan_arg = KASAN_ARG_OFF;
92 else if (!strcmp(arg, "on"))
93 kasan_arg = KASAN_ARG_ON;
94 else
95 return -EINVAL;
96
97 return 0;
98}
99early_param("kasan", early_kasan_flag);
100
101/* kasan.mode=sync/async/asymm */
102static int __init early_kasan_mode(char *arg)
103{
104 if (!arg)
105 return -EINVAL;
106
107 if (!strcmp(arg, "sync"))
108 kasan_arg_mode = KASAN_ARG_MODE_SYNC;
109 else if (!strcmp(arg, "async"))
110 kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
111 else if (!strcmp(arg, "asymm"))
112 kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
113 else
114 return -EINVAL;
115
116 return 0;
117}
118early_param("kasan.mode", early_kasan_mode);
119
120/* kasan.vmalloc=off/on */
121static int __init early_kasan_flag_vmalloc(char *arg)
122{
123 if (!arg)
124 return -EINVAL;
125
126 if (!IS_ENABLED(CONFIG_KASAN_VMALLOC))
127 return 0;
128
129 if (!strcmp(arg, "off"))
130 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF;
131 else if (!strcmp(arg, "on"))
132 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON;
133 else
134 return -EINVAL;
135
136 return 0;
137}
138early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
139
140/* kasan.write_only=off/on */
141static int __init early_kasan_flag_write_only(char *arg)
142{
143 if (!arg)
144 return -EINVAL;
145
146 if (!strcmp(arg, "off"))
147 kasan_flag_write_only = false;
148 else if (!strcmp(arg, "on"))
149 kasan_flag_write_only = true;
150 else
151 return -EINVAL;
152
153 return 0;
154}
155early_param("kasan.write_only", early_kasan_flag_write_only);
156
157static inline const char *kasan_mode_info(void)
158{
159 if (kasan_mode == KASAN_MODE_ASYNC)
160 return "async";
161 else if (kasan_mode == KASAN_MODE_ASYMM)
162 return "asymm";
163 else
164 return "sync";
165}
166
167/* kasan.page_alloc.sample=<sampling interval> */
168static int __init early_kasan_flag_page_alloc_sample(char *arg)
169{
170 int rv;
171
172 if (!arg)
173 return -EINVAL;
174
175 rv = kstrtoul(s: arg, base: 0, res: &kasan_page_alloc_sample);
176 if (rv)
177 return rv;
178
179 if (!kasan_page_alloc_sample || kasan_page_alloc_sample > LONG_MAX) {
180 kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
181 return -EINVAL;
182 }
183
184 return 0;
185}
186early_param("kasan.page_alloc.sample", early_kasan_flag_page_alloc_sample);
187
188/* kasan.page_alloc.sample.order=<minimum page order> */
189static int __init early_kasan_flag_page_alloc_sample_order(char *arg)
190{
191 int rv;
192
193 if (!arg)
194 return -EINVAL;
195
196 rv = kstrtouint(s: arg, base: 0, res: &kasan_page_alloc_sample_order);
197 if (rv)
198 return rv;
199
200 if (kasan_page_alloc_sample_order > INT_MAX) {
201 kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
202 return -EINVAL;
203 }
204
205 return 0;
206}
207early_param("kasan.page_alloc.sample.order", early_kasan_flag_page_alloc_sample_order);
208
209/*
210 * kasan_init_hw_tags_cpu() is called for each CPU.
211 * Not marked as __init as a CPU can be hot-plugged after boot.
212 */
213void kasan_init_hw_tags_cpu(void)
214{
215 /*
216 * There's no need to check that the hardware is MTE-capable here,
217 * as this function is only called for MTE-capable hardware.
218 */
219
220 /*
221 * If KASAN is disabled via command line, don't initialize it.
222 * When this function is called, kasan_flag_enabled is not yet
223 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead.
224 */
225 if (kasan_arg == KASAN_ARG_OFF)
226 return;
227
228 /*
229 * Enable async or asymm modes only when explicitly requested
230 * through the command line.
231 */
232 kasan_enable_hw_tags();
233}
234
235/* kasan_init_hw_tags() is called once on boot CPU. */
236void __init kasan_init_hw_tags(void)
237{
238 /* If hardware doesn't support MTE, don't initialize KASAN. */
239 if (!system_supports_mte())
240 return;
241
242 /* If KASAN is disabled via command line, don't initialize it. */
243 if (kasan_arg == KASAN_ARG_OFF)
244 return;
245
246 switch (kasan_arg_mode) {
247 case KASAN_ARG_MODE_DEFAULT:
248 /* Default is specified by kasan_mode definition. */
249 break;
250 case KASAN_ARG_MODE_SYNC:
251 kasan_mode = KASAN_MODE_SYNC;
252 break;
253 case KASAN_ARG_MODE_ASYNC:
254 kasan_mode = KASAN_MODE_ASYNC;
255 break;
256 case KASAN_ARG_MODE_ASYMM:
257 kasan_mode = KASAN_MODE_ASYMM;
258 break;
259 }
260
261 switch (kasan_arg_vmalloc) {
262 case KASAN_ARG_VMALLOC_DEFAULT:
263 /* Default is specified by kasan_flag_vmalloc definition. */
264 break;
265 case KASAN_ARG_VMALLOC_OFF:
266 static_branch_disable(&kasan_flag_vmalloc);
267 break;
268 case KASAN_ARG_VMALLOC_ON:
269 static_branch_enable(&kasan_flag_vmalloc);
270 break;
271 }
272
273 kasan_init_tags();
274
275 /* KASAN is now initialized, enable it. */
276 kasan_enable();
277
278 pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s, write_only=%s)\n",
279 kasan_mode_info(),
280 str_on_off(kasan_vmalloc_enabled()),
281 str_on_off(kasan_stack_collection_enabled()),
282 str_on_off(kasan_flag_write_only));
283}
284
285#ifdef CONFIG_KASAN_VMALLOC
286
287static void unpoison_vmalloc_pages(const void *addr, u8 tag)
288{
289 struct vm_struct *area;
290 int i;
291
292 /*
293 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations
294 * (see the comment in __kasan_unpoison_vmalloc), all of the pages
295 * should belong to a single area.
296 */
297 area = find_vm_area(addr: (void *)addr);
298 if (WARN_ON(!area))
299 return;
300
301 for (i = 0; i < area->nr_pages; i++) {
302 struct page *page = area->pages[i];
303
304 page_kasan_tag_set(page, tag);
305 }
306}
307
308static void init_vmalloc_pages(const void *start, unsigned long size)
309{
310 const void *addr;
311
312 for (addr = start; addr < start + size; addr += PAGE_SIZE) {
313 struct page *page = vmalloc_to_page(addr);
314
315 clear_highpage_kasan_tagged(page);
316 }
317}
318
319void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
320 kasan_vmalloc_flags_t flags)
321{
322 u8 tag;
323 unsigned long redzone_start, redzone_size;
324
325 if (!kasan_vmalloc_enabled()) {
326 if (flags & KASAN_VMALLOC_INIT)
327 init_vmalloc_pages(start, size);
328 return (void *)start;
329 }
330
331 /*
332 * Don't tag non-VM_ALLOC mappings, as:
333 *
334 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only
335 * supports tagging physical memory. Therefore, it can only tag a
336 * single mapping of normal physical pages.
337 * 2. Hardware tag-based KASAN can only tag memory mapped with special
338 * mapping protection bits, see arch_vmap_pgprot_tagged().
339 * As non-VM_ALLOC mappings can be mapped outside of vmalloc code,
340 * providing these bits would require tracking all non-VM_ALLOC
341 * mappers.
342 *
343 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags
344 * the first virtual mapping, which is created by vmalloc().
345 * Tagging the page_alloc memory backing that vmalloc() allocation is
346 * skipped, see ___GFP_SKIP_KASAN.
347 *
348 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.
349 */
350 if (!(flags & KASAN_VMALLOC_VM_ALLOC)) {
351 WARN_ON(flags & KASAN_VMALLOC_INIT);
352 return (void *)start;
353 }
354
355 /*
356 * Don't tag executable memory.
357 * The kernel doesn't tolerate having the PC register tagged.
358 */
359 if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) {
360 WARN_ON(flags & KASAN_VMALLOC_INIT);
361 return (void *)start;
362 }
363
364 tag = (flags & KASAN_VMALLOC_KEEP_TAG) ? get_tag(start) : kasan_random_tag();
365 start = set_tag(start, tag);
366
367 /* Unpoison and initialize memory up to size. */
368 kasan_unpoison(addr: start, size, init: flags & KASAN_VMALLOC_INIT);
369
370 /*
371 * Explicitly poison and initialize the in-page vmalloc() redzone.
372 * Unlike software KASAN modes, hardware tag-based KASAN doesn't
373 * unpoison memory when populating shadow for vmalloc() space.
374 */
375 redzone_start = round_up((unsigned long)start + size,
376 KASAN_GRANULE_SIZE);
377 redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start;
378 kasan_poison(addr: (void *)redzone_start, size: redzone_size, KASAN_TAG_INVALID,
379 init: flags & KASAN_VMALLOC_INIT);
380
381 /*
382 * Set per-page tag flags to allow accessing physical memory for the
383 * vmalloc() mapping through page_address(vmalloc_to_page()).
384 */
385 unpoison_vmalloc_pages(addr: start, tag);
386
387 return (void *)start;
388}
389
390void __kasan_poison_vmalloc(const void *start, unsigned long size)
391{
392 /*
393 * No tagging here.
394 * The physical pages backing the vmalloc() allocation are poisoned
395 * through the usual page_alloc paths.
396 */
397}
398
399#endif
400
401void kasan_enable_hw_tags(void)
402{
403 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
404 hw_enable_tag_checks_async();
405 else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
406 hw_enable_tag_checks_asymm();
407 else
408 hw_enable_tag_checks_sync();
409
410 /*
411 * CPUs can only be in one of two states:
412 * - All CPUs support the write_only feature
413 * - No CPUs support the write_only feature
414 *
415 * If the first CPU attempts hw_enable_tag_checks_write_only() and
416 * finds the feature unsupported, kasan_flag_write_only is set to OFF
417 * to avoid further unnecessary calls on other CPUs.
418 */
419 if (kasan_flag_write_only && hw_enable_tag_checks_write_only()) {
420 kasan_flag_write_only = false;
421 pr_err_once("write-only mode is not supported and thus not enabled\n");
422 }
423}
424
425#if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
426
427EXPORT_SYMBOL_IF_KUNIT(kasan_enable_hw_tags);
428
429VISIBLE_IF_KUNIT void kasan_force_async_fault(void)
430{
431 hw_force_async_tag_fault();
432}
433EXPORT_SYMBOL_IF_KUNIT(kasan_force_async_fault);
434
435VISIBLE_IF_KUNIT bool kasan_write_only_enabled(void)
436{
437 return kasan_flag_write_only;
438}
439EXPORT_SYMBOL_IF_KUNIT(kasan_write_only_enabled);
440
441#endif
442

source code of linux/mm/kasan/hw_tags.c