1// SPDX-License-Identifier: GPL-2.0
2/*
3 * This file contains common KASAN code.
4 *
5 * Copyright (c) 2014 Samsung Electronics Co., Ltd.
6 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
7 *
8 * Some code borrowed from https://github.com/xairy/kasan-prototype by
9 * Andrey Konovalov <andreyknvl@gmail.com>
10 */
11
12#include <linux/export.h>
13#include <linux/init.h>
14#include <linux/kasan.h>
15#include <linux/kernel.h>
16#include <linux/linkage.h>
17#include <linux/memblock.h>
18#include <linux/memory.h>
19#include <linux/mm.h>
20#include <linux/module.h>
21#include <linux/printk.h>
22#include <linux/sched.h>
23#include <linux/sched/task_stack.h>
24#include <linux/slab.h>
25#include <linux/stacktrace.h>
26#include <linux/string.h>
27#include <linux/types.h>
28#include <linux/bug.h>
29
30#include "kasan.h"
31#include "../slab.h"
32
33depot_stack_handle_t kasan_save_stack(gfp_t flags, bool can_alloc)
34{
35 unsigned long entries[KASAN_STACK_DEPTH];
36 unsigned int nr_entries;
37
38 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
39 return __stack_depot_save(entries, nr_entries, flags, can_alloc);
40}
41
42void kasan_set_track(struct kasan_track *track, gfp_t flags)
43{
44 track->pid = current->pid;
45 track->stack = kasan_save_stack(flags, true);
46}
47
48#if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
49void kasan_enable_current(void)
50{
51 current->kasan_depth++;
52}
53EXPORT_SYMBOL(kasan_enable_current);
54
55void kasan_disable_current(void)
56{
57 current->kasan_depth--;
58}
59EXPORT_SYMBOL(kasan_disable_current);
60
61#endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
62
63void __kasan_unpoison_range(const void *address, size_t size)
64{
65 kasan_unpoison(address, size, false);
66}
67
68#ifdef CONFIG_KASAN_STACK
69/* Unpoison the entire stack for a task. */
70void kasan_unpoison_task_stack(struct task_struct *task)
71{
72 void *base = task_stack_page(task);
73
74 kasan_unpoison(base, THREAD_SIZE, false);
75}
76
77/* Unpoison the stack for the current task beyond a watermark sp value. */
78asmlinkage void kasan_unpoison_task_stack_below(const void *watermark)
79{
80 /*
81 * Calculate the task stack base address. Avoid using 'current'
82 * because this function is called by early resume code which hasn't
83 * yet set up the percpu register (%gs).
84 */
85 void *base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1));
86
87 kasan_unpoison(base, watermark - base, false);
88}
89#endif /* CONFIG_KASAN_STACK */
90
91/*
92 * Only allow cache merging when stack collection is disabled and no metadata
93 * is present.
94 */
95slab_flags_t __kasan_never_merge(void)
96{
97 if (kasan_stack_collection_enabled())
98 return SLAB_KASAN;
99 return 0;
100}
101
102void __kasan_unpoison_pages(struct page *page, unsigned int order, bool init)
103{
104 u8 tag;
105 unsigned long i;
106
107 if (unlikely(PageHighMem(page)))
108 return;
109
110 tag = kasan_random_tag();
111 kasan_unpoison(set_tag(page_address(page), tag),
112 PAGE_SIZE << order, init);
113 for (i = 0; i < (1 << order); i++)
114 page_kasan_tag_set(page + i, tag);
115}
116
117void __kasan_poison_pages(struct page *page, unsigned int order, bool init)
118{
119 if (likely(!PageHighMem(page)))
120 kasan_poison(page_address(page), PAGE_SIZE << order,
121 KASAN_PAGE_FREE, init);
122}
123
124/*
125 * Adaptive redzone policy taken from the userspace AddressSanitizer runtime.
126 * For larger allocations larger redzones are used.
127 */
128static inline unsigned int optimal_redzone(unsigned int object_size)
129{
130 return
131 object_size <= 64 - 16 ? 16 :
132 object_size <= 128 - 32 ? 32 :
133 object_size <= 512 - 64 ? 64 :
134 object_size <= 4096 - 128 ? 128 :
135 object_size <= (1 << 14) - 256 ? 256 :
136 object_size <= (1 << 15) - 512 ? 512 :
137 object_size <= (1 << 16) - 1024 ? 1024 : 2048;
138}
139
140void __kasan_cache_create(struct kmem_cache *cache, unsigned int *size,
141 slab_flags_t *flags)
142{
143 unsigned int ok_size;
144 unsigned int optimal_size;
145
146 /*
147 * SLAB_KASAN is used to mark caches as ones that are sanitized by
148 * KASAN. Currently this flag is used in two places:
149 * 1. In slab_ksize() when calculating the size of the accessible
150 * memory within the object.
151 * 2. In slab_common.c to prevent merging of sanitized caches.
152 */
153 *flags |= SLAB_KASAN;
154
155 if (!kasan_stack_collection_enabled())
156 return;
157
158 ok_size = *size;
159
160 /* Add alloc meta into redzone. */
161 cache->kasan_info.alloc_meta_offset = *size;
162 *size += sizeof(struct kasan_alloc_meta);
163
164 /*
165 * If alloc meta doesn't fit, don't add it.
166 * This can only happen with SLAB, as it has KMALLOC_MAX_SIZE equal
167 * to KMALLOC_MAX_CACHE_SIZE and doesn't fall back to page_alloc for
168 * larger sizes.
169 */
170 if (*size > KMALLOC_MAX_SIZE) {
171 cache->kasan_info.alloc_meta_offset = 0;
172 *size = ok_size;
173 /* Continue, since free meta might still fit. */
174 }
175
176 /* Only the generic mode uses free meta or flexible redzones. */
177 if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) {
178 cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;
179 return;
180 }
181
182 /*
183 * Add free meta into redzone when it's not possible to store
184 * it in the object. This is the case when:
185 * 1. Object is SLAB_TYPESAFE_BY_RCU, which means that it can
186 * be touched after it was freed, or
187 * 2. Object has a constructor, which means it's expected to
188 * retain its content until the next allocation, or
189 * 3. Object is too small.
190 * Otherwise cache->kasan_info.free_meta_offset = 0 is implied.
191 */
192 if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor ||
193 cache->object_size < sizeof(struct kasan_free_meta)) {
194 ok_size = *size;
195
196 cache->kasan_info.free_meta_offset = *size;
197 *size += sizeof(struct kasan_free_meta);
198
199 /* If free meta doesn't fit, don't add it. */
200 if (*size > KMALLOC_MAX_SIZE) {
201 cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;
202 *size = ok_size;
203 }
204 }
205
206 /* Calculate size with optimal redzone. */
207 optimal_size = cache->object_size + optimal_redzone(cache->object_size);
208 /* Limit it with KMALLOC_MAX_SIZE (relevant for SLAB only). */
209 if (optimal_size > KMALLOC_MAX_SIZE)
210 optimal_size = KMALLOC_MAX_SIZE;
211 /* Use optimal size if the size with added metas is not large enough. */
212 if (*size < optimal_size)
213 *size = optimal_size;
214}
215
216void __kasan_cache_create_kmalloc(struct kmem_cache *cache)
217{
218 cache->kasan_info.is_kmalloc = true;
219}
220
221size_t __kasan_metadata_size(struct kmem_cache *cache)
222{
223 if (!kasan_stack_collection_enabled())
224 return 0;
225 return (cache->kasan_info.alloc_meta_offset ?
226 sizeof(struct kasan_alloc_meta) : 0) +
227 (cache->kasan_info.free_meta_offset ?
228 sizeof(struct kasan_free_meta) : 0);
229}
230
231struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,
232 const void *object)
233{
234 if (!cache->kasan_info.alloc_meta_offset)
235 return NULL;
236 return kasan_reset_tag(object) + cache->kasan_info.alloc_meta_offset;
237}
238
239#ifdef CONFIG_KASAN_GENERIC
240struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,
241 const void *object)
242{
243 BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32);
244 if (cache->kasan_info.free_meta_offset == KASAN_NO_FREE_META)
245 return NULL;
246 return kasan_reset_tag(object) + cache->kasan_info.free_meta_offset;
247}
248#endif
249
250void __kasan_poison_slab(struct slab *slab)
251{
252 struct page *page = slab_page(slab);
253 unsigned long i;
254
255 for (i = 0; i < compound_nr(page); i++)
256 page_kasan_tag_reset(page + i);
257 kasan_poison(page_address(page), page_size(page),
258 KASAN_SLAB_REDZONE, false);
259}
260
261void __kasan_unpoison_object_data(struct kmem_cache *cache, void *object)
262{
263 kasan_unpoison(object, cache->object_size, false);
264}
265
266void __kasan_poison_object_data(struct kmem_cache *cache, void *object)
267{
268 kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),
269 KASAN_SLAB_REDZONE, false);
270}
271
272/*
273 * This function assigns a tag to an object considering the following:
274 * 1. A cache might have a constructor, which might save a pointer to a slab
275 * object somewhere (e.g. in the object itself). We preassign a tag for
276 * each object in caches with constructors during slab creation and reuse
277 * the same tag each time a particular object is allocated.
278 * 2. A cache might be SLAB_TYPESAFE_BY_RCU, which means objects can be
279 * accessed after being freed. We preassign tags for objects in these
280 * caches as well.
281 * 3. For SLAB allocator we can't preassign tags randomly since the freelist
282 * is stored as an array of indexes instead of a linked list. Assign tags
283 * based on objects indexes, so that objects that are next to each other
284 * get different tags.
285 */
286static inline u8 assign_tag(struct kmem_cache *cache,
287 const void *object, bool init)
288{
289 if (IS_ENABLED(CONFIG_KASAN_GENERIC))
290 return 0xff;
291
292 /*
293 * If the cache neither has a constructor nor has SLAB_TYPESAFE_BY_RCU
294 * set, assign a tag when the object is being allocated (init == false).
295 */
296 if (!cache->ctor && !(cache->flags & SLAB_TYPESAFE_BY_RCU))
297 return init ? KASAN_TAG_KERNEL : kasan_random_tag();
298
299 /* For caches that either have a constructor or SLAB_TYPESAFE_BY_RCU: */
300#ifdef CONFIG_SLAB
301 /* For SLAB assign tags based on the object index in the freelist. */
302 return (u8)obj_to_index(cache, virt_to_slab(object), (void *)object);
303#else
304 /*
305 * For SLUB assign a random tag during slab creation, otherwise reuse
306 * the already assigned tag.
307 */
308 return init ? kasan_random_tag() : get_tag(object);
309#endif
310}
311
312void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
313 const void *object)
314{
315 struct kasan_alloc_meta *alloc_meta;
316
317 if (kasan_stack_collection_enabled()) {
318 alloc_meta = kasan_get_alloc_meta(cache, object);
319 if (alloc_meta)
320 __memset(alloc_meta, 0, sizeof(*alloc_meta));
321 }
322
323 /* Tag is ignored in set_tag() without CONFIG_KASAN_SW/HW_TAGS */
324 object = set_tag(object, assign_tag(cache, object, true));
325
326 return (void *)object;
327}
328
329static inline bool ____kasan_slab_free(struct kmem_cache *cache, void *object,
330 unsigned long ip, bool quarantine, bool init)
331{
332 u8 tag;
333 void *tagged_object;
334
335 if (!kasan_arch_is_ready())
336 return false;
337
338 tag = get_tag(object);
339 tagged_object = object;
340 object = kasan_reset_tag(object);
341
342 if (is_kfence_address(object))
343 return false;
344
345 if (unlikely(nearest_obj(cache, virt_to_slab(object), object) !=
346 object)) {
347 kasan_report_invalid_free(tagged_object, ip, KASAN_REPORT_INVALID_FREE);
348 return true;
349 }
350
351 /* RCU slabs could be legally used after free within the RCU period */
352 if (unlikely(cache->flags & SLAB_TYPESAFE_BY_RCU))
353 return false;
354
355 if (!kasan_byte_accessible(tagged_object)) {
356 kasan_report_invalid_free(tagged_object, ip, KASAN_REPORT_DOUBLE_FREE);
357 return true;
358 }
359
360 kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),
361 KASAN_SLAB_FREE, init);
362
363 if ((IS_ENABLED(CONFIG_KASAN_GENERIC) && !quarantine))
364 return false;
365
366 if (kasan_stack_collection_enabled())
367 kasan_set_free_info(cache, object, tag);
368
369 return kasan_quarantine_put(cache, object);
370}
371
372bool __kasan_slab_free(struct kmem_cache *cache, void *object,
373 unsigned long ip, bool init)
374{
375 return ____kasan_slab_free(cache, object, ip, true, init);
376}
377
378static inline bool ____kasan_kfree_large(void *ptr, unsigned long ip)
379{
380 if (ptr != page_address(virt_to_head_page(ptr))) {
381 kasan_report_invalid_free(ptr, ip, KASAN_REPORT_INVALID_FREE);
382 return true;
383 }
384
385 if (!kasan_byte_accessible(ptr)) {
386 kasan_report_invalid_free(ptr, ip, KASAN_REPORT_DOUBLE_FREE);
387 return true;
388 }
389
390 /*
391 * The object will be poisoned by kasan_poison_pages() or
392 * kasan_slab_free_mempool().
393 */
394
395 return false;
396}
397
398void __kasan_kfree_large(void *ptr, unsigned long ip)
399{
400 ____kasan_kfree_large(ptr, ip);
401}
402
403void __kasan_slab_free_mempool(void *ptr, unsigned long ip)
404{
405 struct folio *folio;
406
407 folio = virt_to_folio(ptr);
408
409 /*
410 * Even though this function is only called for kmem_cache_alloc and
411 * kmalloc backed mempool allocations, those allocations can still be
412 * !PageSlab() when the size provided to kmalloc is larger than
413 * KMALLOC_MAX_SIZE, and kmalloc falls back onto page_alloc.
414 */
415 if (unlikely(!folio_test_slab(folio))) {
416 if (____kasan_kfree_large(ptr, ip))
417 return;
418 kasan_poison(ptr, folio_size(folio), KASAN_PAGE_FREE, false);
419 } else {
420 struct slab *slab = folio_slab(folio);
421
422 ____kasan_slab_free(slab->slab_cache, ptr, ip, false, false);
423 }
424}
425
426static void set_alloc_info(struct kmem_cache *cache, void *object,
427 gfp_t flags, bool is_kmalloc)
428{
429 struct kasan_alloc_meta *alloc_meta;
430
431 /* Don't save alloc info for kmalloc caches in kasan_slab_alloc(). */
432 if (cache->kasan_info.is_kmalloc && !is_kmalloc)
433 return;
434
435 alloc_meta = kasan_get_alloc_meta(cache, object);
436 if (alloc_meta)
437 kasan_set_track(&alloc_meta->alloc_track, flags);
438}
439
440void * __must_check __kasan_slab_alloc(struct kmem_cache *cache,
441 void *object, gfp_t flags, bool init)
442{
443 u8 tag;
444 void *tagged_object;
445
446 if (gfpflags_allow_blocking(flags))
447 kasan_quarantine_reduce();
448
449 if (unlikely(object == NULL))
450 return NULL;
451
452 if (is_kfence_address(object))
453 return (void *)object;
454
455 /*
456 * Generate and assign random tag for tag-based modes.
457 * Tag is ignored in set_tag() for the generic mode.
458 */
459 tag = assign_tag(cache, object, false);
460 tagged_object = set_tag(object, tag);
461
462 /*
463 * Unpoison the whole object.
464 * For kmalloc() allocations, kasan_kmalloc() will do precise poisoning.
465 */
466 kasan_unpoison(tagged_object, cache->object_size, init);
467
468 /* Save alloc info (if possible) for non-kmalloc() allocations. */
469 if (kasan_stack_collection_enabled())
470 set_alloc_info(cache, (void *)object, flags, false);
471
472 return tagged_object;
473}
474
475static inline void *____kasan_kmalloc(struct kmem_cache *cache,
476 const void *object, size_t size, gfp_t flags)
477{
478 unsigned long redzone_start;
479 unsigned long redzone_end;
480
481 if (gfpflags_allow_blocking(flags))
482 kasan_quarantine_reduce();
483
484 if (unlikely(object == NULL))
485 return NULL;
486
487 if (is_kfence_address(kasan_reset_tag(object)))
488 return (void *)object;
489
490 /*
491 * The object has already been unpoisoned by kasan_slab_alloc() for
492 * kmalloc() or by kasan_krealloc() for krealloc().
493 */
494
495 /*
496 * The redzone has byte-level precision for the generic mode.
497 * Partially poison the last object granule to cover the unaligned
498 * part of the redzone.
499 */
500 if (IS_ENABLED(CONFIG_KASAN_GENERIC))
501 kasan_poison_last_granule((void *)object, size);
502
503 /* Poison the aligned part of the redzone. */
504 redzone_start = round_up((unsigned long)(object + size),
505 KASAN_GRANULE_SIZE);
506 redzone_end = round_up((unsigned long)(object + cache->object_size),
507 KASAN_GRANULE_SIZE);
508 kasan_poison((void *)redzone_start, redzone_end - redzone_start,
509 KASAN_SLAB_REDZONE, false);
510
511 /*
512 * Save alloc info (if possible) for kmalloc() allocations.
513 * This also rewrites the alloc info when called from kasan_krealloc().
514 */
515 if (kasan_stack_collection_enabled())
516 set_alloc_info(cache, (void *)object, flags, true);
517
518 /* Keep the tag that was set by kasan_slab_alloc(). */
519 return (void *)object;
520}
521
522void * __must_check __kasan_kmalloc(struct kmem_cache *cache, const void *object,
523 size_t size, gfp_t flags)
524{
525 return ____kasan_kmalloc(cache, object, size, flags);
526}
527EXPORT_SYMBOL(__kasan_kmalloc);
528
529void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size,
530 gfp_t flags)
531{
532 unsigned long redzone_start;
533 unsigned long redzone_end;
534
535 if (gfpflags_allow_blocking(flags))
536 kasan_quarantine_reduce();
537
538 if (unlikely(ptr == NULL))
539 return NULL;
540
541 /*
542 * The object has already been unpoisoned by kasan_unpoison_pages() for
543 * alloc_pages() or by kasan_krealloc() for krealloc().
544 */
545
546 /*
547 * The redzone has byte-level precision for the generic mode.
548 * Partially poison the last object granule to cover the unaligned
549 * part of the redzone.
550 */
551 if (IS_ENABLED(CONFIG_KASAN_GENERIC))
552 kasan_poison_last_granule(ptr, size);
553
554 /* Poison the aligned part of the redzone. */
555 redzone_start = round_up((unsigned long)(ptr + size),
556 KASAN_GRANULE_SIZE);
557 redzone_end = (unsigned long)ptr + page_size(virt_to_page(ptr));
558 kasan_poison((void *)redzone_start, redzone_end - redzone_start,
559 KASAN_PAGE_REDZONE, false);
560
561 return (void *)ptr;
562}
563
564void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flags)
565{
566 struct slab *slab;
567
568 if (unlikely(object == ZERO_SIZE_PTR))
569 return (void *)object;
570
571 /*
572 * Unpoison the object's data.
573 * Part of it might already have been unpoisoned, but it's unknown
574 * how big that part is.
575 */
576 kasan_unpoison(object, size, false);
577
578 slab = virt_to_slab(object);
579
580 /* Piggy-back on kmalloc() instrumentation to poison the redzone. */
581 if (unlikely(!slab))
582 return __kasan_kmalloc_large(object, size, flags);
583 else
584 return ____kasan_kmalloc(slab->slab_cache, object, size, flags);
585}
586
587bool __kasan_check_byte(const void *address, unsigned long ip)
588{
589 if (!kasan_byte_accessible(address)) {
590 kasan_report((unsigned long)address, 1, false, ip);
591 return false;
592 }
593 return true;
594}
595

source code of linux/mm/kasan/common.c