1/* SPDX-License-Identifier: GPL-2.0 */
2
3/*
4 * This header provides generic wrappers for memory access instrumentation that
5 * the compiler cannot emit for: KASAN, KCSAN, KMSAN.
6 */
7#ifndef _LINUX_INSTRUMENTED_H
8#define _LINUX_INSTRUMENTED_H
9
10#include <linux/compiler.h>
11#include <linux/kasan-checks.h>
12#include <linux/kcsan-checks.h>
13#include <linux/kmsan-checks.h>
14#include <linux/types.h>
15
16/**
17 * instrument_read - instrument regular read access
18 * @v: address of access
19 * @size: size of access
20 *
21 * Instrument a regular read access. The instrumentation should be inserted
22 * before the actual read happens.
23 */
24static __always_inline void instrument_read(const volatile void *v, size_t size)
25{
26 kasan_check_read(p: v, size);
27 kcsan_check_read(v, size);
28}
29
30/**
31 * instrument_write - instrument regular write access
32 * @v: address of access
33 * @size: size of access
34 *
35 * Instrument a regular write access. The instrumentation should be inserted
36 * before the actual write happens.
37 */
38static __always_inline void instrument_write(const volatile void *v, size_t size)
39{
40 kasan_check_write(p: v, size);
41 kcsan_check_write(v, size);
42}
43
44/**
45 * instrument_read_write - instrument regular read-write access
46 * @v: address of access
47 * @size: size of access
48 *
49 * Instrument a regular write access. The instrumentation should be inserted
50 * before the actual write happens.
51 */
52static __always_inline void instrument_read_write(const volatile void *v, size_t size)
53{
54 kasan_check_write(p: v, size);
55 kcsan_check_read_write(v, size);
56}
57
58/**
59 * instrument_atomic_read - instrument atomic read access
60 * @v: address of access
61 * @size: size of access
62 *
63 * Instrument an atomic read access. The instrumentation should be inserted
64 * before the actual read happens.
65 */
66static __always_inline void instrument_atomic_read(const volatile void *v, size_t size)
67{
68 kasan_check_read(p: v, size);
69 kcsan_check_atomic_read(v, size);
70}
71
72/**
73 * instrument_atomic_write - instrument atomic write access
74 * @v: address of access
75 * @size: size of access
76 *
77 * Instrument an atomic write access. The instrumentation should be inserted
78 * before the actual write happens.
79 */
80static __always_inline void instrument_atomic_write(const volatile void *v, size_t size)
81{
82 kasan_check_write(p: v, size);
83 kcsan_check_atomic_write(v, size);
84}
85
86/**
87 * instrument_atomic_read_write - instrument atomic read-write access
88 * @v: address of access
89 * @size: size of access
90 *
91 * Instrument an atomic read-write access. The instrumentation should be
92 * inserted before the actual write happens.
93 */
94static __always_inline void instrument_atomic_read_write(const volatile void *v, size_t size)
95{
96 kasan_check_write(p: v, size);
97 kcsan_check_atomic_read_write(v, size);
98}
99
100/**
101 * instrument_copy_to_user - instrument reads of copy_to_user
102 * @to: destination address
103 * @from: source address
104 * @n: number of bytes to copy
105 *
106 * Instrument reads from kernel memory, that are due to copy_to_user (and
107 * variants). The instrumentation must be inserted before the accesses.
108 */
109static __always_inline void
110instrument_copy_to_user(void __user *to, const void *from, unsigned long n)
111{
112 kasan_check_read(p: from, size: n);
113 kcsan_check_read(from, n);
114 kmsan_copy_to_user(to, from, to_copy: n, left: 0);
115}
116
117/**
118 * instrument_copy_from_user_before - add instrumentation before copy_from_user
119 * @to: destination address
120 * @from: source address
121 * @n: number of bytes to copy
122 *
123 * Instrument writes to kernel memory, that are due to copy_from_user (and
124 * variants). The instrumentation should be inserted before the accesses.
125 */
126static __always_inline void
127instrument_copy_from_user_before(const void *to, const void __user *from, unsigned long n)
128{
129 kasan_check_write(p: to, size: n);
130 kcsan_check_write(to, n);
131}
132
133/**
134 * instrument_copy_from_user_after - add instrumentation after copy_from_user
135 * @to: destination address
136 * @from: source address
137 * @n: number of bytes to copy
138 * @left: number of bytes not copied (as returned by copy_from_user)
139 *
140 * Instrument writes to kernel memory, that are due to copy_from_user (and
141 * variants). The instrumentation should be inserted after the accesses.
142 */
143static __always_inline void
144instrument_copy_from_user_after(const void *to, const void __user *from,
145 unsigned long n, unsigned long left)
146{
147 kmsan_unpoison_memory(address: to, size: n - left);
148}
149
150/**
151 * instrument_get_user() - add instrumentation to get_user()-like macros
152 * @to: destination variable, may not be address-taken
153 *
154 * get_user() and friends are fragile, so it may depend on the implementation
155 * whether the instrumentation happens before or after the data is copied from
156 * the userspace.
157 */
158#define instrument_get_user(to) \
159({ \
160 u64 __tmp = (u64)(to); \
161 kmsan_unpoison_memory(&__tmp, sizeof(__tmp)); \
162 to = __tmp; \
163})
164
165
166/**
167 * instrument_put_user() - add instrumentation to put_user()-like macros
168 * @from: source address
169 * @ptr: userspace pointer to copy to
170 * @size: number of bytes to copy
171 *
172 * put_user() and friends are fragile, so it may depend on the implementation
173 * whether the instrumentation happens before or after the data is copied from
174 * the userspace.
175 */
176#define instrument_put_user(from, ptr, size) \
177({ \
178 kmsan_copy_to_user(ptr, &from, sizeof(from), 0); \
179})
180
181#endif /* _LINUX_INSTRUMENTED_H */
182

source code of linux/include/linux/instrumented.h