1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Test cases for SL[AOU]B/page initialization at alloc/free time. |
4 | */ |
5 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
6 | |
7 | #include <linux/init.h> |
8 | #include <linux/kernel.h> |
9 | #include <linux/mm.h> |
10 | #include <linux/module.h> |
11 | #include <linux/slab.h> |
12 | #include <linux/string.h> |
13 | #include <linux/vmalloc.h> |
14 | |
15 | #define GARBAGE_INT (0x09A7BA9E) |
16 | #define GARBAGE_BYTE (0x9E) |
17 | |
18 | #define REPORT_FAILURES_IN_FN() \ |
19 | do { \ |
20 | if (failures) \ |
21 | pr_info("%s failed %d out of %d times\n", \ |
22 | __func__, failures, num_tests); \ |
23 | else \ |
24 | pr_info("all %d tests in %s passed\n", \ |
25 | num_tests, __func__); \ |
26 | } while (0) |
27 | |
28 | /* Calculate the number of uninitialized bytes in the buffer. */ |
29 | static int __init count_nonzero_bytes(void *ptr, size_t size) |
30 | { |
31 | int i, ret = 0; |
32 | unsigned char *p = (unsigned char *)ptr; |
33 | |
34 | for (i = 0; i < size; i++) |
35 | if (p[i]) |
36 | ret++; |
37 | return ret; |
38 | } |
39 | |
40 | /* Fill a buffer with garbage, skipping |skip| first bytes. */ |
41 | static void __init fill_with_garbage_skip(void *ptr, int size, size_t skip) |
42 | { |
43 | unsigned int *p = (unsigned int *)((char *)ptr + skip); |
44 | int i = 0; |
45 | |
46 | WARN_ON(skip > size); |
47 | size -= skip; |
48 | |
49 | while (size >= sizeof(*p)) { |
50 | p[i] = GARBAGE_INT; |
51 | i++; |
52 | size -= sizeof(*p); |
53 | } |
54 | if (size) |
55 | memset(&p[i], GARBAGE_BYTE, size); |
56 | } |
57 | |
58 | static void __init fill_with_garbage(void *ptr, size_t size) |
59 | { |
60 | fill_with_garbage_skip(ptr, size, skip: 0); |
61 | } |
62 | |
63 | static int __init do_alloc_pages_order(int order, int *total_failures) |
64 | { |
65 | struct page *page; |
66 | void *buf; |
67 | size_t size = PAGE_SIZE << order; |
68 | |
69 | page = alloc_pages(GFP_KERNEL, order); |
70 | if (!page) |
71 | goto err; |
72 | buf = page_address(page); |
73 | fill_with_garbage(ptr: buf, size); |
74 | __free_pages(page, order); |
75 | |
76 | page = alloc_pages(GFP_KERNEL, order); |
77 | if (!page) |
78 | goto err; |
79 | buf = page_address(page); |
80 | if (count_nonzero_bytes(ptr: buf, size)) |
81 | (*total_failures)++; |
82 | fill_with_garbage(ptr: buf, size); |
83 | __free_pages(page, order); |
84 | return 1; |
85 | err: |
86 | (*total_failures)++; |
87 | return 1; |
88 | } |
89 | |
90 | /* Test the page allocator by calling alloc_pages with different orders. */ |
91 | static int __init test_pages(int *total_failures) |
92 | { |
93 | int failures = 0, num_tests = 0; |
94 | int i; |
95 | |
96 | for (i = 0; i <= MAX_ORDER; i++) |
97 | num_tests += do_alloc_pages_order(order: i, total_failures: &failures); |
98 | |
99 | REPORT_FAILURES_IN_FN(); |
100 | *total_failures += failures; |
101 | return num_tests; |
102 | } |
103 | |
104 | /* Test kmalloc() with given parameters. */ |
105 | static int __init do_kmalloc_size(size_t size, int *total_failures) |
106 | { |
107 | void *buf; |
108 | |
109 | buf = kmalloc(size, GFP_KERNEL); |
110 | if (!buf) |
111 | goto err; |
112 | fill_with_garbage(ptr: buf, size); |
113 | kfree(objp: buf); |
114 | |
115 | buf = kmalloc(size, GFP_KERNEL); |
116 | if (!buf) |
117 | goto err; |
118 | if (count_nonzero_bytes(ptr: buf, size)) |
119 | (*total_failures)++; |
120 | fill_with_garbage(ptr: buf, size); |
121 | kfree(objp: buf); |
122 | return 1; |
123 | err: |
124 | (*total_failures)++; |
125 | return 1; |
126 | } |
127 | |
128 | /* Test vmalloc() with given parameters. */ |
129 | static int __init do_vmalloc_size(size_t size, int *total_failures) |
130 | { |
131 | void *buf; |
132 | |
133 | buf = vmalloc(size); |
134 | if (!buf) |
135 | goto err; |
136 | fill_with_garbage(ptr: buf, size); |
137 | vfree(addr: buf); |
138 | |
139 | buf = vmalloc(size); |
140 | if (!buf) |
141 | goto err; |
142 | if (count_nonzero_bytes(ptr: buf, size)) |
143 | (*total_failures)++; |
144 | fill_with_garbage(ptr: buf, size); |
145 | vfree(addr: buf); |
146 | return 1; |
147 | err: |
148 | (*total_failures)++; |
149 | return 1; |
150 | } |
151 | |
152 | /* Test kmalloc()/vmalloc() by allocating objects of different sizes. */ |
153 | static int __init test_kvmalloc(int *total_failures) |
154 | { |
155 | int failures = 0, num_tests = 0; |
156 | int i, size; |
157 | |
158 | for (i = 0; i < 20; i++) { |
159 | size = 1 << i; |
160 | num_tests += do_kmalloc_size(size, total_failures: &failures); |
161 | num_tests += do_vmalloc_size(size, total_failures: &failures); |
162 | } |
163 | |
164 | REPORT_FAILURES_IN_FN(); |
165 | *total_failures += failures; |
166 | return num_tests; |
167 | } |
168 | |
169 | #define CTOR_BYTES (sizeof(unsigned int)) |
170 | #define CTOR_PATTERN (0x41414141) |
171 | /* Initialize the first 4 bytes of the object. */ |
172 | static void test_ctor(void *obj) |
173 | { |
174 | *(unsigned int *)obj = CTOR_PATTERN; |
175 | } |
176 | |
177 | /* |
178 | * Check the invariants for the buffer allocated from a slab cache. |
179 | * If the cache has a test constructor, the first 4 bytes of the object must |
180 | * always remain equal to CTOR_PATTERN. |
181 | * If the cache isn't an RCU-typesafe one, or if the allocation is done with |
182 | * __GFP_ZERO, then the object contents must be zeroed after allocation. |
183 | * If the cache is an RCU-typesafe one, the object contents must never be |
184 | * zeroed after the first use. This is checked by memcmp() in |
185 | * do_kmem_cache_size(). |
186 | */ |
187 | static bool __init check_buf(void *buf, int size, bool want_ctor, |
188 | bool want_rcu, bool want_zero) |
189 | { |
190 | int bytes; |
191 | bool fail = false; |
192 | |
193 | bytes = count_nonzero_bytes(ptr: buf, size); |
194 | WARN_ON(want_ctor && want_zero); |
195 | if (want_zero) |
196 | return bytes; |
197 | if (want_ctor) { |
198 | if (*(unsigned int *)buf != CTOR_PATTERN) |
199 | fail = 1; |
200 | } else { |
201 | if (bytes) |
202 | fail = !want_rcu; |
203 | } |
204 | return fail; |
205 | } |
206 | |
207 | #define BULK_SIZE 100 |
208 | static void *bulk_array[BULK_SIZE]; |
209 | |
210 | /* |
211 | * Test kmem_cache with given parameters: |
212 | * want_ctor - use a constructor; |
213 | * want_rcu - use SLAB_TYPESAFE_BY_RCU; |
214 | * want_zero - use __GFP_ZERO. |
215 | */ |
216 | static int __init do_kmem_cache_size(size_t size, bool want_ctor, |
217 | bool want_rcu, bool want_zero, |
218 | int *total_failures) |
219 | { |
220 | struct kmem_cache *c; |
221 | int iter; |
222 | bool fail = false; |
223 | gfp_t alloc_mask = GFP_KERNEL | (want_zero ? __GFP_ZERO : 0); |
224 | void *buf, *buf_copy; |
225 | |
226 | c = kmem_cache_create(name: "test_cache" , size, align: 1, |
227 | flags: want_rcu ? SLAB_TYPESAFE_BY_RCU : 0, |
228 | ctor: want_ctor ? test_ctor : NULL); |
229 | for (iter = 0; iter < 10; iter++) { |
230 | /* Do a test of bulk allocations */ |
231 | if (!want_rcu && !want_ctor) { |
232 | int ret; |
233 | |
234 | ret = kmem_cache_alloc_bulk(s: c, flags: alloc_mask, BULK_SIZE, p: bulk_array); |
235 | if (!ret) { |
236 | fail = true; |
237 | } else { |
238 | int i; |
239 | for (i = 0; i < ret; i++) |
240 | fail |= check_buf(buf: bulk_array[i], size, want_ctor, want_rcu, want_zero); |
241 | kmem_cache_free_bulk(s: c, size: ret, p: bulk_array); |
242 | } |
243 | } |
244 | |
245 | buf = kmem_cache_alloc(cachep: c, flags: alloc_mask); |
246 | /* Check that buf is zeroed, if it must be. */ |
247 | fail |= check_buf(buf, size, want_ctor, want_rcu, want_zero); |
248 | fill_with_garbage_skip(ptr: buf, size, skip: want_ctor ? CTOR_BYTES : 0); |
249 | |
250 | if (!want_rcu) { |
251 | kmem_cache_free(s: c, objp: buf); |
252 | continue; |
253 | } |
254 | |
255 | /* |
256 | * If this is an RCU cache, use a critical section to ensure we |
257 | * can touch objects after they're freed. |
258 | */ |
259 | rcu_read_lock(); |
260 | /* |
261 | * Copy the buffer to check that it's not wiped on |
262 | * free(). |
263 | */ |
264 | buf_copy = kmalloc(size, GFP_ATOMIC); |
265 | if (buf_copy) |
266 | memcpy(buf_copy, buf, size); |
267 | |
268 | kmem_cache_free(s: c, objp: buf); |
269 | /* |
270 | * Check that |buf| is intact after kmem_cache_free(). |
271 | * |want_zero| is false, because we wrote garbage to |
272 | * the buffer already. |
273 | */ |
274 | fail |= check_buf(buf, size, want_ctor, want_rcu, |
275 | want_zero: false); |
276 | if (buf_copy) { |
277 | fail |= (bool)memcmp(p: buf, q: buf_copy, size); |
278 | kfree(objp: buf_copy); |
279 | } |
280 | rcu_read_unlock(); |
281 | } |
282 | kmem_cache_destroy(s: c); |
283 | |
284 | *total_failures += fail; |
285 | return 1; |
286 | } |
287 | |
288 | /* |
289 | * Check that the data written to an RCU-allocated object survives |
290 | * reallocation. |
291 | */ |
292 | static int __init do_kmem_cache_rcu_persistent(int size, int *total_failures) |
293 | { |
294 | struct kmem_cache *c; |
295 | void *buf, *buf_contents, *saved_ptr; |
296 | void **used_objects; |
297 | int i, iter, maxiter = 1024; |
298 | bool fail = false; |
299 | |
300 | c = kmem_cache_create(name: "test_cache" , size, align: size, SLAB_TYPESAFE_BY_RCU, |
301 | NULL); |
302 | buf = kmem_cache_alloc(cachep: c, GFP_KERNEL); |
303 | if (!buf) |
304 | goto out; |
305 | saved_ptr = buf; |
306 | fill_with_garbage(ptr: buf, size); |
307 | buf_contents = kmalloc(size, GFP_KERNEL); |
308 | if (!buf_contents) { |
309 | kmem_cache_free(s: c, objp: buf); |
310 | goto out; |
311 | } |
312 | used_objects = kmalloc_array(n: maxiter, size: sizeof(void *), GFP_KERNEL); |
313 | if (!used_objects) { |
314 | kmem_cache_free(s: c, objp: buf); |
315 | kfree(objp: buf_contents); |
316 | goto out; |
317 | } |
318 | memcpy(buf_contents, buf, size); |
319 | kmem_cache_free(s: c, objp: buf); |
320 | /* |
321 | * Run for a fixed number of iterations. If we never hit saved_ptr, |
322 | * assume the test passes. |
323 | */ |
324 | for (iter = 0; iter < maxiter; iter++) { |
325 | buf = kmem_cache_alloc(cachep: c, GFP_KERNEL); |
326 | used_objects[iter] = buf; |
327 | if (buf == saved_ptr) { |
328 | fail = memcmp(p: buf_contents, q: buf, size); |
329 | for (i = 0; i <= iter; i++) |
330 | kmem_cache_free(s: c, objp: used_objects[i]); |
331 | goto free_out; |
332 | } |
333 | } |
334 | |
335 | for (iter = 0; iter < maxiter; iter++) |
336 | kmem_cache_free(s: c, objp: used_objects[iter]); |
337 | |
338 | free_out: |
339 | kfree(objp: buf_contents); |
340 | kfree(objp: used_objects); |
341 | out: |
342 | kmem_cache_destroy(s: c); |
343 | *total_failures += fail; |
344 | return 1; |
345 | } |
346 | |
347 | static int __init do_kmem_cache_size_bulk(int size, int *total_failures) |
348 | { |
349 | struct kmem_cache *c; |
350 | int i, iter, maxiter = 1024; |
351 | int num, bytes; |
352 | bool fail = false; |
353 | void *objects[10]; |
354 | |
355 | c = kmem_cache_create(name: "test_cache" , size, align: size, flags: 0, NULL); |
356 | for (iter = 0; (iter < maxiter) && !fail; iter++) { |
357 | num = kmem_cache_alloc_bulk(s: c, GFP_KERNEL, ARRAY_SIZE(objects), |
358 | p: objects); |
359 | for (i = 0; i < num; i++) { |
360 | bytes = count_nonzero_bytes(ptr: objects[i], size); |
361 | if (bytes) |
362 | fail = true; |
363 | fill_with_garbage(ptr: objects[i], size); |
364 | } |
365 | |
366 | if (num) |
367 | kmem_cache_free_bulk(s: c, size: num, p: objects); |
368 | } |
369 | kmem_cache_destroy(s: c); |
370 | *total_failures += fail; |
371 | return 1; |
372 | } |
373 | |
374 | /* |
375 | * Test kmem_cache allocation by creating caches of different sizes, with and |
376 | * without constructors, with and without SLAB_TYPESAFE_BY_RCU. |
377 | */ |
378 | static int __init test_kmemcache(int *total_failures) |
379 | { |
380 | int failures = 0, num_tests = 0; |
381 | int i, flags, size; |
382 | bool ctor, rcu, zero; |
383 | |
384 | for (i = 0; i < 10; i++) { |
385 | size = 8 << i; |
386 | for (flags = 0; flags < 8; flags++) { |
387 | ctor = flags & 1; |
388 | rcu = flags & 2; |
389 | zero = flags & 4; |
390 | if (ctor & zero) |
391 | continue; |
392 | num_tests += do_kmem_cache_size(size, want_ctor: ctor, want_rcu: rcu, want_zero: zero, |
393 | total_failures: &failures); |
394 | } |
395 | num_tests += do_kmem_cache_size_bulk(size, total_failures: &failures); |
396 | } |
397 | REPORT_FAILURES_IN_FN(); |
398 | *total_failures += failures; |
399 | return num_tests; |
400 | } |
401 | |
402 | /* Test the behavior of SLAB_TYPESAFE_BY_RCU caches of different sizes. */ |
403 | static int __init test_rcu_persistent(int *total_failures) |
404 | { |
405 | int failures = 0, num_tests = 0; |
406 | int i, size; |
407 | |
408 | for (i = 0; i < 10; i++) { |
409 | size = 8 << i; |
410 | num_tests += do_kmem_cache_rcu_persistent(size, total_failures: &failures); |
411 | } |
412 | REPORT_FAILURES_IN_FN(); |
413 | *total_failures += failures; |
414 | return num_tests; |
415 | } |
416 | |
417 | /* |
418 | * Run the tests. Each test function returns the number of executed tests and |
419 | * updates |failures| with the number of failed tests. |
420 | */ |
421 | static int __init test_meminit_init(void) |
422 | { |
423 | int failures = 0, num_tests = 0; |
424 | |
425 | num_tests += test_pages(total_failures: &failures); |
426 | num_tests += test_kvmalloc(total_failures: &failures); |
427 | num_tests += test_kmemcache(total_failures: &failures); |
428 | num_tests += test_rcu_persistent(total_failures: &failures); |
429 | |
430 | if (failures == 0) |
431 | pr_info("all %d tests passed!\n" , num_tests); |
432 | else |
433 | pr_info("failures: %d out of %d\n" , failures, num_tests); |
434 | |
435 | return failures ? -EINVAL : 0; |
436 | } |
437 | module_init(test_meminit_init); |
438 | |
439 | MODULE_LICENSE("GPL" ); |
440 | |