1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) Gao Xiang <xiang@kernel.org> |
4 | * |
5 | * For low-latency decompression algorithms (e.g. lz4), reserve consecutive |
6 | * per-CPU virtual memory (in pages) in advance to store such inplace I/O |
7 | * data if inplace decompression is failed (due to unmet inplace margin for |
8 | * example). |
9 | */ |
10 | #include "internal.h" |
11 | |
12 | struct erofs_pcpubuf { |
13 | raw_spinlock_t lock; |
14 | void *ptr; |
15 | struct page **pages; |
16 | unsigned int nrpages; |
17 | }; |
18 | |
19 | static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb); |
20 | |
21 | void *erofs_get_pcpubuf(unsigned int requiredpages) |
22 | __acquires(pcb->lock) |
23 | { |
24 | struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb); |
25 | |
26 | raw_spin_lock(&pcb->lock); |
27 | /* check if the per-CPU buffer is too small */ |
28 | if (requiredpages > pcb->nrpages) { |
29 | raw_spin_unlock(&pcb->lock); |
30 | put_cpu_var(erofs_pcb); |
31 | /* (for sparse checker) pretend pcb->lock is still taken */ |
32 | __acquire(pcb->lock); |
33 | return NULL; |
34 | } |
35 | return pcb->ptr; |
36 | } |
37 | |
38 | void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock) |
39 | { |
40 | struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id()); |
41 | |
42 | DBG_BUGON(pcb->ptr != ptr); |
43 | raw_spin_unlock(&pcb->lock); |
44 | put_cpu_var(erofs_pcb); |
45 | } |
46 | |
47 | /* the next step: support per-CPU page buffers hotplug */ |
48 | int erofs_pcpubuf_growsize(unsigned int nrpages) |
49 | { |
50 | static DEFINE_MUTEX(pcb_resize_mutex); |
51 | static unsigned int pcb_nrpages; |
52 | struct page *pagepool = NULL; |
53 | int delta, cpu, ret, i; |
54 | |
55 | mutex_lock(&pcb_resize_mutex); |
56 | delta = nrpages - pcb_nrpages; |
57 | ret = 0; |
58 | /* avoid shrinking pcpubuf, since no idea how many fses rely on */ |
59 | if (delta <= 0) |
60 | goto out; |
61 | |
62 | for_each_possible_cpu(cpu) { |
63 | struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); |
64 | struct page **pages, **oldpages; |
65 | void *ptr, *old_ptr; |
66 | |
67 | pages = kmalloc_array(n: nrpages, size: sizeof(*pages), GFP_KERNEL); |
68 | if (!pages) { |
69 | ret = -ENOMEM; |
70 | break; |
71 | } |
72 | |
73 | for (i = 0; i < nrpages; ++i) { |
74 | pages[i] = erofs_allocpage(pagepool: &pagepool, GFP_KERNEL); |
75 | if (!pages[i]) { |
76 | ret = -ENOMEM; |
77 | oldpages = pages; |
78 | goto free_pagearray; |
79 | } |
80 | } |
81 | ptr = vmap(pages, count: nrpages, VM_MAP, PAGE_KERNEL); |
82 | if (!ptr) { |
83 | ret = -ENOMEM; |
84 | oldpages = pages; |
85 | goto free_pagearray; |
86 | } |
87 | raw_spin_lock(&pcb->lock); |
88 | old_ptr = pcb->ptr; |
89 | pcb->ptr = ptr; |
90 | oldpages = pcb->pages; |
91 | pcb->pages = pages; |
92 | i = pcb->nrpages; |
93 | pcb->nrpages = nrpages; |
94 | raw_spin_unlock(&pcb->lock); |
95 | |
96 | if (!oldpages) { |
97 | DBG_BUGON(old_ptr); |
98 | continue; |
99 | } |
100 | |
101 | if (old_ptr) |
102 | vunmap(addr: old_ptr); |
103 | free_pagearray: |
104 | while (i) |
105 | erofs_pagepool_add(pagepool: &pagepool, page: oldpages[--i]); |
106 | kfree(objp: oldpages); |
107 | if (ret) |
108 | break; |
109 | } |
110 | pcb_nrpages = nrpages; |
111 | erofs_release_pages(pagepool: &pagepool); |
112 | out: |
113 | mutex_unlock(lock: &pcb_resize_mutex); |
114 | return ret; |
115 | } |
116 | |
117 | void __init erofs_pcpubuf_init(void) |
118 | { |
119 | int cpu; |
120 | |
121 | for_each_possible_cpu(cpu) { |
122 | struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); |
123 | |
124 | raw_spin_lock_init(&pcb->lock); |
125 | } |
126 | } |
127 | |
128 | void erofs_pcpubuf_exit(void) |
129 | { |
130 | int cpu, i; |
131 | |
132 | for_each_possible_cpu(cpu) { |
133 | struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu); |
134 | |
135 | if (pcb->ptr) { |
136 | vunmap(addr: pcb->ptr); |
137 | pcb->ptr = NULL; |
138 | } |
139 | if (!pcb->pages) |
140 | continue; |
141 | |
142 | for (i = 0; i < pcb->nrpages; ++i) |
143 | if (pcb->pages[i]) |
144 | put_page(page: pcb->pages[i]); |
145 | kfree(objp: pcb->pages); |
146 | pcb->pages = NULL; |
147 | } |
148 | } |
149 | |