1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (C) 2004 Benjamin Herrenschmidt, IBM Corp. |
4 | * <benh@kernel.crashing.org> |
5 | * Copyright (C) 2012 ARM Limited |
6 | * Copyright (C) 2015 Regents of the University of California |
7 | */ |
8 | |
9 | #include <linux/elf.h> |
10 | #include <linux/mm.h> |
11 | #include <linux/slab.h> |
12 | #include <linux/binfmts.h> |
13 | #include <linux/err.h> |
14 | #include <asm/page.h> |
15 | #include <asm/vdso.h> |
16 | #include <linux/time_namespace.h> |
17 | #include <vdso/datapage.h> |
18 | #include <vdso/vsyscall.h> |
19 | |
20 | enum vvar_pages { |
21 | VVAR_DATA_PAGE_OFFSET, |
22 | VVAR_TIMENS_PAGE_OFFSET, |
23 | VVAR_NR_PAGES, |
24 | }; |
25 | |
26 | enum rv_vdso_map { |
27 | RV_VDSO_MAP_VVAR, |
28 | RV_VDSO_MAP_VDSO, |
29 | }; |
30 | |
31 | #define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT) |
32 | |
33 | static union vdso_data_store vdso_data_store __page_aligned_data; |
34 | struct vdso_data *vdso_data = vdso_data_store.data; |
35 | |
36 | struct __vdso_info { |
37 | const char *name; |
38 | const char *vdso_code_start; |
39 | const char *vdso_code_end; |
40 | unsigned long vdso_pages; |
41 | /* Data Mapping */ |
42 | struct vm_special_mapping *dm; |
43 | /* Code Mapping */ |
44 | struct vm_special_mapping *cm; |
45 | }; |
46 | |
47 | static struct __vdso_info vdso_info; |
48 | #ifdef CONFIG_COMPAT |
49 | static struct __vdso_info compat_vdso_info; |
50 | #endif |
51 | |
52 | static int vdso_mremap(const struct vm_special_mapping *sm, |
53 | struct vm_area_struct *new_vma) |
54 | { |
55 | current->mm->context.vdso = (void *)new_vma->vm_start; |
56 | |
57 | return 0; |
58 | } |
59 | |
60 | static void __init __vdso_init(struct __vdso_info *vdso_info) |
61 | { |
62 | unsigned int i; |
63 | struct page **vdso_pagelist; |
64 | unsigned long pfn; |
65 | |
66 | if (memcmp(p: vdso_info->vdso_code_start, q: "\177ELF" , size: 4)) |
67 | panic(fmt: "vDSO is not a valid ELF object!\n" ); |
68 | |
69 | vdso_info->vdso_pages = ( |
70 | vdso_info->vdso_code_end - |
71 | vdso_info->vdso_code_start) >> |
72 | PAGE_SHIFT; |
73 | |
74 | vdso_pagelist = kcalloc(n: vdso_info->vdso_pages, |
75 | size: sizeof(struct page *), |
76 | GFP_KERNEL); |
77 | if (vdso_pagelist == NULL) |
78 | panic(fmt: "vDSO kcalloc failed!\n" ); |
79 | |
80 | /* Grab the vDSO code pages. */ |
81 | pfn = sym_to_pfn(vdso_info->vdso_code_start); |
82 | |
83 | for (i = 0; i < vdso_info->vdso_pages; i++) |
84 | vdso_pagelist[i] = pfn_to_page(pfn + i); |
85 | |
86 | vdso_info->cm->pages = vdso_pagelist; |
87 | } |
88 | |
89 | #ifdef CONFIG_TIME_NS |
90 | struct vdso_data *arch_get_vdso_data(void *vvar_page) |
91 | { |
92 | return (struct vdso_data *)(vvar_page); |
93 | } |
94 | |
95 | /* |
96 | * The vvar mapping contains data for a specific time namespace, so when a task |
97 | * changes namespace we must unmap its vvar data for the old namespace. |
98 | * Subsequent faults will map in data for the new namespace. |
99 | * |
100 | * For more details see timens_setup_vdso_data(). |
101 | */ |
102 | int vdso_join_timens(struct task_struct *task, struct time_namespace *ns) |
103 | { |
104 | struct mm_struct *mm = task->mm; |
105 | struct vm_area_struct *vma; |
106 | VMA_ITERATOR(vmi, mm, 0); |
107 | |
108 | mmap_read_lock(mm); |
109 | |
110 | for_each_vma(vmi, vma) { |
111 | if (vma_is_special_mapping(vma, sm: vdso_info.dm)) |
112 | zap_vma_pages(vma); |
113 | #ifdef CONFIG_COMPAT |
114 | if (vma_is_special_mapping(vma, sm: compat_vdso_info.dm)) |
115 | zap_vma_pages(vma); |
116 | #endif |
117 | } |
118 | |
119 | mmap_read_unlock(mm); |
120 | return 0; |
121 | } |
122 | #endif |
123 | |
124 | static vm_fault_t vvar_fault(const struct vm_special_mapping *sm, |
125 | struct vm_area_struct *vma, struct vm_fault *vmf) |
126 | { |
127 | struct page *timens_page = find_timens_vvar_page(vma); |
128 | unsigned long pfn; |
129 | |
130 | switch (vmf->pgoff) { |
131 | case VVAR_DATA_PAGE_OFFSET: |
132 | if (timens_page) |
133 | pfn = page_to_pfn(timens_page); |
134 | else |
135 | pfn = sym_to_pfn(vdso_data); |
136 | break; |
137 | #ifdef CONFIG_TIME_NS |
138 | case VVAR_TIMENS_PAGE_OFFSET: |
139 | /* |
140 | * If a task belongs to a time namespace then a namespace |
141 | * specific VVAR is mapped with the VVAR_DATA_PAGE_OFFSET and |
142 | * the real VVAR page is mapped with the VVAR_TIMENS_PAGE_OFFSET |
143 | * offset. |
144 | * See also the comment near timens_setup_vdso_data(). |
145 | */ |
146 | if (!timens_page) |
147 | return VM_FAULT_SIGBUS; |
148 | pfn = sym_to_pfn(vdso_data); |
149 | break; |
150 | #endif /* CONFIG_TIME_NS */ |
151 | default: |
152 | return VM_FAULT_SIGBUS; |
153 | } |
154 | |
155 | return vmf_insert_pfn(vma, addr: vmf->address, pfn); |
156 | } |
157 | |
158 | static struct vm_special_mapping rv_vdso_maps[] __ro_after_init = { |
159 | [RV_VDSO_MAP_VVAR] = { |
160 | .name = "[vvar]" , |
161 | .fault = vvar_fault, |
162 | }, |
163 | [RV_VDSO_MAP_VDSO] = { |
164 | .name = "[vdso]" , |
165 | .mremap = vdso_mremap, |
166 | }, |
167 | }; |
168 | |
169 | static struct __vdso_info vdso_info __ro_after_init = { |
170 | .name = "vdso" , |
171 | .vdso_code_start = vdso_start, |
172 | .vdso_code_end = vdso_end, |
173 | .dm = &rv_vdso_maps[RV_VDSO_MAP_VVAR], |
174 | .cm = &rv_vdso_maps[RV_VDSO_MAP_VDSO], |
175 | }; |
176 | |
177 | #ifdef CONFIG_COMPAT |
178 | static struct vm_special_mapping rv_compat_vdso_maps[] __ro_after_init = { |
179 | [RV_VDSO_MAP_VVAR] = { |
180 | .name = "[vvar]" , |
181 | .fault = vvar_fault, |
182 | }, |
183 | [RV_VDSO_MAP_VDSO] = { |
184 | .name = "[vdso]" , |
185 | .mremap = vdso_mremap, |
186 | }, |
187 | }; |
188 | |
189 | static struct __vdso_info compat_vdso_info __ro_after_init = { |
190 | .name = "compat_vdso" , |
191 | .vdso_code_start = compat_vdso_start, |
192 | .vdso_code_end = compat_vdso_end, |
193 | .dm = &rv_compat_vdso_maps[RV_VDSO_MAP_VVAR], |
194 | .cm = &rv_compat_vdso_maps[RV_VDSO_MAP_VDSO], |
195 | }; |
196 | #endif |
197 | |
198 | static int __init vdso_init(void) |
199 | { |
200 | __vdso_init(vdso_info: &vdso_info); |
201 | #ifdef CONFIG_COMPAT |
202 | __vdso_init(vdso_info: &compat_vdso_info); |
203 | #endif |
204 | |
205 | return 0; |
206 | } |
207 | arch_initcall(vdso_init); |
208 | |
209 | static int __setup_additional_pages(struct mm_struct *mm, |
210 | struct linux_binprm *bprm, |
211 | int uses_interp, |
212 | struct __vdso_info *vdso_info) |
213 | { |
214 | unsigned long vdso_base, vdso_text_len, vdso_mapping_len; |
215 | void *ret; |
216 | |
217 | BUILD_BUG_ON(VVAR_NR_PAGES != __VVAR_PAGES); |
218 | |
219 | vdso_text_len = vdso_info->vdso_pages << PAGE_SHIFT; |
220 | /* Be sure to map the data page */ |
221 | vdso_mapping_len = vdso_text_len + VVAR_SIZE; |
222 | |
223 | vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); |
224 | if (IS_ERR_VALUE(vdso_base)) { |
225 | ret = ERR_PTR(error: vdso_base); |
226 | goto up_fail; |
227 | } |
228 | |
229 | ret = _install_special_mapping(mm, addr: vdso_base, VVAR_SIZE, |
230 | flags: (VM_READ | VM_MAYREAD | VM_PFNMAP), spec: vdso_info->dm); |
231 | if (IS_ERR(ptr: ret)) |
232 | goto up_fail; |
233 | |
234 | vdso_base += VVAR_SIZE; |
235 | mm->context.vdso = (void *)vdso_base; |
236 | |
237 | ret = |
238 | _install_special_mapping(mm, addr: vdso_base, len: vdso_text_len, |
239 | flags: (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), |
240 | spec: vdso_info->cm); |
241 | |
242 | if (IS_ERR(ptr: ret)) |
243 | goto up_fail; |
244 | |
245 | return 0; |
246 | |
247 | up_fail: |
248 | mm->context.vdso = NULL; |
249 | return PTR_ERR(ptr: ret); |
250 | } |
251 | |
252 | #ifdef CONFIG_COMPAT |
253 | int compat_arch_setup_additional_pages(struct linux_binprm *bprm, |
254 | int uses_interp) |
255 | { |
256 | struct mm_struct *mm = current->mm; |
257 | int ret; |
258 | |
259 | if (mmap_write_lock_killable(mm)) |
260 | return -EINTR; |
261 | |
262 | ret = __setup_additional_pages(mm, bprm, uses_interp, |
263 | vdso_info: &compat_vdso_info); |
264 | mmap_write_unlock(mm); |
265 | |
266 | return ret; |
267 | } |
268 | #endif |
269 | |
270 | int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) |
271 | { |
272 | struct mm_struct *mm = current->mm; |
273 | int ret; |
274 | |
275 | if (mmap_write_lock_killable(mm)) |
276 | return -EINTR; |
277 | |
278 | ret = __setup_additional_pages(mm, bprm, uses_interp, vdso_info: &vdso_info); |
279 | mmap_write_unlock(mm); |
280 | |
281 | return ret; |
282 | } |
283 | |