1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* Module internals |
3 | * |
4 | * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved. |
5 | * Written by David Howells (dhowells@redhat.com) |
6 | * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org> |
7 | */ |
8 | |
9 | #include <linux/elf.h> |
10 | #include <linux/compiler.h> |
11 | #include <linux/module.h> |
12 | #include <linux/mutex.h> |
13 | #include <linux/rculist.h> |
14 | #include <linux/rcupdate.h> |
15 | #include <linux/mm.h> |
16 | |
17 | #ifndef ARCH_SHF_SMALL |
18 | #define ARCH_SHF_SMALL 0 |
19 | #endif |
20 | |
21 | /* |
22 | * Use highest 4 bits of sh_entsize to store the mod_mem_type of this |
23 | * section. This leaves 28 bits for offset on 32-bit systems, which is |
24 | * about 256 MiB (WARN_ON_ONCE if we exceed that). |
25 | */ |
26 | |
27 | #define SH_ENTSIZE_TYPE_BITS 4 |
28 | #define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS) |
29 | #define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1) |
30 | #define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1) |
31 | |
32 | /* Maximum number of characters written by module_flags() */ |
33 | #define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4) |
34 | |
35 | struct kernel_symbol { |
36 | #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS |
37 | int value_offset; |
38 | int name_offset; |
39 | int namespace_offset; |
40 | #else |
41 | unsigned long value; |
42 | const char *name; |
43 | const char *namespace; |
44 | #endif |
45 | }; |
46 | |
47 | extern struct mutex module_mutex; |
48 | extern struct list_head modules; |
49 | |
50 | extern const struct module_attribute *const modinfo_attrs[]; |
51 | extern const size_t modinfo_attrs_count; |
52 | |
53 | /* Provided by the linker */ |
54 | extern const struct kernel_symbol __start___ksymtab[]; |
55 | extern const struct kernel_symbol __stop___ksymtab[]; |
56 | extern const struct kernel_symbol __start___ksymtab_gpl[]; |
57 | extern const struct kernel_symbol __stop___ksymtab_gpl[]; |
58 | extern const u32 __start___kcrctab[]; |
59 | extern const u32 __start___kcrctab_gpl[]; |
60 | |
61 | struct load_info { |
62 | const char *name; |
63 | /* pointer to module in temporary copy, freed at end of load_module() */ |
64 | struct module *mod; |
65 | Elf_Ehdr *hdr; |
66 | unsigned long len; |
67 | Elf_Shdr *sechdrs; |
68 | char *secstrings, *strtab; |
69 | unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs; |
70 | bool sig_ok; |
71 | #ifdef CONFIG_KALLSYMS |
72 | unsigned long mod_kallsyms_init_off; |
73 | #endif |
74 | #ifdef CONFIG_MODULE_DECOMPRESS |
75 | #ifdef CONFIG_MODULE_STATS |
76 | unsigned long compressed_len; |
77 | #endif |
78 | struct page **pages; |
79 | unsigned int max_pages; |
80 | unsigned int used_pages; |
81 | #endif |
82 | struct { |
83 | unsigned int sym; |
84 | unsigned int str; |
85 | unsigned int mod; |
86 | unsigned int vers; |
87 | unsigned int info; |
88 | unsigned int pcpu; |
89 | unsigned int vers_ext_crc; |
90 | unsigned int vers_ext_name; |
91 | } index; |
92 | }; |
93 | |
94 | enum mod_license { |
95 | NOT_GPL_ONLY, |
96 | GPL_ONLY, |
97 | }; |
98 | |
99 | struct find_symbol_arg { |
100 | /* Input */ |
101 | const char *name; |
102 | bool gplok; |
103 | bool warn; |
104 | |
105 | /* Output */ |
106 | struct module *owner; |
107 | const u32 *crc; |
108 | const struct kernel_symbol *sym; |
109 | enum mod_license license; |
110 | }; |
111 | |
112 | int mod_verify_sig(const void *mod, struct load_info *info); |
113 | int try_to_force_load(struct module *mod, const char *reason); |
114 | bool find_symbol(struct find_symbol_arg *fsa); |
115 | struct module *find_module_all(const char *name, size_t len, bool even_unformed); |
116 | int cmp_name(const void *name, const void *sym); |
117 | long module_get_offset_and_type(struct module *mod, enum mod_mem_type type, |
118 | Elf_Shdr *sechdr, unsigned int section); |
119 | char *module_flags(struct module *mod, char *buf, bool show_state); |
120 | size_t module_flags_taint(unsigned long taints, char *buf); |
121 | |
122 | char *module_next_tag_pair(char *string, unsigned long *secsize); |
123 | |
124 | #define for_each_modinfo_entry(entry, info, name) \ |
125 | for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry)) |
126 | |
127 | static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym) |
128 | { |
129 | #ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS |
130 | return (unsigned long)offset_to_ptr(off: &sym->value_offset); |
131 | #else |
132 | return sym->value; |
133 | #endif |
134 | } |
135 | |
136 | #ifdef CONFIG_LIVEPATCH |
137 | int copy_module_elf(struct module *mod, struct load_info *info); |
138 | void free_module_elf(struct module *mod); |
139 | #else /* !CONFIG_LIVEPATCH */ |
140 | static inline int copy_module_elf(struct module *mod, struct load_info *info) |
141 | { |
142 | return 0; |
143 | } |
144 | |
145 | static inline void free_module_elf(struct module *mod) { } |
146 | #endif /* CONFIG_LIVEPATCH */ |
147 | |
148 | static inline bool set_livepatch_module(struct module *mod) |
149 | { |
150 | #ifdef CONFIG_LIVEPATCH |
151 | mod->klp = true; |
152 | return true; |
153 | #else |
154 | return false; |
155 | #endif |
156 | } |
157 | |
158 | /** |
159 | * enum fail_dup_mod_reason - state at which a duplicate module was detected |
160 | * |
161 | * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but |
162 | * we've determined that another module with the same name is already loaded |
163 | * or being processed on our &modules list. This happens on early_mod_check() |
164 | * right before layout_and_allocate(). The kernel would have already |
165 | * vmalloc()'d space for the entire module through finit_module(). If |
166 | * decompression was used two vmap() spaces were used. These failures can |
167 | * happen when userspace has not seen the module present on the kernel and |
168 | * tries to load the module multiple times at same time. |
169 | * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation |
170 | * checks and the kernel determines that the module was unique and because |
171 | * of this allocated yet another private kernel copy of the module space in |
172 | * layout_and_allocate() but after this determined in add_unformed_module() |
173 | * that another module with the same name is already loaded or being processed. |
174 | * These failures should be mitigated as much as possible and are indicative |
175 | * of really fast races in loading modules. Without module decompression |
176 | * they waste twice as much vmap space. With module decompression three |
177 | * times the module's size vmap space is wasted. |
178 | */ |
179 | enum fail_dup_mod_reason { |
180 | FAIL_DUP_MOD_BECOMING = 0, |
181 | FAIL_DUP_MOD_LOAD, |
182 | }; |
183 | |
184 | #ifdef CONFIG_MODULE_DEBUGFS |
185 | extern struct dentry *mod_debugfs_root; |
186 | #endif |
187 | |
188 | #ifdef CONFIG_MODULE_STATS |
189 | |
190 | #define mod_stat_add_long(count, var) atomic_long_add(count, var) |
191 | #define mod_stat_inc(name) atomic_inc(name) |
192 | |
193 | extern atomic_long_t total_mod_size; |
194 | extern atomic_long_t total_text_size; |
195 | extern atomic_long_t invalid_kread_bytes; |
196 | extern atomic_long_t invalid_decompress_bytes; |
197 | |
198 | extern atomic_t modcount; |
199 | extern atomic_t failed_kreads; |
200 | extern atomic_t failed_decompress; |
201 | struct mod_fail_load { |
202 | struct list_head list; |
203 | char name[MODULE_NAME_LEN]; |
204 | atomic_long_t count; |
205 | unsigned long dup_fail_mask; |
206 | }; |
207 | |
208 | int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason); |
209 | void mod_stat_bump_invalid(struct load_info *info, int flags); |
210 | void mod_stat_bump_becoming(struct load_info *info, int flags); |
211 | |
212 | #else |
213 | |
214 | #define mod_stat_add_long(name, var) |
215 | #define mod_stat_inc(name) |
216 | |
217 | static inline int try_add_failed_module(const char *name, |
218 | enum fail_dup_mod_reason reason) |
219 | { |
220 | return 0; |
221 | } |
222 | |
223 | static inline void mod_stat_bump_invalid(struct load_info *info, int flags) |
224 | { |
225 | } |
226 | |
227 | static inline void mod_stat_bump_becoming(struct load_info *info, int flags) |
228 | { |
229 | } |
230 | |
231 | #endif /* CONFIG_MODULE_STATS */ |
232 | |
233 | #ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS |
234 | bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret); |
235 | void kmod_dup_request_announce(char *module_name, int ret); |
236 | #else |
237 | static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret) |
238 | { |
239 | return false; |
240 | } |
241 | |
242 | static inline void kmod_dup_request_announce(char *module_name, int ret) |
243 | { |
244 | } |
245 | #endif |
246 | |
247 | #ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING |
248 | struct mod_unload_taint { |
249 | struct list_head list; |
250 | char name[MODULE_NAME_LEN]; |
251 | unsigned long taints; |
252 | u64 count; |
253 | }; |
254 | |
255 | int try_add_tainted_module(struct module *mod); |
256 | void print_unloaded_tainted_modules(void); |
257 | #else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */ |
258 | static inline int try_add_tainted_module(struct module *mod) |
259 | { |
260 | return 0; |
261 | } |
262 | |
263 | static inline void print_unloaded_tainted_modules(void) |
264 | { |
265 | } |
266 | #endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */ |
267 | |
268 | #ifdef CONFIG_MODULE_DECOMPRESS |
269 | int module_decompress(struct load_info *info, const void *buf, size_t size); |
270 | void module_decompress_cleanup(struct load_info *info); |
271 | #else |
272 | static inline int module_decompress(struct load_info *info, |
273 | const void *buf, size_t size) |
274 | { |
275 | return -EOPNOTSUPP; |
276 | } |
277 | |
278 | static inline void module_decompress_cleanup(struct load_info *info) |
279 | { |
280 | } |
281 | #endif |
282 | |
283 | struct mod_tree_root { |
284 | #ifdef CONFIG_MODULES_TREE_LOOKUP |
285 | struct latch_tree_root root; |
286 | #endif |
287 | unsigned long addr_min; |
288 | unsigned long addr_max; |
289 | #ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC |
290 | unsigned long data_addr_min; |
291 | unsigned long data_addr_max; |
292 | #endif |
293 | }; |
294 | |
295 | extern struct mod_tree_root mod_tree; |
296 | |
297 | #ifdef CONFIG_MODULES_TREE_LOOKUP |
298 | void mod_tree_insert(struct module *mod); |
299 | void mod_tree_remove_init(struct module *mod); |
300 | void mod_tree_remove(struct module *mod); |
301 | struct module *mod_find(unsigned long addr, struct mod_tree_root *tree); |
302 | #else /* !CONFIG_MODULES_TREE_LOOKUP */ |
303 | |
304 | static inline void mod_tree_insert(struct module *mod) { } |
305 | static inline void mod_tree_remove_init(struct module *mod) { } |
306 | static inline void mod_tree_remove(struct module *mod) { } |
307 | static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree) |
308 | { |
309 | struct module *mod; |
310 | |
311 | list_for_each_entry_rcu(mod, &modules, list, |
312 | lockdep_is_held(&module_mutex)) { |
313 | if (within_module(addr, mod)) |
314 | return mod; |
315 | } |
316 | |
317 | return NULL; |
318 | } |
319 | #endif /* CONFIG_MODULES_TREE_LOOKUP */ |
320 | |
321 | int module_enable_rodata_ro(const struct module *mod); |
322 | int module_enable_rodata_ro_after_init(const struct module *mod); |
323 | int module_enable_data_nx(const struct module *mod); |
324 | int module_enable_text_rox(const struct module *mod); |
325 | int module_enforce_rwx_sections(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs, |
326 | const char *secstrings, |
327 | const struct module *mod); |
328 | void module_mark_ro_after_init(const Elf_Ehdr *hdr, Elf_Shdr *sechdrs, |
329 | const char *secstrings); |
330 | |
331 | #ifdef CONFIG_MODULE_SIG |
332 | int module_sig_check(struct load_info *info, int flags); |
333 | #else /* !CONFIG_MODULE_SIG */ |
334 | static inline int module_sig_check(struct load_info *info, int flags) |
335 | { |
336 | return 0; |
337 | } |
338 | #endif /* !CONFIG_MODULE_SIG */ |
339 | |
340 | #ifdef CONFIG_DEBUG_KMEMLEAK |
341 | void kmemleak_load_module(const struct module *mod, const struct load_info *info); |
342 | #else /* !CONFIG_DEBUG_KMEMLEAK */ |
343 | static inline void kmemleak_load_module(const struct module *mod, |
344 | const struct load_info *info) { } |
345 | #endif /* CONFIG_DEBUG_KMEMLEAK */ |
346 | |
347 | #ifdef CONFIG_KALLSYMS |
348 | void init_build_id(struct module *mod, const struct load_info *info); |
349 | void layout_symtab(struct module *mod, struct load_info *info); |
350 | void add_kallsyms(struct module *mod, const struct load_info *info); |
351 | |
352 | static inline bool sect_empty(const Elf_Shdr *sect) |
353 | { |
354 | return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0; |
355 | } |
356 | #else /* !CONFIG_KALLSYMS */ |
357 | static inline void init_build_id(struct module *mod, const struct load_info *info) { } |
358 | static inline void layout_symtab(struct module *mod, struct load_info *info) { } |
359 | static inline void add_kallsyms(struct module *mod, const struct load_info *info) { } |
360 | #endif /* CONFIG_KALLSYMS */ |
361 | |
362 | #ifdef CONFIG_SYSFS |
363 | int mod_sysfs_setup(struct module *mod, const struct load_info *info, |
364 | struct kernel_param *kparam, unsigned int num_params); |
365 | void mod_sysfs_teardown(struct module *mod); |
366 | void init_param_lock(struct module *mod); |
367 | #else /* !CONFIG_SYSFS */ |
368 | static inline int mod_sysfs_setup(struct module *mod, |
369 | const struct load_info *info, |
370 | struct kernel_param *kparam, |
371 | unsigned int num_params) |
372 | { |
373 | return 0; |
374 | } |
375 | |
376 | static inline void mod_sysfs_teardown(struct module *mod) { } |
377 | static inline void init_param_lock(struct module *mod) { } |
378 | #endif /* CONFIG_SYSFS */ |
379 | |
380 | #ifdef CONFIG_MODVERSIONS |
381 | int check_version(const struct load_info *info, |
382 | const char *symname, struct module *mod, const u32 *crc); |
383 | void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp, |
384 | struct kernel_symbol *ks, struct tracepoint * const *tp); |
385 | int check_modstruct_version(const struct load_info *info, struct module *mod); |
386 | int same_magic(const char *amagic, const char *bmagic, bool has_crcs); |
387 | struct modversion_info_ext { |
388 | size_t remaining; |
389 | const u32 *crc; |
390 | const char *name; |
391 | }; |
392 | void modversion_ext_start(const struct load_info *info, struct modversion_info_ext *ver); |
393 | void modversion_ext_advance(struct modversion_info_ext *ver); |
394 | #define for_each_modversion_info_ext(ver, info) \ |
395 | for (modversion_ext_start(info, &ver); ver.remaining > 0; modversion_ext_advance(&ver)) |
396 | #else /* !CONFIG_MODVERSIONS */ |
397 | static inline int check_version(const struct load_info *info, |
398 | const char *symname, |
399 | struct module *mod, |
400 | const u32 *crc) |
401 | { |
402 | return 1; |
403 | } |
404 | |
405 | static inline int check_modstruct_version(const struct load_info *info, |
406 | struct module *mod) |
407 | { |
408 | return 1; |
409 | } |
410 | |
411 | static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs) |
412 | { |
413 | return strcmp(amagic, bmagic) == 0; |
414 | } |
415 | #endif /* CONFIG_MODVERSIONS */ |
416 | |