1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/* Module internals
3 *
4 * Copyright (C) 2012 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 * Copyright (C) 2023 Luis Chamberlain <mcgrof@kernel.org>
7 */
8
9#include <linux/elf.h>
10#include <linux/compiler.h>
11#include <linux/module.h>
12#include <linux/mutex.h>
13#include <linux/rculist.h>
14#include <linux/rcupdate.h>
15#include <linux/mm.h>
16
17#ifndef ARCH_SHF_SMALL
18#define ARCH_SHF_SMALL 0
19#endif
20
21/*
22 * Use highest 4 bits of sh_entsize to store the mod_mem_type of this
23 * section. This leaves 28 bits for offset on 32-bit systems, which is
24 * about 256 MiB (WARN_ON_ONCE if we exceed that).
25 */
26
27#define SH_ENTSIZE_TYPE_BITS 4
28#define SH_ENTSIZE_TYPE_SHIFT (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)
29#define SH_ENTSIZE_TYPE_MASK ((1UL << SH_ENTSIZE_TYPE_BITS) - 1)
30#define SH_ENTSIZE_OFFSET_MASK ((1UL << (BITS_PER_LONG - SH_ENTSIZE_TYPE_BITS)) - 1)
31
32/* Maximum number of characters written by module_flags() */
33#define MODULE_FLAGS_BUF_SIZE (TAINT_FLAGS_COUNT + 4)
34
35struct kernel_symbol {
36#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
37 int value_offset;
38 int name_offset;
39 int namespace_offset;
40#else
41 unsigned long value;
42 const char *name;
43 const char *namespace;
44#endif
45};
46
47extern struct mutex module_mutex;
48extern struct list_head modules;
49
50extern const struct module_attribute *const modinfo_attrs[];
51extern const size_t modinfo_attrs_count;
52
53/* Provided by the linker */
54extern const struct kernel_symbol __start___ksymtab[];
55extern const struct kernel_symbol __stop___ksymtab[];
56extern const struct kernel_symbol __start___ksymtab_gpl[];
57extern const struct kernel_symbol __stop___ksymtab_gpl[];
58extern const u32 __start___kcrctab[];
59extern const u32 __start___kcrctab_gpl[];
60
61struct load_info {
62 const char *name;
63 /* pointer to module in temporary copy, freed at end of load_module() */
64 struct module *mod;
65 Elf_Ehdr *hdr;
66 unsigned long len;
67 Elf_Shdr *sechdrs;
68 char *secstrings, *strtab;
69 unsigned long symoffs, stroffs, init_typeoffs, core_typeoffs;
70 bool sig_ok;
71#ifdef CONFIG_KALLSYMS
72 unsigned long mod_kallsyms_init_off;
73#endif
74#ifdef CONFIG_MODULE_DECOMPRESS
75#ifdef CONFIG_MODULE_STATS
76 unsigned long compressed_len;
77#endif
78 struct page **pages;
79 unsigned int max_pages;
80 unsigned int used_pages;
81#endif
82 struct {
83 unsigned int sym;
84 unsigned int str;
85 unsigned int mod;
86 unsigned int vers;
87 unsigned int info;
88 unsigned int pcpu;
89 unsigned int vers_ext_crc;
90 unsigned int vers_ext_name;
91 } index;
92};
93
94enum mod_license {
95 NOT_GPL_ONLY,
96 GPL_ONLY,
97};
98
99struct find_symbol_arg {
100 /* Input */
101 const char *name;
102 bool gplok;
103 bool warn;
104
105 /* Output */
106 struct module *owner;
107 const u32 *crc;
108 const struct kernel_symbol *sym;
109 enum mod_license license;
110};
111
112int mod_verify_sig(const void *mod, struct load_info *info);
113int try_to_force_load(struct module *mod, const char *reason);
114bool find_symbol(struct find_symbol_arg *fsa);
115struct module *find_module_all(const char *name, size_t len, bool even_unformed);
116int cmp_name(const void *name, const void *sym);
117long module_get_offset_and_type(struct module *mod, enum mod_mem_type type,
118 Elf_Shdr *sechdr, unsigned int section);
119char *module_flags(struct module *mod, char *buf, bool show_state);
120size_t module_flags_taint(unsigned long taints, char *buf);
121
122char *module_next_tag_pair(char *string, unsigned long *secsize);
123
124#define for_each_modinfo_entry(entry, info, name) \
125 for (entry = get_modinfo(info, name); entry; entry = get_next_modinfo(info, name, entry))
126
127static inline unsigned long kernel_symbol_value(const struct kernel_symbol *sym)
128{
129#ifdef CONFIG_HAVE_ARCH_PREL32_RELOCATIONS
130 return (unsigned long)offset_to_ptr(off: &sym->value_offset);
131#else
132 return sym->value;
133#endif
134}
135
136#ifdef CONFIG_LIVEPATCH
137int copy_module_elf(struct module *mod, struct load_info *info);
138void free_module_elf(struct module *mod);
139#else /* !CONFIG_LIVEPATCH */
140static inline int copy_module_elf(struct module *mod, struct load_info *info)
141{
142 return 0;
143}
144
145static inline void free_module_elf(struct module *mod) { }
146#endif /* CONFIG_LIVEPATCH */
147
148static inline bool set_livepatch_module(struct module *mod)
149{
150#ifdef CONFIG_LIVEPATCH
151 mod->klp = true;
152 return true;
153#else
154 return false;
155#endif
156}
157
158/**
159 * enum fail_dup_mod_reason - state at which a duplicate module was detected
160 *
161 * @FAIL_DUP_MOD_BECOMING: the module is read properly, passes all checks but
162 * we've determined that another module with the same name is already loaded
163 * or being processed on our &modules list. This happens on early_mod_check()
164 * right before layout_and_allocate(). The kernel would have already
165 * vmalloc()'d space for the entire module through finit_module(). If
166 * decompression was used two vmap() spaces were used. These failures can
167 * happen when userspace has not seen the module present on the kernel and
168 * tries to load the module multiple times at same time.
169 * @FAIL_DUP_MOD_LOAD: the module has been read properly, passes all validation
170 * checks and the kernel determines that the module was unique and because
171 * of this allocated yet another private kernel copy of the module space in
172 * layout_and_allocate() but after this determined in add_unformed_module()
173 * that another module with the same name is already loaded or being processed.
174 * These failures should be mitigated as much as possible and are indicative
175 * of really fast races in loading modules. Without module decompression
176 * they waste twice as much vmap space. With module decompression three
177 * times the module's size vmap space is wasted.
178 */
179enum fail_dup_mod_reason {
180 FAIL_DUP_MOD_BECOMING = 0,
181 FAIL_DUP_MOD_LOAD,
182};
183
184#ifdef CONFIG_MODULE_DEBUGFS
185extern struct dentry *mod_debugfs_root;
186#endif
187
188#ifdef CONFIG_MODULE_STATS
189
190#define mod_stat_add_long(count, var) atomic_long_add(count, var)
191#define mod_stat_inc(name) atomic_inc(name)
192
193extern atomic_long_t total_mod_size;
194extern atomic_long_t total_text_size;
195extern atomic_long_t invalid_kread_bytes;
196extern atomic_long_t invalid_decompress_bytes;
197
198extern atomic_t modcount;
199extern atomic_t failed_kreads;
200extern atomic_t failed_decompress;
201struct mod_fail_load {
202 struct list_head list;
203 char name[MODULE_NAME_LEN];
204 atomic_long_t count;
205 unsigned long dup_fail_mask;
206};
207
208int try_add_failed_module(const char *name, enum fail_dup_mod_reason reason);
209void mod_stat_bump_invalid(struct load_info *info, int flags);
210void mod_stat_bump_becoming(struct load_info *info, int flags);
211
212#else
213
214#define mod_stat_add_long(name, var)
215#define mod_stat_inc(name)
216
217static inline int try_add_failed_module(const char *name,
218 enum fail_dup_mod_reason reason)
219{
220 return 0;
221}
222
223static inline void mod_stat_bump_invalid(struct load_info *info, int flags)
224{
225}
226
227static inline void mod_stat_bump_becoming(struct load_info *info, int flags)
228{
229}
230
231#endif /* CONFIG_MODULE_STATS */
232
233#ifdef CONFIG_MODULE_DEBUG_AUTOLOAD_DUPS
234bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret);
235void kmod_dup_request_announce(char *module_name, int ret);
236#else
237static inline bool kmod_dup_request_exists_wait(char *module_name, bool wait, int *dup_ret)
238{
239 return false;
240}
241
242static inline void kmod_dup_request_announce(char *module_name, int ret)
243{
244}
245#endif
246
247#ifdef CONFIG_MODULE_UNLOAD_TAINT_TRACKING
248struct mod_unload_taint {
249 struct list_head list;
250 char name[MODULE_NAME_LEN];
251 unsigned long taints;
252 u64 count;
253};
254
255int try_add_tainted_module(struct module *mod);
256void print_unloaded_tainted_modules(void);
257#else /* !CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
258static inline int try_add_tainted_module(struct module *mod)
259{
260 return 0;
261}
262
263static inline void print_unloaded_tainted_modules(void)
264{
265}
266#endif /* CONFIG_MODULE_UNLOAD_TAINT_TRACKING */
267
268#ifdef CONFIG_MODULE_DECOMPRESS
269int module_decompress(struct load_info *info, const void *buf, size_t size);
270void module_decompress_cleanup(struct load_info *info);
271#else
272static inline int module_decompress(struct load_info *info,
273 const void *buf, size_t size)
274{
275 return -EOPNOTSUPP;
276}
277
278static inline void module_decompress_cleanup(struct load_info *info)
279{
280}
281#endif
282
283struct mod_tree_root {
284#ifdef CONFIG_MODULES_TREE_LOOKUP
285 struct latch_tree_root root;
286#endif
287 unsigned long addr_min;
288 unsigned long addr_max;
289#ifdef CONFIG_ARCH_WANTS_MODULES_DATA_IN_VMALLOC
290 unsigned long data_addr_min;
291 unsigned long data_addr_max;
292#endif
293};
294
295extern struct mod_tree_root mod_tree;
296
297#ifdef CONFIG_MODULES_TREE_LOOKUP
298void mod_tree_insert(struct module *mod);
299void mod_tree_remove_init(struct module *mod);
300void mod_tree_remove(struct module *mod);
301struct module *mod_find(unsigned long addr, struct mod_tree_root *tree);
302#else /* !CONFIG_MODULES_TREE_LOOKUP */
303
304static inline void mod_tree_insert(struct module *mod) { }
305static inline void mod_tree_remove_init(struct module *mod) { }
306static inline void mod_tree_remove(struct module *mod) { }
307static inline struct module *mod_find(unsigned long addr, struct mod_tree_root *tree)
308{
309 struct module *mod;
310
311 list_for_each_entry_rcu(mod, &modules, list,
312 lockdep_is_held(&module_mutex)) {
313 if (within_module(addr, mod))
314 return mod;
315 }
316
317 return NULL;
318}
319#endif /* CONFIG_MODULES_TREE_LOOKUP */
320
321int module_enable_rodata_ro(const struct module *mod);
322int module_enable_rodata_ro_after_init(const struct module *mod);
323int module_enable_data_nx(const struct module *mod);
324int module_enable_text_rox(const struct module *mod);
325int module_enforce_rwx_sections(const Elf_Ehdr *hdr, const Elf_Shdr *sechdrs,
326 const char *secstrings,
327 const struct module *mod);
328void module_mark_ro_after_init(const Elf_Ehdr *hdr, Elf_Shdr *sechdrs,
329 const char *secstrings);
330
331#ifdef CONFIG_MODULE_SIG
332int module_sig_check(struct load_info *info, int flags);
333#else /* !CONFIG_MODULE_SIG */
334static inline int module_sig_check(struct load_info *info, int flags)
335{
336 return 0;
337}
338#endif /* !CONFIG_MODULE_SIG */
339
340#ifdef CONFIG_DEBUG_KMEMLEAK
341void kmemleak_load_module(const struct module *mod, const struct load_info *info);
342#else /* !CONFIG_DEBUG_KMEMLEAK */
343static inline void kmemleak_load_module(const struct module *mod,
344 const struct load_info *info) { }
345#endif /* CONFIG_DEBUG_KMEMLEAK */
346
347#ifdef CONFIG_KALLSYMS
348void init_build_id(struct module *mod, const struct load_info *info);
349void layout_symtab(struct module *mod, struct load_info *info);
350void add_kallsyms(struct module *mod, const struct load_info *info);
351
352static inline bool sect_empty(const Elf_Shdr *sect)
353{
354 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
355}
356#else /* !CONFIG_KALLSYMS */
357static inline void init_build_id(struct module *mod, const struct load_info *info) { }
358static inline void layout_symtab(struct module *mod, struct load_info *info) { }
359static inline void add_kallsyms(struct module *mod, const struct load_info *info) { }
360#endif /* CONFIG_KALLSYMS */
361
362#ifdef CONFIG_SYSFS
363int mod_sysfs_setup(struct module *mod, const struct load_info *info,
364 struct kernel_param *kparam, unsigned int num_params);
365void mod_sysfs_teardown(struct module *mod);
366void init_param_lock(struct module *mod);
367#else /* !CONFIG_SYSFS */
368static inline int mod_sysfs_setup(struct module *mod,
369 const struct load_info *info,
370 struct kernel_param *kparam,
371 unsigned int num_params)
372{
373 return 0;
374}
375
376static inline void mod_sysfs_teardown(struct module *mod) { }
377static inline void init_param_lock(struct module *mod) { }
378#endif /* CONFIG_SYSFS */
379
380#ifdef CONFIG_MODVERSIONS
381int check_version(const struct load_info *info,
382 const char *symname, struct module *mod, const u32 *crc);
383void module_layout(struct module *mod, struct modversion_info *ver, struct kernel_param *kp,
384 struct kernel_symbol *ks, struct tracepoint * const *tp);
385int check_modstruct_version(const struct load_info *info, struct module *mod);
386int same_magic(const char *amagic, const char *bmagic, bool has_crcs);
387struct modversion_info_ext {
388 size_t remaining;
389 const u32 *crc;
390 const char *name;
391};
392void modversion_ext_start(const struct load_info *info, struct modversion_info_ext *ver);
393void modversion_ext_advance(struct modversion_info_ext *ver);
394#define for_each_modversion_info_ext(ver, info) \
395 for (modversion_ext_start(info, &ver); ver.remaining > 0; modversion_ext_advance(&ver))
396#else /* !CONFIG_MODVERSIONS */
397static inline int check_version(const struct load_info *info,
398 const char *symname,
399 struct module *mod,
400 const u32 *crc)
401{
402 return 1;
403}
404
405static inline int check_modstruct_version(const struct load_info *info,
406 struct module *mod)
407{
408 return 1;
409}
410
411static inline int same_magic(const char *amagic, const char *bmagic, bool has_crcs)
412{
413 return strcmp(amagic, bmagic) == 0;
414}
415#endif /* CONFIG_MODVERSIONS */
416

source code of linux/kernel/module/internal.h