blob: b14a4f31221f36bf5af471d7dcfc7935f0f68375 [file] [log] [blame]
Arjan van de Venf71d20e2006-06-28 04:26:45 -07001/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 Copyright (C) 2002 Richard Henderson
Rusty Russell51f3d0f2010-08-05 12:59:13 -06003 Copyright (C) 2001 Rusty Russell, 2002, 2010 Rusty Russell IBM.
Linus Torvalds1da177e2005-04-16 15:20:36 -07004
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published by
7 the Free Software Foundation; either version 2 of the License, or
8 (at your option) any later version.
9
10 This program is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 GNU General Public License for more details.
14
15 You should have received a copy of the GNU General Public License
16 along with this program; if not, write to the Free Software
17 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18*/
Paul Gortmaker9984de12011-05-23 14:51:41 -040019#include <linux/export.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/moduleloader.h>
Steven Rostedt (Red Hat)af658dc2015-04-29 14:36:05 -040021#include <linux/trace_events.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070022#include <linux/init.h>
Alexey Dobriyanae84e322007-05-08 00:28:38 -070023#include <linux/kallsyms.h>
Kees Cook34e11692012-10-16 07:31:07 +103024#include <linux/file.h>
Alexey Dobriyan3b5d5c62008-10-06 13:19:27 +040025#include <linux/fs.h>
Roland McGrath6d760132007-10-16 23:26:40 -070026#include <linux/sysfs.h>
Randy Dunlap9f158332005-09-13 01:25:16 -070027#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/slab.h>
29#include <linux/vmalloc.h>
30#include <linux/elf.h>
Alexey Dobriyan3b5d5c62008-10-06 13:19:27 +040031#include <linux/proc_fs.h>
Kees Cook2e72d512012-10-16 07:32:07 +103032#include <linux/security.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#include <linux/seq_file.h>
34#include <linux/syscalls.h>
35#include <linux/fcntl.h>
36#include <linux/rcupdate.h>
Randy.Dunlapc59ede72006-01-11 12:17:46 -080037#include <linux/capability.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070038#include <linux/cpu.h>
39#include <linux/moduleparam.h>
40#include <linux/errno.h>
41#include <linux/err.h>
42#include <linux/vermagic.h>
43#include <linux/notifier.h>
Al Virof6a57032006-10-18 01:47:25 -040044#include <linux/sched.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070045#include <linux/device.h>
Matt Domschc988d2b2005-06-23 22:05:15 -070046#include <linux/string.h>
Arjan van de Ven97d1f152006-03-23 03:00:24 -080047#include <linux/mutex.h>
Andi Kleend72b3752008-08-30 10:09:00 +020048#include <linux/rculist.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070049#include <asm/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050#include <asm/cacheflush.h>
Bernd Schmidteb8cdec2009-09-21 17:03:57 -070051#include <asm/mmu_context.h>
Sam Ravnborgb817f6f2006-06-09 21:53:55 +020052#include <linux/license.h>
Christoph Lameter6d762392008-02-08 04:18:42 -080053#include <asm/sections.h>
Mathieu Desnoyers97e1c182008-07-18 12:16:16 -040054#include <linux/tracepoint.h>
Steven Rostedt90d595f2008-08-14 15:45:09 -040055#include <linux/ftrace.h>
Arjan van de Ven22a9d642009-01-07 08:45:46 -080056#include <linux/async.h>
Tejun Heofbf59bc2009-02-20 16:29:08 +090057#include <linux/percpu.h>
Catalin Marinas4f2294b2009-06-11 13:23:20 +010058#include <linux/kmemleak.h>
Jason Baronbf5438fc2010-09-17 11:09:00 -040059#include <linux/jump_label.h>
matthieu castet84e1c6b2010-11-16 22:35:16 +010060#include <linux/pfn.h>
Alessio Igor Bogani403ed272011-04-20 11:10:52 +020061#include <linux/bsearch.h>
Rusty Russell2f3238a2012-10-22 18:09:41 +103062#include <uapi/linux/module.h>
Rusty Russell106a4ee2012-09-26 10:09:40 +010063#include "module-internal.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070064
Li Zefan7ead8b82009-08-17 16:56:28 +080065#define CREATE_TRACE_POINTS
66#include <trace/events/module.h>
67
Linus Torvalds1da177e2005-04-16 15:20:36 -070068#ifndef ARCH_SHF_SMALL
69#define ARCH_SHF_SMALL 0
70#endif
71
matthieu castet84e1c6b2010-11-16 22:35:16 +010072/*
73 * Modules' sections will be aligned on page boundaries
74 * to ensure complete separation of code and data, but
75 * only when CONFIG_DEBUG_SET_MODULE_RONX=y
76 */
77#ifdef CONFIG_DEBUG_SET_MODULE_RONX
78# define debug_align(X) ALIGN(X, PAGE_SIZE)
79#else
80# define debug_align(X) (X)
81#endif
82
83/*
84 * Given BASE and SIZE this macro calculates the number of pages the
85 * memory regions occupies
86 */
87#define MOD_NUMBER_OF_PAGES(BASE, SIZE) (((SIZE) > 0) ? \
88 (PFN_DOWN((unsigned long)(BASE) + (SIZE) - 1) - \
89 PFN_DOWN((unsigned long)BASE) + 1) \
90 : (0UL))
91
Linus Torvalds1da177e2005-04-16 15:20:36 -070092/* If this is set, the section belongs in the init part of the module */
93#define INIT_OFFSET_MASK (1UL << (BITS_PER_LONG-1))
94
Rusty Russell75676502010-06-05 11:17:36 -060095/*
96 * Mutex protects:
97 * 1) List of modules (also safely readable with preempt_disable),
98 * 2) module_use links,
99 * 3) module_addr_min/module_addr_max.
Masami Hiramatsue513cc12014-11-10 09:30:29 +1030100 * (delete and add uses RCU list operations). */
Tim Abbottc6b37802008-12-05 19:03:59 -0500101DEFINE_MUTEX(module_mutex);
102EXPORT_SYMBOL_GPL(module_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700103static LIST_HEAD(modules);
Peter Zijlstra93c2e102015-05-27 11:09:37 +0930104
Peter Zijlstra6c9692e2015-05-27 11:09:37 +0930105#ifdef CONFIG_MODULES_TREE_LOOKUP
106
Peter Zijlstra93c2e102015-05-27 11:09:37 +0930107/*
108 * Use a latched RB-tree for __module_address(); this allows us to use
109 * RCU-sched lookups of the address from any context.
110 *
111 * Because modules have two address ranges: init and core, we need two
112 * latch_tree_nodes entries. Therefore we need the back-pointer from
113 * mod_tree_node.
114 *
115 * Because init ranges are short lived we mark them unlikely and have placed
116 * them outside the critical cacheline in struct module.
Peter Zijlstra6c9692e2015-05-27 11:09:37 +0930117 *
118 * This is conditional on PERF_EVENTS || TRACING because those can really hit
119 * __module_address() hard by doing a lot of stack unwinding; potentially from
120 * NMI context.
Peter Zijlstra93c2e102015-05-27 11:09:37 +0930121 */
122
123static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n)
124{
125 struct mod_tree_node *mtn = container_of(n, struct mod_tree_node, node);
126 struct module *mod = mtn->mod;
127
128 if (unlikely(mtn == &mod->mtn_init))
129 return (unsigned long)mod->module_init;
130
131 return (unsigned long)mod->module_core;
132}
133
134static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n)
135{
136 struct mod_tree_node *mtn = container_of(n, struct mod_tree_node, node);
137 struct module *mod = mtn->mod;
138
139 if (unlikely(mtn == &mod->mtn_init))
140 return (unsigned long)mod->init_size;
141
142 return (unsigned long)mod->core_size;
143}
144
145static __always_inline bool
146mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b)
147{
148 return __mod_tree_val(a) < __mod_tree_val(b);
149}
150
151static __always_inline int
152mod_tree_comp(void *key, struct latch_tree_node *n)
153{
154 unsigned long val = (unsigned long)key;
155 unsigned long start, end;
156
157 start = __mod_tree_val(n);
158 if (val < start)
159 return -1;
160
161 end = start + __mod_tree_size(n);
162 if (val >= end)
163 return 1;
164
165 return 0;
166}
167
168static const struct latch_tree_ops mod_tree_ops = {
169 .less = mod_tree_less,
170 .comp = mod_tree_comp,
171};
172
Peter Zijlstra4f6665462015-05-27 11:09:38 +0930173static struct mod_tree_root {
174 struct latch_tree_root root;
175 unsigned long addr_min;
176 unsigned long addr_max;
177} mod_tree __cacheline_aligned = {
178 .addr_min = -1UL,
179};
180
181#define module_addr_min mod_tree.addr_min
182#define module_addr_max mod_tree.addr_max
183
184static noinline void __mod_tree_insert(struct mod_tree_node *node)
185{
186 latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops);
187}
188
189static void __mod_tree_remove(struct mod_tree_node *node)
190{
191 latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops);
192}
Peter Zijlstra93c2e102015-05-27 11:09:37 +0930193
194/*
195 * These modifications: insert, remove_init and remove; are serialized by the
196 * module_mutex.
197 */
198static void mod_tree_insert(struct module *mod)
199{
200 mod->mtn_core.mod = mod;
201 mod->mtn_init.mod = mod;
202
Peter Zijlstra4f6665462015-05-27 11:09:38 +0930203 __mod_tree_insert(&mod->mtn_core);
Peter Zijlstra93c2e102015-05-27 11:09:37 +0930204 if (mod->init_size)
Peter Zijlstra4f6665462015-05-27 11:09:38 +0930205 __mod_tree_insert(&mod->mtn_init);
Peter Zijlstra93c2e102015-05-27 11:09:37 +0930206}
207
208static void mod_tree_remove_init(struct module *mod)
209{
210 if (mod->init_size)
Peter Zijlstra4f6665462015-05-27 11:09:38 +0930211 __mod_tree_remove(&mod->mtn_init);
Peter Zijlstra93c2e102015-05-27 11:09:37 +0930212}
213
214static void mod_tree_remove(struct module *mod)
215{
Peter Zijlstra4f6665462015-05-27 11:09:38 +0930216 __mod_tree_remove(&mod->mtn_core);
Peter Zijlstra93c2e102015-05-27 11:09:37 +0930217 mod_tree_remove_init(mod);
218}
219
Peter Zijlstra6c9692e2015-05-27 11:09:37 +0930220static struct module *mod_find(unsigned long addr)
Peter Zijlstra93c2e102015-05-27 11:09:37 +0930221{
222 struct latch_tree_node *ltn;
223
Peter Zijlstra4f6665462015-05-27 11:09:38 +0930224 ltn = latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops);
Peter Zijlstra93c2e102015-05-27 11:09:37 +0930225 if (!ltn)
226 return NULL;
227
228 return container_of(ltn, struct mod_tree_node, node)->mod;
229}
230
Peter Zijlstra6c9692e2015-05-27 11:09:37 +0930231#else /* MODULES_TREE_LOOKUP */
232
Peter Zijlstra4f6665462015-05-27 11:09:38 +0930233static unsigned long module_addr_min = -1UL, module_addr_max = 0;
234
Peter Zijlstra6c9692e2015-05-27 11:09:37 +0930235static void mod_tree_insert(struct module *mod) { }
236static void mod_tree_remove_init(struct module *mod) { }
237static void mod_tree_remove(struct module *mod) { }
238
239static struct module *mod_find(unsigned long addr)
240{
241 struct module *mod;
242
243 list_for_each_entry_rcu(mod, &modules, list) {
244 if (within_module(addr, mod))
245 return mod;
246 }
247
248 return NULL;
249}
250
251#endif /* MODULES_TREE_LOOKUP */
252
Peter Zijlstra4f6665462015-05-27 11:09:38 +0930253/*
254 * Bounds of module text, for speeding up __module_address.
255 * Protected by module_mutex.
256 */
257static void __mod_update_bounds(void *base, unsigned int size)
258{
259 unsigned long min = (unsigned long)base;
260 unsigned long max = min + size;
261
262 if (min < module_addr_min)
263 module_addr_min = min;
264 if (max > module_addr_max)
265 module_addr_max = max;
266}
267
268static void mod_update_bounds(struct module *mod)
269{
270 __mod_update_bounds(mod->module_core, mod->core_size);
271 if (mod->init_size)
272 __mod_update_bounds(mod->module_init, mod->init_size);
273}
274
Jason Wessel67fc4e02010-05-20 21:04:21 -0500275#ifdef CONFIG_KGDB_KDB
276struct list_head *kdb_modules = &modules; /* kdb needs the list of modules */
277#endif /* CONFIG_KGDB_KDB */
278
Peter Zijlstra0be964b2015-05-27 11:09:35 +0930279static void module_assert_mutex(void)
Rusty Russell106a4ee2012-09-26 10:09:40 +0100280{
Peter Zijlstra0be964b2015-05-27 11:09:35 +0930281 lockdep_assert_held(&module_mutex);
Rusty Russell106a4ee2012-09-26 10:09:40 +0100282}
283
Peter Zijlstra0be964b2015-05-27 11:09:35 +0930284static void module_assert_mutex_or_preempt(void)
285{
286#ifdef CONFIG_LOCKDEP
287 if (unlikely(!debug_locks))
288 return;
Rusty Russell106a4ee2012-09-26 10:09:40 +0100289
Peter Zijlstra0be964b2015-05-27 11:09:35 +0930290 WARN_ON(!rcu_read_lock_sched_held() &&
291 !lockdep_is_held(&module_mutex));
292#endif
293}
294
Luis R. Rodriguez6727bb92015-05-27 11:09:39 +0930295static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE);
296#ifndef CONFIG_MODULE_SIG_FORCE
Rusty Russell106a4ee2012-09-26 10:09:40 +0100297module_param(sig_enforce, bool_enable_only, 0644);
298#endif /* !CONFIG_MODULE_SIG_FORCE */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700299
Stephen Rothwell19e45292009-04-14 17:27:18 +1000300/* Block module loading/unloading? */
301int modules_disabled = 0;
Dave Young02608be2012-02-01 10:33:14 +0800302core_param(nomodule, modules_disabled, bint, 0);
Stephen Rothwell19e45292009-04-14 17:27:18 +1000303
Rusty Russellc9a3ba52008-01-29 17:13:18 -0500304/* Waiting for a module to finish initializing? */
305static DECLARE_WAIT_QUEUE_HEAD(module_wq);
306
Alan Sterne041c682006-03-27 01:16:30 -0800307static BLOCKING_NOTIFIER_HEAD(module_notify_list);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308
Ionut Alexa6da0b562014-11-10 09:31:29 +1030309int register_module_notifier(struct notifier_block *nb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700310{
Alan Sterne041c682006-03-27 01:16:30 -0800311 return blocking_notifier_chain_register(&module_notify_list, nb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312}
313EXPORT_SYMBOL(register_module_notifier);
314
Ionut Alexa6da0b562014-11-10 09:31:29 +1030315int unregister_module_notifier(struct notifier_block *nb)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700316{
Alan Sterne041c682006-03-27 01:16:30 -0800317 return blocking_notifier_chain_unregister(&module_notify_list, nb);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318}
319EXPORT_SYMBOL(unregister_module_notifier);
320
Rusty Russelleded41c2010-08-05 12:59:07 -0600321struct load_info {
322 Elf_Ehdr *hdr;
323 unsigned long len;
324 Elf_Shdr *sechdrs;
Rusty Russell6526c532010-08-05 12:59:10 -0600325 char *secstrings, *strtab;
Rusty Russelld9131882010-08-05 12:59:08 -0600326 unsigned long symoffs, stroffs;
Rusty Russell811d66a2010-08-05 12:59:12 -0600327 struct _ddebug *debug;
328 unsigned int num_debug;
Rusty Russell106a4ee2012-09-26 10:09:40 +0100329 bool sig_ok;
Rusty Russell610dde52016-02-03 16:55:26 +1030330#ifdef CONFIG_KALLSYMS
331 unsigned long mod_kallsyms_init_off;
332#endif
Rusty Russelleded41c2010-08-05 12:59:07 -0600333 struct {
334 unsigned int sym, str, mod, vers, info, pcpu;
335 } index;
336};
337
Matti Linnanvuori9a4b9702007-11-08 08:37:38 -0800338/* We require a truly strong try_module_get(): 0 means failure due to
339 ongoing or failed initialization etc. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340static inline int strong_try_module_get(struct module *mod)
341{
Rusty Russell0d21b0e2013-01-12 11:38:44 +1030342 BUG_ON(mod && mod->state == MODULE_STATE_UNFORMED);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 if (mod && mod->state == MODULE_STATE_COMING)
Rusty Russellc9a3ba52008-01-29 17:13:18 -0500344 return -EBUSY;
345 if (try_module_get(mod))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346 return 0;
Rusty Russellc9a3ba52008-01-29 17:13:18 -0500347 else
348 return -ENOENT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700349}
350
Rusty Russell373d4d02013-01-21 17:17:39 +1030351static inline void add_taint_module(struct module *mod, unsigned flag,
352 enum lockdep_ok lockdep_ok)
Florin Malitafa3ba2e82006-10-11 01:21:48 -0700353{
Rusty Russell373d4d02013-01-21 17:17:39 +1030354 add_taint(flag, lockdep_ok);
Andi Kleen25ddbb12008-10-15 22:01:41 -0700355 mod->taints |= (1U << flag);
Florin Malitafa3ba2e82006-10-11 01:21:48 -0700356}
357
Robert P. J. Day02a3e592007-05-09 07:26:28 +0200358/*
359 * A thread that wants to hold a reference to a module only while it
360 * is running can call this to safely exit. nfsd and lockd use this.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700361 */
362void __module_put_and_exit(struct module *mod, long code)
363{
364 module_put(mod);
365 do_exit(code);
366}
367EXPORT_SYMBOL(__module_put_and_exit);
Daniel Walker22a8bde2007-10-18 03:06:07 -0700368
Linus Torvalds1da177e2005-04-16 15:20:36 -0700369/* Find a module section: 0 means not found. */
Rusty Russell49668682010-08-05 12:59:10 -0600370static unsigned int find_sec(const struct load_info *info, const char *name)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371{
372 unsigned int i;
373
Rusty Russell49668682010-08-05 12:59:10 -0600374 for (i = 1; i < info->hdr->e_shnum; i++) {
375 Elf_Shdr *shdr = &info->sechdrs[i];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700376 /* Alloc bit cleared means "ignore it." */
Rusty Russell49668682010-08-05 12:59:10 -0600377 if ((shdr->sh_flags & SHF_ALLOC)
378 && strcmp(info->secstrings + shdr->sh_name, name) == 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 return i;
Rusty Russell49668682010-08-05 12:59:10 -0600380 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381 return 0;
382}
383
Rusty Russell5e458cc2008-10-22 10:00:13 -0500384/* Find a module section, or NULL. */
Rusty Russell49668682010-08-05 12:59:10 -0600385static void *section_addr(const struct load_info *info, const char *name)
Rusty Russell5e458cc2008-10-22 10:00:13 -0500386{
387 /* Section 0 has sh_addr 0. */
Rusty Russell49668682010-08-05 12:59:10 -0600388 return (void *)info->sechdrs[find_sec(info, name)].sh_addr;
Rusty Russell5e458cc2008-10-22 10:00:13 -0500389}
390
391/* Find a module section, or NULL. Fill in number of "objects" in section. */
Rusty Russell49668682010-08-05 12:59:10 -0600392static void *section_objs(const struct load_info *info,
Rusty Russell5e458cc2008-10-22 10:00:13 -0500393 const char *name,
394 size_t object_size,
395 unsigned int *num)
396{
Rusty Russell49668682010-08-05 12:59:10 -0600397 unsigned int sec = find_sec(info, name);
Rusty Russell5e458cc2008-10-22 10:00:13 -0500398
399 /* Section 0 has sh_addr 0 and sh_size 0. */
Rusty Russell49668682010-08-05 12:59:10 -0600400 *num = info->sechdrs[sec].sh_size / object_size;
401 return (void *)info->sechdrs[sec].sh_addr;
Rusty Russell5e458cc2008-10-22 10:00:13 -0500402}
403
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404/* Provided by the linker */
405extern const struct kernel_symbol __start___ksymtab[];
406extern const struct kernel_symbol __stop___ksymtab[];
407extern const struct kernel_symbol __start___ksymtab_gpl[];
408extern const struct kernel_symbol __stop___ksymtab_gpl[];
Greg Kroah-Hartman9f28bb72006-03-20 13:17:13 -0800409extern const struct kernel_symbol __start___ksymtab_gpl_future[];
410extern const struct kernel_symbol __stop___ksymtab_gpl_future[];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700411extern const unsigned long __start___kcrctab[];
412extern const unsigned long __start___kcrctab_gpl[];
Greg Kroah-Hartman9f28bb72006-03-20 13:17:13 -0800413extern const unsigned long __start___kcrctab_gpl_future[];
Denys Vlasenkof7f5b672008-07-22 19:24:26 -0500414#ifdef CONFIG_UNUSED_SYMBOLS
415extern const struct kernel_symbol __start___ksymtab_unused[];
416extern const struct kernel_symbol __stop___ksymtab_unused[];
417extern const struct kernel_symbol __start___ksymtab_unused_gpl[];
418extern const struct kernel_symbol __stop___ksymtab_unused_gpl[];
Arjan van de Venf71d20e2006-06-28 04:26:45 -0700419extern const unsigned long __start___kcrctab_unused[];
420extern const unsigned long __start___kcrctab_unused_gpl[];
Denys Vlasenkof7f5b672008-07-22 19:24:26 -0500421#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700422
423#ifndef CONFIG_MODVERSIONS
424#define symversion(base, idx) NULL
425#else
Andrew Mortonf83ca9f2006-03-28 01:56:20 -0800426#define symversion(base, idx) ((base != NULL) ? ((base) + (idx)) : NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700427#endif
428
Rusty Russelldafd0942008-07-22 19:24:25 -0500429static bool each_symbol_in_section(const struct symsearch *arr,
430 unsigned int arrsize,
431 struct module *owner,
432 bool (*fn)(const struct symsearch *syms,
433 struct module *owner,
Rusty Russellde4d8d52011-04-19 21:49:58 +0200434 void *data),
Rusty Russelldafd0942008-07-22 19:24:25 -0500435 void *data)
Sam Ravnborg3fd68052006-02-08 21:16:45 +0100436{
Rusty Russellde4d8d52011-04-19 21:49:58 +0200437 unsigned int j;
Rusty Russelldafd0942008-07-22 19:24:25 -0500438
439 for (j = 0; j < arrsize; j++) {
Rusty Russellde4d8d52011-04-19 21:49:58 +0200440 if (fn(&arr[j], owner, data))
441 return true;
Rusty Russelldafd0942008-07-22 19:24:25 -0500442 }
443
444 return false;
Sam Ravnborg3fd68052006-02-08 21:16:45 +0100445}
446
Rusty Russelldafd0942008-07-22 19:24:25 -0500447/* Returns true as soon as fn returns true, otherwise false. */
Rusty Russellde4d8d52011-04-19 21:49:58 +0200448bool each_symbol_section(bool (*fn)(const struct symsearch *arr,
449 struct module *owner,
450 void *data),
451 void *data)
Arjan van de Venf71d20e2006-06-28 04:26:45 -0700452{
Rusty Russelldafd0942008-07-22 19:24:25 -0500453 struct module *mod;
Linus Torvalds44032e62010-08-05 12:59:05 -0600454 static const struct symsearch arr[] = {
Rusty Russelldafd0942008-07-22 19:24:25 -0500455 { __start___ksymtab, __stop___ksymtab, __start___kcrctab,
456 NOT_GPL_ONLY, false },
457 { __start___ksymtab_gpl, __stop___ksymtab_gpl,
458 __start___kcrctab_gpl,
459 GPL_ONLY, false },
460 { __start___ksymtab_gpl_future, __stop___ksymtab_gpl_future,
461 __start___kcrctab_gpl_future,
462 WILL_BE_GPL_ONLY, false },
Denys Vlasenkof7f5b672008-07-22 19:24:26 -0500463#ifdef CONFIG_UNUSED_SYMBOLS
Rusty Russelldafd0942008-07-22 19:24:25 -0500464 { __start___ksymtab_unused, __stop___ksymtab_unused,
465 __start___kcrctab_unused,
466 NOT_GPL_ONLY, true },
467 { __start___ksymtab_unused_gpl, __stop___ksymtab_unused_gpl,
468 __start___kcrctab_unused_gpl,
469 GPL_ONLY, true },
Denys Vlasenkof7f5b672008-07-22 19:24:26 -0500470#endif
Rusty Russelldafd0942008-07-22 19:24:25 -0500471 };
472
Peter Zijlstra0be964b2015-05-27 11:09:35 +0930473 module_assert_mutex_or_preempt();
474
Rusty Russelldafd0942008-07-22 19:24:25 -0500475 if (each_symbol_in_section(arr, ARRAY_SIZE(arr), NULL, fn, data))
476 return true;
477
Andi Kleend72b3752008-08-30 10:09:00 +0200478 list_for_each_entry_rcu(mod, &modules, list) {
Rusty Russelldafd0942008-07-22 19:24:25 -0500479 struct symsearch arr[] = {
480 { mod->syms, mod->syms + mod->num_syms, mod->crcs,
481 NOT_GPL_ONLY, false },
482 { mod->gpl_syms, mod->gpl_syms + mod->num_gpl_syms,
483 mod->gpl_crcs,
484 GPL_ONLY, false },
485 { mod->gpl_future_syms,
486 mod->gpl_future_syms + mod->num_gpl_future_syms,
487 mod->gpl_future_crcs,
488 WILL_BE_GPL_ONLY, false },
Denys Vlasenkof7f5b672008-07-22 19:24:26 -0500489#ifdef CONFIG_UNUSED_SYMBOLS
Rusty Russelldafd0942008-07-22 19:24:25 -0500490 { mod->unused_syms,
491 mod->unused_syms + mod->num_unused_syms,
492 mod->unused_crcs,
493 NOT_GPL_ONLY, true },
494 { mod->unused_gpl_syms,
495 mod->unused_gpl_syms + mod->num_unused_gpl_syms,
496 mod->unused_gpl_crcs,
497 GPL_ONLY, true },
Denys Vlasenkof7f5b672008-07-22 19:24:26 -0500498#endif
Rusty Russelldafd0942008-07-22 19:24:25 -0500499 };
500
Rusty Russell0d21b0e2013-01-12 11:38:44 +1030501 if (mod->state == MODULE_STATE_UNFORMED)
502 continue;
503
Rusty Russelldafd0942008-07-22 19:24:25 -0500504 if (each_symbol_in_section(arr, ARRAY_SIZE(arr), mod, fn, data))
505 return true;
506 }
507 return false;
Arjan van de Venf71d20e2006-06-28 04:26:45 -0700508}
Rusty Russellde4d8d52011-04-19 21:49:58 +0200509EXPORT_SYMBOL_GPL(each_symbol_section);
Arjan van de Venf71d20e2006-06-28 04:26:45 -0700510
Rusty Russelldafd0942008-07-22 19:24:25 -0500511struct find_symbol_arg {
512 /* Input */
513 const char *name;
514 bool gplok;
515 bool warn;
516
517 /* Output */
518 struct module *owner;
519 const unsigned long *crc;
Tim Abbott414fd312008-12-05 19:03:56 -0500520 const struct kernel_symbol *sym;
Rusty Russelldafd0942008-07-22 19:24:25 -0500521};
522
Rusty Russellde4d8d52011-04-19 21:49:58 +0200523static bool check_symbol(const struct symsearch *syms,
524 struct module *owner,
525 unsigned int symnum, void *data)
Rusty Russellad9546c2008-05-01 21:14:59 -0500526{
Rusty Russelldafd0942008-07-22 19:24:25 -0500527 struct find_symbol_arg *fsa = data;
528
Rusty Russelldafd0942008-07-22 19:24:25 -0500529 if (!fsa->gplok) {
530 if (syms->licence == GPL_ONLY)
531 return false;
532 if (syms->licence == WILL_BE_GPL_ONLY && fsa->warn) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -0800533 pr_warn("Symbol %s is being used by a non-GPL module, "
534 "which will not be allowed in the future\n",
535 fsa->name);
Rusty Russelldafd0942008-07-22 19:24:25 -0500536 }
537 }
538
Denys Vlasenkof7f5b672008-07-22 19:24:26 -0500539#ifdef CONFIG_UNUSED_SYMBOLS
Rusty Russelldafd0942008-07-22 19:24:25 -0500540 if (syms->unused && fsa->warn) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -0800541 pr_warn("Symbol %s is marked as UNUSED, however this module is "
542 "using it.\n", fsa->name);
543 pr_warn("This symbol will go away in the future.\n");
Yannick Guerrini7b63c3a2015-03-24 12:31:40 +1030544 pr_warn("Please evaluate if this is the right api to use and "
545 "if it really is, submit a report to the linux kernel "
546 "mailing list together with submitting your code for "
Andrew Mortonbddb12b2013-11-12 15:11:28 -0800547 "inclusion.\n");
Rusty Russellad9546c2008-05-01 21:14:59 -0500548 }
Denys Vlasenkof7f5b672008-07-22 19:24:26 -0500549#endif
Rusty Russelldafd0942008-07-22 19:24:25 -0500550
551 fsa->owner = owner;
552 fsa->crc = symversion(syms->crcs, symnum);
Tim Abbott414fd312008-12-05 19:03:56 -0500553 fsa->sym = &syms->start[symnum];
Rusty Russellad9546c2008-05-01 21:14:59 -0500554 return true;
555}
556
Alessio Igor Bogani403ed272011-04-20 11:10:52 +0200557static int cmp_name(const void *va, const void *vb)
558{
559 const char *a;
560 const struct kernel_symbol *b;
561 a = va; b = vb;
562 return strcmp(a, b->name);
563}
564
Rusty Russellde4d8d52011-04-19 21:49:58 +0200565static bool find_symbol_in_section(const struct symsearch *syms,
566 struct module *owner,
567 void *data)
568{
569 struct find_symbol_arg *fsa = data;
Alessio Igor Bogani403ed272011-04-20 11:10:52 +0200570 struct kernel_symbol *sym;
Rusty Russellde4d8d52011-04-19 21:49:58 +0200571
Alessio Igor Bogani403ed272011-04-20 11:10:52 +0200572 sym = bsearch(fsa->name, syms->start, syms->stop - syms->start,
573 sizeof(struct kernel_symbol), cmp_name);
574
575 if (sym != NULL && check_symbol(syms, owner, sym - syms->start, data))
576 return true;
577
Rusty Russellde4d8d52011-04-19 21:49:58 +0200578 return false;
579}
580
Tim Abbott414fd312008-12-05 19:03:56 -0500581/* Find a symbol and return it, along with, (optional) crc and
Rusty Russell75676502010-06-05 11:17:36 -0600582 * (optional) module which owns it. Needs preempt disabled or module_mutex. */
Tim Abbottc6b37802008-12-05 19:03:59 -0500583const struct kernel_symbol *find_symbol(const char *name,
584 struct module **owner,
585 const unsigned long **crc,
586 bool gplok,
587 bool warn)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700588{
Rusty Russelldafd0942008-07-22 19:24:25 -0500589 struct find_symbol_arg fsa;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700590
Rusty Russelldafd0942008-07-22 19:24:25 -0500591 fsa.name = name;
592 fsa.gplok = gplok;
593 fsa.warn = warn;
594
Rusty Russellde4d8d52011-04-19 21:49:58 +0200595 if (each_symbol_section(find_symbol_in_section, &fsa)) {
Rusty Russellad9546c2008-05-01 21:14:59 -0500596 if (owner)
Rusty Russelldafd0942008-07-22 19:24:25 -0500597 *owner = fsa.owner;
598 if (crc)
599 *crc = fsa.crc;
Tim Abbott414fd312008-12-05 19:03:56 -0500600 return fsa.sym;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601 }
Rusty Russellad9546c2008-05-01 21:14:59 -0500602
Jim Cromie5e124162011-12-06 12:11:31 -0700603 pr_debug("Failed to find symbol %s\n", name);
Tim Abbott414fd312008-12-05 19:03:56 -0500604 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700605}
Tim Abbottc6b37802008-12-05 19:03:59 -0500606EXPORT_SYMBOL_GPL(find_symbol);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607
Rusty Russellfe0d34d2015-07-29 05:52:14 +0930608/*
609 * Search for module by name: must hold module_mutex (or preempt disabled
610 * for read-only access).
611 */
Mathias Krause4f6de4d2013-07-02 15:35:11 +0930612static struct module *find_module_all(const char *name, size_t len,
Rusty Russell0d21b0e2013-01-12 11:38:44 +1030613 bool even_unformed)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614{
615 struct module *mod;
616
Rusty Russellfe0d34d2015-07-29 05:52:14 +0930617 module_assert_mutex_or_preempt();
Peter Zijlstra0be964b2015-05-27 11:09:35 +0930618
Linus Torvalds1da177e2005-04-16 15:20:36 -0700619 list_for_each_entry(mod, &modules, list) {
Rusty Russell0d21b0e2013-01-12 11:38:44 +1030620 if (!even_unformed && mod->state == MODULE_STATE_UNFORMED)
621 continue;
Mathias Krause4f6de4d2013-07-02 15:35:11 +0930622 if (strlen(mod->name) == len && !memcmp(mod->name, name, len))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 return mod;
624 }
625 return NULL;
626}
Rusty Russell0d21b0e2013-01-12 11:38:44 +1030627
628struct module *find_module(const char *name)
629{
Rusty Russellfe0d34d2015-07-29 05:52:14 +0930630 module_assert_mutex();
Mathias Krause4f6de4d2013-07-02 15:35:11 +0930631 return find_module_all(name, strlen(name), false);
Rusty Russell0d21b0e2013-01-12 11:38:44 +1030632}
Tim Abbottc6b37802008-12-05 19:03:59 -0500633EXPORT_SYMBOL_GPL(find_module);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634
635#ifdef CONFIG_SMP
Tejun Heofbf59bc2009-02-20 16:29:08 +0900636
Tejun Heo259354d2010-03-10 18:56:10 +0900637static inline void __percpu *mod_percpu(struct module *mod)
Tejun Heofbf59bc2009-02-20 16:29:08 +0900638{
Tejun Heo259354d2010-03-10 18:56:10 +0900639 return mod->percpu;
640}
Tejun Heofbf59bc2009-02-20 16:29:08 +0900641
Rusty Russell9eb76d72013-07-03 10:06:29 +0930642static int percpu_modalloc(struct module *mod, struct load_info *info)
Tejun Heo259354d2010-03-10 18:56:10 +0900643{
Rusty Russell9eb76d72013-07-03 10:06:29 +0930644 Elf_Shdr *pcpusec = &info->sechdrs[info->index.pcpu];
645 unsigned long align = pcpusec->sh_addralign;
646
647 if (!pcpusec->sh_size)
648 return 0;
649
Tejun Heofbf59bc2009-02-20 16:29:08 +0900650 if (align > PAGE_SIZE) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -0800651 pr_warn("%s: per-cpu alignment %li > %li\n",
652 mod->name, align, PAGE_SIZE);
Tejun Heofbf59bc2009-02-20 16:29:08 +0900653 align = PAGE_SIZE;
654 }
655
Rusty Russell9eb76d72013-07-03 10:06:29 +0930656 mod->percpu = __alloc_reserved_percpu(pcpusec->sh_size, align);
Tejun Heo259354d2010-03-10 18:56:10 +0900657 if (!mod->percpu) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -0800658 pr_warn("%s: Could not allocate %lu bytes percpu data\n",
659 mod->name, (unsigned long)pcpusec->sh_size);
Tejun Heo259354d2010-03-10 18:56:10 +0900660 return -ENOMEM;
661 }
Rusty Russell9eb76d72013-07-03 10:06:29 +0930662 mod->percpu_size = pcpusec->sh_size;
Tejun Heo259354d2010-03-10 18:56:10 +0900663 return 0;
Tejun Heofbf59bc2009-02-20 16:29:08 +0900664}
665
Tejun Heo259354d2010-03-10 18:56:10 +0900666static void percpu_modfree(struct module *mod)
Tejun Heofbf59bc2009-02-20 16:29:08 +0900667{
Tejun Heo259354d2010-03-10 18:56:10 +0900668 free_percpu(mod->percpu);
Tejun Heofbf59bc2009-02-20 16:29:08 +0900669}
670
Rusty Russell49668682010-08-05 12:59:10 -0600671static unsigned int find_pcpusec(struct load_info *info)
Tejun Heo6b588c12009-02-20 16:29:07 +0900672{
Rusty Russell49668682010-08-05 12:59:10 -0600673 return find_sec(info, ".data..percpu");
Tejun Heo6b588c12009-02-20 16:29:07 +0900674}
675
Tejun Heo259354d2010-03-10 18:56:10 +0900676static void percpu_modcopy(struct module *mod,
677 const void *from, unsigned long size)
Tejun Heo6b588c12009-02-20 16:29:07 +0900678{
679 int cpu;
680
681 for_each_possible_cpu(cpu)
Tejun Heo259354d2010-03-10 18:56:10 +0900682 memcpy(per_cpu_ptr(mod->percpu, cpu), from, size);
Tejun Heo6b588c12009-02-20 16:29:07 +0900683}
684
Tejun Heo10fad5e2010-03-10 18:57:54 +0900685/**
686 * is_module_percpu_address - test whether address is from module static percpu
687 * @addr: address to test
688 *
689 * Test whether @addr belongs to module static percpu area.
690 *
691 * RETURNS:
692 * %true if @addr is from module static percpu area
693 */
694bool is_module_percpu_address(unsigned long addr)
695{
696 struct module *mod;
697 unsigned int cpu;
698
699 preempt_disable();
700
701 list_for_each_entry_rcu(mod, &modules, list) {
Rusty Russell0d21b0e2013-01-12 11:38:44 +1030702 if (mod->state == MODULE_STATE_UNFORMED)
703 continue;
Tejun Heo10fad5e2010-03-10 18:57:54 +0900704 if (!mod->percpu_size)
705 continue;
706 for_each_possible_cpu(cpu) {
707 void *start = per_cpu_ptr(mod->percpu, cpu);
708
709 if ((void *)addr >= start &&
710 (void *)addr < start + mod->percpu_size) {
711 preempt_enable();
712 return true;
713 }
714 }
715 }
716
717 preempt_enable();
718 return false;
Daniel Walker22a8bde2007-10-18 03:06:07 -0700719}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720
721#else /* ... !CONFIG_SMP */
Tejun Heo6b588c12009-02-20 16:29:07 +0900722
Tejun Heo259354d2010-03-10 18:56:10 +0900723static inline void __percpu *mod_percpu(struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724{
725 return NULL;
726}
Rusty Russell9eb76d72013-07-03 10:06:29 +0930727static int percpu_modalloc(struct module *mod, struct load_info *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728{
Rusty Russell9eb76d72013-07-03 10:06:29 +0930729 /* UP modules shouldn't have this section: ENOMEM isn't quite right */
730 if (info->sechdrs[info->index.pcpu].sh_size != 0)
731 return -ENOMEM;
732 return 0;
Tejun Heo259354d2010-03-10 18:56:10 +0900733}
734static inline void percpu_modfree(struct module *mod)
735{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736}
Rusty Russell49668682010-08-05 12:59:10 -0600737static unsigned int find_pcpusec(struct load_info *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700738{
739 return 0;
740}
Tejun Heo259354d2010-03-10 18:56:10 +0900741static inline void percpu_modcopy(struct module *mod,
742 const void *from, unsigned long size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743{
744 /* pcpusec should be 0, and size of that section should be 0. */
745 BUG_ON(size != 0);
746}
Tejun Heo10fad5e2010-03-10 18:57:54 +0900747bool is_module_percpu_address(unsigned long addr)
748{
749 return false;
750}
Tejun Heo6b588c12009-02-20 16:29:07 +0900751
Linus Torvalds1da177e2005-04-16 15:20:36 -0700752#endif /* CONFIG_SMP */
753
Matt Domschc988d2b2005-06-23 22:05:15 -0700754#define MODINFO_ATTR(field) \
755static void setup_modinfo_##field(struct module *mod, const char *s) \
756{ \
757 mod->field = kstrdup(s, GFP_KERNEL); \
758} \
759static ssize_t show_modinfo_##field(struct module_attribute *mattr, \
Kay Sievers4befb022011-07-24 22:06:04 +0930760 struct module_kobject *mk, char *buffer) \
Matt Domschc988d2b2005-06-23 22:05:15 -0700761{ \
Chen Gangcc56ded2013-08-20 15:34:21 +0930762 return scnprintf(buffer, PAGE_SIZE, "%s\n", mk->mod->field); \
Matt Domschc988d2b2005-06-23 22:05:15 -0700763} \
764static int modinfo_##field##_exists(struct module *mod) \
765{ \
766 return mod->field != NULL; \
767} \
768static void free_modinfo_##field(struct module *mod) \
769{ \
Daniel Walker22a8bde2007-10-18 03:06:07 -0700770 kfree(mod->field); \
771 mod->field = NULL; \
Matt Domschc988d2b2005-06-23 22:05:15 -0700772} \
773static struct module_attribute modinfo_##field = { \
Tejun Heo7b595752007-06-14 03:45:17 +0900774 .attr = { .name = __stringify(field), .mode = 0444 }, \
Matt Domschc988d2b2005-06-23 22:05:15 -0700775 .show = show_modinfo_##field, \
776 .setup = setup_modinfo_##field, \
777 .test = modinfo_##field##_exists, \
778 .free = free_modinfo_##field, \
779};
780
781MODINFO_ATTR(version);
782MODINFO_ATTR(srcversion);
783
Arjan van de Vene14af7e2008-01-25 21:08:33 +0100784static char last_unloaded_module[MODULE_NAME_LEN+1];
785
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -0800786#ifdef CONFIG_MODULE_UNLOAD
Steven Rostedteb0c5372010-03-29 14:25:18 -0400787
788EXPORT_TRACEPOINT_SYMBOL(module_get);
789
Masami Hiramatsue513cc12014-11-10 09:30:29 +1030790/* MODULE_REF_BASE is the base reference count by kmodule loader. */
791#define MODULE_REF_BASE 1
792
Linus Torvalds1da177e2005-04-16 15:20:36 -0700793/* Init the unload section of the module. */
Rusty Russell9f85a4b2010-08-05 12:59:04 -0600794static int module_unload_init(struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700795{
Masami Hiramatsue513cc12014-11-10 09:30:29 +1030796 /*
797 * Initialize reference counter to MODULE_REF_BASE.
798 * refcnt == 0 means module is going.
799 */
800 atomic_set(&mod->refcnt, MODULE_REF_BASE);
Rusty Russell9f85a4b2010-08-05 12:59:04 -0600801
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700802 INIT_LIST_HEAD(&mod->source_list);
803 INIT_LIST_HEAD(&mod->target_list);
Christoph Lametere1783a22010-01-05 15:34:50 +0900804
Linus Torvalds1da177e2005-04-16 15:20:36 -0700805 /* Hold reference count during initialization. */
Masami Hiramatsue513cc12014-11-10 09:30:29 +1030806 atomic_inc(&mod->refcnt);
Rusty Russell9f85a4b2010-08-05 12:59:04 -0600807
808 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700809}
810
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811/* Does a already use b? */
812static int already_uses(struct module *a, struct module *b)
813{
814 struct module_use *use;
815
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700816 list_for_each_entry(use, &b->source_list, source_list) {
817 if (use->source == a) {
Jim Cromie5e124162011-12-06 12:11:31 -0700818 pr_debug("%s uses %s!\n", a->name, b->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700819 return 1;
820 }
821 }
Jim Cromie5e124162011-12-06 12:11:31 -0700822 pr_debug("%s does not use %s!\n", a->name, b->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823 return 0;
824}
825
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700826/*
827 * Module a uses b
828 * - we add 'a' as a "source", 'b' as a "target" of module use
829 * - the module_use is added to the list of 'b' sources (so
830 * 'b' can walk the list to see who sourced them), and of 'a'
831 * targets (so 'a' can see what modules it targets).
832 */
833static int add_module_usage(struct module *a, struct module *b)
834{
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700835 struct module_use *use;
836
Jim Cromie5e124162011-12-06 12:11:31 -0700837 pr_debug("Allocating new usage for %s.\n", a->name);
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700838 use = kmalloc(sizeof(*use), GFP_ATOMIC);
839 if (!use) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -0800840 pr_warn("%s: out of memory loading\n", a->name);
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700841 return -ENOMEM;
842 }
843
844 use->source = a;
845 use->target = b;
846 list_add(&use->source_list, &b->source_list);
847 list_add(&use->target_list, &a->target_list);
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700848 return 0;
849}
850
Rusty Russell75676502010-06-05 11:17:36 -0600851/* Module a uses b: caller needs module_mutex() */
Rusty Russell9bea7f22010-06-05 11:17:37 -0600852int ref_module(struct module *a, struct module *b)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700853{
Rusty Russellc8e21ce2010-06-05 11:17:35 -0600854 int err;
Kay Sievers270a6c42007-01-18 13:26:15 +0100855
Rusty Russell9bea7f22010-06-05 11:17:37 -0600856 if (b == NULL || already_uses(a, b))
Linus Torvalds218ce732010-05-25 16:48:30 -0700857 return 0;
Linus Torvalds218ce732010-05-25 16:48:30 -0700858
Rusty Russell9bea7f22010-06-05 11:17:37 -0600859 /* If module isn't available, we fail. */
860 err = strong_try_module_get(b);
Rusty Russellc9a3ba52008-01-29 17:13:18 -0500861 if (err)
Rusty Russell9bea7f22010-06-05 11:17:37 -0600862 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700864 err = add_module_usage(a, b);
865 if (err) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700866 module_put(b);
Rusty Russell9bea7f22010-06-05 11:17:37 -0600867 return err;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700868 }
Rusty Russell9bea7f22010-06-05 11:17:37 -0600869 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870}
Rusty Russell9bea7f22010-06-05 11:17:37 -0600871EXPORT_SYMBOL_GPL(ref_module);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872
873/* Clear the unload stuff of the module. */
874static void module_unload_free(struct module *mod)
875{
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700876 struct module_use *use, *tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877
Rusty Russell75676502010-06-05 11:17:36 -0600878 mutex_lock(&module_mutex);
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700879 list_for_each_entry_safe(use, tmp, &mod->target_list, target_list) {
880 struct module *i = use->target;
Jim Cromie5e124162011-12-06 12:11:31 -0700881 pr_debug("%s unusing %s\n", mod->name, i->name);
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700882 module_put(i);
883 list_del(&use->source_list);
884 list_del(&use->target_list);
885 kfree(use);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 }
Rusty Russell75676502010-06-05 11:17:36 -0600887 mutex_unlock(&module_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700888}
889
890#ifdef CONFIG_MODULE_FORCE_UNLOAD
Akinobu Mitafb169792006-01-08 01:04:29 -0800891static inline int try_force_unload(unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700892{
893 int ret = (flags & O_TRUNC);
894 if (ret)
Rusty Russell373d4d02013-01-21 17:17:39 +1030895 add_taint(TAINT_FORCED_RMMOD, LOCKDEP_NOW_UNRELIABLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896 return ret;
897}
898#else
Akinobu Mitafb169792006-01-08 01:04:29 -0800899static inline int try_force_unload(unsigned int flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900{
901 return 0;
902}
903#endif /* CONFIG_MODULE_FORCE_UNLOAD */
904
Masami Hiramatsue513cc12014-11-10 09:30:29 +1030905/* Try to release refcount of module, 0 means success. */
906static int try_release_module_ref(struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700907{
Masami Hiramatsue513cc12014-11-10 09:30:29 +1030908 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700909
Masami Hiramatsue513cc12014-11-10 09:30:29 +1030910 /* Try to decrement refcnt which we set at loading */
911 ret = atomic_sub_return(MODULE_REF_BASE, &mod->refcnt);
912 BUG_ON(ret < 0);
913 if (ret)
914 /* Someone can put this right now, recover with checking */
915 ret = atomic_add_unless(&mod->refcnt, MODULE_REF_BASE, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916
Masami Hiramatsue513cc12014-11-10 09:30:29 +1030917 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700918}
919
920static int try_stop_module(struct module *mod, int flags, int *forced)
921{
Masami Hiramatsue513cc12014-11-10 09:30:29 +1030922 /* If it's not unused, quit unless we're forcing. */
923 if (try_release_module_ref(mod) != 0) {
924 *forced = try_force_unload(flags);
925 if (!(*forced))
926 return -EWOULDBLOCK;
927 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928
Masami Hiramatsue513cc12014-11-10 09:30:29 +1030929 /* Mark it as dying. */
930 mod->state = MODULE_STATE_GOING;
931
932 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700933}
934
Rusty Russelld5db1392015-01-22 11:13:14 +1030935/**
936 * module_refcount - return the refcount or -1 if unloading
937 *
938 * @mod: the module we're checking
939 *
940 * Returns:
941 * -1 if the module is in the process of unloading
942 * otherwise the number of references in the kernel to the module
943 */
944int module_refcount(struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700945{
Rusty Russelld5db1392015-01-22 11:13:14 +1030946 return atomic_read(&mod->refcnt) - MODULE_REF_BASE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947}
948EXPORT_SYMBOL(module_refcount);
949
950/* This exists whether we can unload or not */
951static void free_module(struct module *mod);
952
Heiko Carstens17da2bd2009-01-14 14:14:10 +0100953SYSCALL_DEFINE2(delete_module, const char __user *, name_user,
954 unsigned int, flags)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700955{
956 struct module *mod;
Greg Kroah-Hartmandfff0a02007-02-23 14:54:57 -0800957 char name[MODULE_NAME_LEN];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958 int ret, forced = 0;
959
Kees Cook3d433212009-04-02 15:49:29 -0700960 if (!capable(CAP_SYS_MODULE) || modules_disabled)
Greg Kroah-Hartmandfff0a02007-02-23 14:54:57 -0800961 return -EPERM;
962
963 if (strncpy_from_user(name, name_user, MODULE_NAME_LEN-1) < 0)
964 return -EFAULT;
965 name[MODULE_NAME_LEN-1] = '\0';
966
Tejun Heo3fc1f1e2010-05-06 18:49:20 +0200967 if (mutex_lock_interruptible(&module_mutex) != 0)
968 return -EINTR;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700969
970 mod = find_module(name);
971 if (!mod) {
972 ret = -ENOENT;
973 goto out;
974 }
975
Linus Torvalds2c02dfe2010-05-31 12:19:37 -0700976 if (!list_empty(&mod->source_list)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977 /* Other modules depend on us: get rid of them first. */
978 ret = -EWOULDBLOCK;
979 goto out;
980 }
981
982 /* Doing init or already dying? */
983 if (mod->state != MODULE_STATE_LIVE) {
Rusty Russell3f2b9c92013-09-17 05:48:51 +0930984 /* FIXME: if (force), slam module count damn the torpedoes */
Jim Cromie5e124162011-12-06 12:11:31 -0700985 pr_debug("%s already dying\n", mod->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700986 ret = -EBUSY;
987 goto out;
988 }
989
990 /* If it has an init func, it must have an exit func to unload */
Rusty Russellaf49d922007-10-16 23:26:27 -0700991 if (mod->init && !mod->exit) {
Akinobu Mitafb169792006-01-08 01:04:29 -0800992 forced = try_force_unload(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700993 if (!forced) {
994 /* This module can't be removed */
995 ret = -EBUSY;
996 goto out;
997 }
998 }
999
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000 /* Stop the machine so refcounts can't move and disable module. */
1001 ret = try_stop_module(mod, flags, &forced);
1002 if (ret != 0)
1003 goto out;
1004
Peter Oberparleiterdf4b5652008-04-21 14:34:31 +02001005 mutex_unlock(&module_mutex);
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001006 /* Final destruction now no one is using it. */
Peter Oberparleiterdf4b5652008-04-21 14:34:31 +02001007 if (mod->exit != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001008 mod->exit();
Peter Oberparleiterdf4b5652008-04-21 14:34:31 +02001009 blocking_notifier_call_chain(&module_notify_list,
1010 MODULE_STATE_GOING, mod);
Arjan van de Ven22a9d642009-01-07 08:45:46 -08001011 async_synchronize_full();
Rusty Russell75676502010-06-05 11:17:36 -06001012
Arjan van de Vene14af7e2008-01-25 21:08:33 +01001013 /* Store the name of the last unloaded module for diagnostic purposes */
Rusty Russellefa53452008-01-29 17:13:20 -05001014 strlcpy(last_unloaded_module, mod->name, sizeof(last_unloaded_module));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015
Rusty Russell75676502010-06-05 11:17:36 -06001016 free_module(mod);
1017 return 0;
1018out:
Ashutosh Naik6389a382006-03-23 03:00:46 -08001019 mutex_unlock(&module_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001020 return ret;
1021}
1022
Jianjun Kongd1e99d72008-12-08 14:26:29 +08001023static inline void print_unload_info(struct seq_file *m, struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024{
1025 struct module_use *use;
1026 int printed_something = 0;
1027
Rusty Russelld5db1392015-01-22 11:13:14 +10301028 seq_printf(m, " %i ", module_refcount(mod));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001029
Ionut Alexa6da0b562014-11-10 09:31:29 +10301030 /*
1031 * Always include a trailing , so userspace can differentiate
1032 * between this and the old multi-field proc format.
1033 */
Linus Torvalds2c02dfe2010-05-31 12:19:37 -07001034 list_for_each_entry(use, &mod->source_list, source_list) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001035 printed_something = 1;
Linus Torvalds2c02dfe2010-05-31 12:19:37 -07001036 seq_printf(m, "%s,", use->source->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001037 }
1038
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 if (mod->init != NULL && mod->exit == NULL) {
1040 printed_something = 1;
Ionut Alexa6da0b562014-11-10 09:31:29 +10301041 seq_puts(m, "[permanent],");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001042 }
1043
1044 if (!printed_something)
Ionut Alexa6da0b562014-11-10 09:31:29 +10301045 seq_puts(m, "-");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001046}
1047
1048void __symbol_put(const char *symbol)
1049{
1050 struct module *owner;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051
Rusty Russell24da1cb2007-07-15 23:41:46 -07001052 preempt_disable();
Tim Abbott414fd312008-12-05 19:03:56 -05001053 if (!find_symbol(symbol, &owner, NULL, true, false))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054 BUG();
1055 module_put(owner);
Rusty Russell24da1cb2007-07-15 23:41:46 -07001056 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001057}
1058EXPORT_SYMBOL(__symbol_put);
1059
Rusty Russell7d1d16e2009-08-26 22:02:54 +09301060/* Note this assumes addr is a function, which it currently always is. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061void symbol_put_addr(void *addr)
1062{
Trent Piepho5e376612006-05-15 09:44:06 -07001063 struct module *modaddr;
Rusty Russell7d1d16e2009-08-26 22:02:54 +09301064 unsigned long a = (unsigned long)dereference_function_descriptor(addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001065
Rusty Russell7d1d16e2009-08-26 22:02:54 +09301066 if (core_kernel_text(a))
Trent Piepho5e376612006-05-15 09:44:06 -07001067 return;
1068
Peter Zijlstra275d7d42015-08-20 10:34:59 +09301069 /*
1070 * Even though we hold a reference on the module; we still need to
1071 * disable preemption in order to safely traverse the data structure.
1072 */
1073 preempt_disable();
Rusty Russell7d1d16e2009-08-26 22:02:54 +09301074 modaddr = __module_text_address(a);
Rusty Russella6e6abd2009-03-31 13:05:31 -06001075 BUG_ON(!modaddr);
Trent Piepho5e376612006-05-15 09:44:06 -07001076 module_put(modaddr);
Peter Zijlstra275d7d42015-08-20 10:34:59 +09301077 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001078}
1079EXPORT_SYMBOL_GPL(symbol_put_addr);
1080
1081static ssize_t show_refcnt(struct module_attribute *mattr,
Kay Sievers4befb022011-07-24 22:06:04 +09301082 struct module_kobject *mk, char *buffer)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001083{
Rusty Russelld5db1392015-01-22 11:13:14 +10301084 return sprintf(buffer, "%i\n", module_refcount(mk->mod));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001085}
1086
Kay Sieverscca3e702012-01-13 09:32:15 +10301087static struct module_attribute modinfo_refcnt =
1088 __ATTR(refcnt, 0444, show_refcnt, NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001089
Steven Rostedtd53799b2012-03-26 12:50:52 +10301090void __module_get(struct module *module)
1091{
1092 if (module) {
1093 preempt_disable();
Masami Hiramatsu2f35c412014-11-10 09:29:29 +10301094 atomic_inc(&module->refcnt);
Steven Rostedtd53799b2012-03-26 12:50:52 +10301095 trace_module_get(module, _RET_IP_);
1096 preempt_enable();
1097 }
1098}
1099EXPORT_SYMBOL(__module_get);
1100
1101bool try_module_get(struct module *module)
1102{
1103 bool ret = true;
1104
1105 if (module) {
1106 preempt_disable();
Masami Hiramatsue513cc12014-11-10 09:30:29 +10301107 /* Note: here, we can fail to get a reference */
1108 if (likely(module_is_live(module) &&
1109 atomic_inc_not_zero(&module->refcnt) != 0))
Steven Rostedtd53799b2012-03-26 12:50:52 +10301110 trace_module_get(module, _RET_IP_);
Masami Hiramatsue513cc12014-11-10 09:30:29 +10301111 else
Steven Rostedtd53799b2012-03-26 12:50:52 +10301112 ret = false;
1113
1114 preempt_enable();
1115 }
1116 return ret;
1117}
1118EXPORT_SYMBOL(try_module_get);
1119
Al Virof6a57032006-10-18 01:47:25 -04001120void module_put(struct module *module)
1121{
Masami Hiramatsue513cc12014-11-10 09:30:29 +10301122 int ret;
1123
Al Virof6a57032006-10-18 01:47:25 -04001124 if (module) {
Christoph Lametere1783a22010-01-05 15:34:50 +09001125 preempt_disable();
Masami Hiramatsue513cc12014-11-10 09:30:29 +10301126 ret = atomic_dec_if_positive(&module->refcnt);
1127 WARN_ON(ret < 0); /* Failed to put refcount */
Li Zefanae832d12010-03-24 10:57:43 +08001128 trace_module_put(module, _RET_IP_);
Christoph Lametere1783a22010-01-05 15:34:50 +09001129 preempt_enable();
Al Virof6a57032006-10-18 01:47:25 -04001130 }
1131}
1132EXPORT_SYMBOL(module_put);
1133
Linus Torvalds1da177e2005-04-16 15:20:36 -07001134#else /* !CONFIG_MODULE_UNLOAD */
Jianjun Kongd1e99d72008-12-08 14:26:29 +08001135static inline void print_unload_info(struct seq_file *m, struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136{
1137 /* We don't know the usage count, or what modules are using. */
Ionut Alexa6da0b562014-11-10 09:31:29 +10301138 seq_puts(m, " - -");
Linus Torvalds1da177e2005-04-16 15:20:36 -07001139}
1140
1141static inline void module_unload_free(struct module *mod)
1142{
1143}
1144
Rusty Russell9bea7f22010-06-05 11:17:37 -06001145int ref_module(struct module *a, struct module *b)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001146{
Rusty Russell9bea7f22010-06-05 11:17:37 -06001147 return strong_try_module_get(b);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001148}
Rusty Russell9bea7f22010-06-05 11:17:37 -06001149EXPORT_SYMBOL_GPL(ref_module);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001150
Rusty Russell9f85a4b2010-08-05 12:59:04 -06001151static inline int module_unload_init(struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001152{
Rusty Russell9f85a4b2010-08-05 12:59:04 -06001153 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001154}
1155#endif /* CONFIG_MODULE_UNLOAD */
1156
Kevin Winchester53999bf2012-01-15 19:32:55 -04001157static size_t module_flags_taint(struct module *mod, char *buf)
1158{
1159 size_t l = 0;
1160
1161 if (mod->taints & (1 << TAINT_PROPRIETARY_MODULE))
1162 buf[l++] = 'P';
1163 if (mod->taints & (1 << TAINT_OOT_MODULE))
1164 buf[l++] = 'O';
1165 if (mod->taints & (1 << TAINT_FORCED_MODULE))
1166 buf[l++] = 'F';
1167 if (mod->taints & (1 << TAINT_CRAP))
1168 buf[l++] = 'C';
Mathieu Desnoyers66cc69e2014-03-13 12:11:30 +10301169 if (mod->taints & (1 << TAINT_UNSIGNED_MODULE))
Rusty Russell57673c22014-03-31 14:39:57 +10301170 buf[l++] = 'E';
Kevin Winchester53999bf2012-01-15 19:32:55 -04001171 /*
1172 * TAINT_FORCED_RMMOD: could be added.
Dave Jones8c904872014-02-26 10:49:49 -05001173 * TAINT_CPU_OUT_OF_SPEC, TAINT_MACHINE_CHECK, TAINT_BAD_PAGE don't
Kevin Winchester53999bf2012-01-15 19:32:55 -04001174 * apply to modules.
1175 */
1176 return l;
1177}
1178
Kay Sievers1f717402006-11-24 12:15:25 +01001179static ssize_t show_initstate(struct module_attribute *mattr,
Kay Sievers4befb022011-07-24 22:06:04 +09301180 struct module_kobject *mk, char *buffer)
Kay Sievers1f717402006-11-24 12:15:25 +01001181{
1182 const char *state = "unknown";
1183
Kay Sievers4befb022011-07-24 22:06:04 +09301184 switch (mk->mod->state) {
Kay Sievers1f717402006-11-24 12:15:25 +01001185 case MODULE_STATE_LIVE:
1186 state = "live";
1187 break;
1188 case MODULE_STATE_COMING:
1189 state = "coming";
1190 break;
1191 case MODULE_STATE_GOING:
1192 state = "going";
1193 break;
Rusty Russell0d21b0e2013-01-12 11:38:44 +10301194 default:
1195 BUG();
Kay Sievers1f717402006-11-24 12:15:25 +01001196 }
1197 return sprintf(buffer, "%s\n", state);
1198}
1199
Kay Sieverscca3e702012-01-13 09:32:15 +10301200static struct module_attribute modinfo_initstate =
1201 __ATTR(initstate, 0444, show_initstate, NULL);
Kay Sievers1f717402006-11-24 12:15:25 +01001202
Kay Sievers88bfa3242011-07-24 22:06:04 +09301203static ssize_t store_uevent(struct module_attribute *mattr,
1204 struct module_kobject *mk,
1205 const char *buffer, size_t count)
1206{
1207 enum kobject_action action;
1208
1209 if (kobject_action_type(buffer, count, &action) == 0)
1210 kobject_uevent(&mk->kobj, action);
1211 return count;
1212}
1213
Kay Sieverscca3e702012-01-13 09:32:15 +10301214struct module_attribute module_uevent =
1215 __ATTR(uevent, 0200, NULL, store_uevent);
1216
1217static ssize_t show_coresize(struct module_attribute *mattr,
1218 struct module_kobject *mk, char *buffer)
1219{
1220 return sprintf(buffer, "%u\n", mk->mod->core_size);
1221}
1222
1223static struct module_attribute modinfo_coresize =
1224 __ATTR(coresize, 0444, show_coresize, NULL);
1225
1226static ssize_t show_initsize(struct module_attribute *mattr,
1227 struct module_kobject *mk, char *buffer)
1228{
1229 return sprintf(buffer, "%u\n", mk->mod->init_size);
1230}
1231
1232static struct module_attribute modinfo_initsize =
1233 __ATTR(initsize, 0444, show_initsize, NULL);
1234
1235static ssize_t show_taint(struct module_attribute *mattr,
1236 struct module_kobject *mk, char *buffer)
1237{
1238 size_t l;
1239
1240 l = module_flags_taint(mk->mod, buffer);
1241 buffer[l++] = '\n';
1242 return l;
1243}
1244
1245static struct module_attribute modinfo_taint =
1246 __ATTR(taint, 0444, show_taint, NULL);
Kay Sievers88bfa3242011-07-24 22:06:04 +09301247
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001248static struct module_attribute *modinfo_attrs[] = {
Kay Sieverscca3e702012-01-13 09:32:15 +10301249 &module_uevent,
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001250 &modinfo_version,
1251 &modinfo_srcversion,
Kay Sieverscca3e702012-01-13 09:32:15 +10301252 &modinfo_initstate,
1253 &modinfo_coresize,
1254 &modinfo_initsize,
1255 &modinfo_taint,
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001256#ifdef CONFIG_MODULE_UNLOAD
Kay Sieverscca3e702012-01-13 09:32:15 +10301257 &modinfo_refcnt,
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001258#endif
1259 NULL,
1260};
1261
Linus Torvalds1da177e2005-04-16 15:20:36 -07001262static const char vermagic[] = VERMAGIC_STRING;
1263
Rusty Russellc6e665c2009-03-31 13:05:33 -06001264static int try_to_force_load(struct module *mod, const char *reason)
Linus Torvalds826e4502008-05-04 17:04:16 -07001265{
1266#ifdef CONFIG_MODULE_FORCE_LOAD
Andi Kleen25ddbb12008-10-15 22:01:41 -07001267 if (!test_taint(TAINT_FORCED_MODULE))
Andrew Mortonbddb12b2013-11-12 15:11:28 -08001268 pr_warn("%s: %s: kernel tainted.\n", mod->name, reason);
Rusty Russell373d4d02013-01-21 17:17:39 +10301269 add_taint_module(mod, TAINT_FORCED_MODULE, LOCKDEP_NOW_UNRELIABLE);
Linus Torvalds826e4502008-05-04 17:04:16 -07001270 return 0;
1271#else
1272 return -ENOEXEC;
1273#endif
1274}
1275
Linus Torvalds1da177e2005-04-16 15:20:36 -07001276#ifdef CONFIG_MODVERSIONS
Rusty Russelld4703ae2009-12-15 16:28:32 -06001277/* If the arch applies (non-zero) relocations to kernel kcrctab, unapply it. */
1278static unsigned long maybe_relocated(unsigned long crc,
1279 const struct module *crc_owner)
1280{
1281#ifdef ARCH_RELOCATES_KCRCTAB
1282 if (crc_owner == NULL)
1283 return crc - (unsigned long)reloc_start;
1284#endif
1285 return crc;
1286}
1287
Linus Torvalds1da177e2005-04-16 15:20:36 -07001288static int check_version(Elf_Shdr *sechdrs,
1289 unsigned int versindex,
1290 const char *symname,
Ionut Alexa6da0b562014-11-10 09:31:29 +10301291 struct module *mod,
Rusty Russelld4703ae2009-12-15 16:28:32 -06001292 const unsigned long *crc,
1293 const struct module *crc_owner)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001294{
1295 unsigned int i, num_versions;
1296 struct modversion_info *versions;
1297
1298 /* Exporting module didn't supply crcs? OK, we're already tainted. */
1299 if (!crc)
1300 return 1;
1301
Rusty Russella5dd6972008-05-09 16:24:21 +10001302 /* No versions at all? modprobe --force does this. */
1303 if (versindex == 0)
1304 return try_to_force_load(mod, symname) == 0;
1305
Linus Torvalds1da177e2005-04-16 15:20:36 -07001306 versions = (void *) sechdrs[versindex].sh_addr;
1307 num_versions = sechdrs[versindex].sh_size
1308 / sizeof(struct modversion_info);
1309
1310 for (i = 0; i < num_versions; i++) {
1311 if (strcmp(versions[i].name, symname) != 0)
1312 continue;
1313
Rusty Russelld4703ae2009-12-15 16:28:32 -06001314 if (versions[i].crc == maybe_relocated(*crc, crc_owner))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315 return 1;
Jim Cromie5e124162011-12-06 12:11:31 -07001316 pr_debug("Found checksum %lX vs module %lX\n",
Rusty Russelld4703ae2009-12-15 16:28:32 -06001317 maybe_relocated(*crc, crc_owner), versions[i].crc);
Linus Torvalds826e4502008-05-04 17:04:16 -07001318 goto bad_version;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001319 }
Linus Torvalds826e4502008-05-04 17:04:16 -07001320
Andrew Mortonbddb12b2013-11-12 15:11:28 -08001321 pr_warn("%s: no symbol version for %s\n", mod->name, symname);
Rusty Russella5dd6972008-05-09 16:24:21 +10001322 return 0;
Linus Torvalds826e4502008-05-04 17:04:16 -07001323
1324bad_version:
Ionut Alexa6da0b562014-11-10 09:31:29 +10301325 pr_warn("%s: disagrees about version of symbol %s\n",
Linus Torvalds826e4502008-05-04 17:04:16 -07001326 mod->name, symname);
1327 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001328}
1329
1330static inline int check_modstruct_version(Elf_Shdr *sechdrs,
1331 unsigned int versindex,
1332 struct module *mod)
1333{
1334 const unsigned long *crc;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001335
Peter Zijlstra926a59b2015-05-27 11:09:35 +09301336 /*
1337 * Since this should be found in kernel (which can't be removed), no
1338 * locking is necessary -- use preempt_disable() to placate lockdep.
1339 */
1340 preempt_disable();
Rusty Russellb92021b2013-03-15 15:04:17 +10301341 if (!find_symbol(VMLINUX_SYMBOL_STR(module_layout), NULL,
Peter Zijlstra926a59b2015-05-27 11:09:35 +09301342 &crc, true, false)) {
1343 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344 BUG();
Peter Zijlstra926a59b2015-05-27 11:09:35 +09301345 }
1346 preempt_enable();
James Hogana4b6a772013-03-18 19:38:56 +10301347 return check_version(sechdrs, versindex,
1348 VMLINUX_SYMBOL_STR(module_layout), mod, crc,
Rusty Russelld4703ae2009-12-15 16:28:32 -06001349 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350}
1351
Rusty Russell91e37a72008-05-09 16:25:28 +10001352/* First part is kernel version, which we ignore if module has crcs. */
1353static inline int same_magic(const char *amagic, const char *bmagic,
1354 bool has_crcs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355{
Rusty Russell91e37a72008-05-09 16:25:28 +10001356 if (has_crcs) {
1357 amagic += strcspn(amagic, " ");
1358 bmagic += strcspn(bmagic, " ");
1359 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001360 return strcmp(amagic, bmagic) == 0;
1361}
1362#else
1363static inline int check_version(Elf_Shdr *sechdrs,
1364 unsigned int versindex,
1365 const char *symname,
Ionut Alexa6da0b562014-11-10 09:31:29 +10301366 struct module *mod,
Rusty Russelld4703ae2009-12-15 16:28:32 -06001367 const unsigned long *crc,
1368 const struct module *crc_owner)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001369{
1370 return 1;
1371}
1372
1373static inline int check_modstruct_version(Elf_Shdr *sechdrs,
1374 unsigned int versindex,
1375 struct module *mod)
1376{
1377 return 1;
1378}
1379
Rusty Russell91e37a72008-05-09 16:25:28 +10001380static inline int same_magic(const char *amagic, const char *bmagic,
1381 bool has_crcs)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001382{
1383 return strcmp(amagic, bmagic) == 0;
1384}
1385#endif /* CONFIG_MODVERSIONS */
1386
Rusty Russell75676502010-06-05 11:17:36 -06001387/* Resolve a symbol for this module. I.e. if we find one, record usage. */
Rusty Russell49668682010-08-05 12:59:10 -06001388static const struct kernel_symbol *resolve_symbol(struct module *mod,
1389 const struct load_info *info,
Tim Abbott414fd312008-12-05 19:03:56 -05001390 const char *name,
Rusty Russell9bea7f22010-06-05 11:17:37 -06001391 char ownername[])
Linus Torvalds1da177e2005-04-16 15:20:36 -07001392{
1393 struct module *owner;
Tim Abbott414fd312008-12-05 19:03:56 -05001394 const struct kernel_symbol *sym;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001395 const unsigned long *crc;
Rusty Russell9bea7f22010-06-05 11:17:37 -06001396 int err;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001397
Peter Zijlstrad64810f2015-02-11 15:01:13 +10301398 /*
1399 * The module_mutex should not be a heavily contended lock;
1400 * if we get the occasional sleep here, we'll go an extra iteration
1401 * in the wait_event_interruptible(), which is harmless.
1402 */
1403 sched_annotate_sleep();
Rusty Russell75676502010-06-05 11:17:36 -06001404 mutex_lock(&module_mutex);
Tim Abbott414fd312008-12-05 19:03:56 -05001405 sym = find_symbol(name, &owner, &crc,
Andi Kleen25ddbb12008-10-15 22:01:41 -07001406 !(mod->taints & (1 << TAINT_PROPRIETARY_MODULE)), true);
Rusty Russell9bea7f22010-06-05 11:17:37 -06001407 if (!sym)
1408 goto unlock;
1409
Rusty Russell49668682010-08-05 12:59:10 -06001410 if (!check_version(info->sechdrs, info->index.vers, name, mod, crc,
1411 owner)) {
Rusty Russell9bea7f22010-06-05 11:17:37 -06001412 sym = ERR_PTR(-EINVAL);
1413 goto getname;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001414 }
Rusty Russell9bea7f22010-06-05 11:17:37 -06001415
1416 err = ref_module(mod, owner);
1417 if (err) {
1418 sym = ERR_PTR(err);
1419 goto getname;
1420 }
1421
1422getname:
1423 /* We must make copy under the lock if we failed to get ref. */
1424 strncpy(ownername, module_name(owner), MODULE_NAME_LEN);
1425unlock:
Rusty Russell75676502010-06-05 11:17:36 -06001426 mutex_unlock(&module_mutex);
Linus Torvalds218ce732010-05-25 16:48:30 -07001427 return sym;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001428}
1429
Rusty Russell49668682010-08-05 12:59:10 -06001430static const struct kernel_symbol *
1431resolve_symbol_wait(struct module *mod,
1432 const struct load_info *info,
1433 const char *name)
Rusty Russell9bea7f22010-06-05 11:17:37 -06001434{
1435 const struct kernel_symbol *ksym;
Rusty Russell49668682010-08-05 12:59:10 -06001436 char owner[MODULE_NAME_LEN];
Rusty Russell9bea7f22010-06-05 11:17:37 -06001437
1438 if (wait_event_interruptible_timeout(module_wq,
Rusty Russell49668682010-08-05 12:59:10 -06001439 !IS_ERR(ksym = resolve_symbol(mod, info, name, owner))
1440 || PTR_ERR(ksym) != -EBUSY,
Rusty Russell9bea7f22010-06-05 11:17:37 -06001441 30 * HZ) <= 0) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -08001442 pr_warn("%s: gave up waiting for init of module %s.\n",
1443 mod->name, owner);
Rusty Russell9bea7f22010-06-05 11:17:37 -06001444 }
1445 return ksym;
1446}
1447
Linus Torvalds1da177e2005-04-16 15:20:36 -07001448/*
1449 * /sys/module/foo/sections stuff
1450 * J. Corbet <corbet@lwn.net>
1451 */
Rusty Russell8f6d0372010-08-05 12:59:09 -06001452#ifdef CONFIG_SYSFS
Ben Hutchings10b465a2009-12-19 14:43:01 +00001453
Rusty Russell8f6d0372010-08-05 12:59:09 -06001454#ifdef CONFIG_KALLSYMS
Ben Hutchings10b465a2009-12-19 14:43:01 +00001455static inline bool sect_empty(const Elf_Shdr *sect)
1456{
1457 return !(sect->sh_flags & SHF_ALLOC) || sect->sh_size == 0;
1458}
1459
Ionut Alexa6da0b562014-11-10 09:31:29 +10301460struct module_sect_attr {
Rusty Russella58730c2008-03-13 09:03:44 +00001461 struct module_attribute mattr;
1462 char *name;
1463 unsigned long address;
1464};
1465
Ionut Alexa6da0b562014-11-10 09:31:29 +10301466struct module_sect_attrs {
Rusty Russella58730c2008-03-13 09:03:44 +00001467 struct attribute_group grp;
1468 unsigned int nsections;
1469 struct module_sect_attr attrs[0];
1470};
1471
Linus Torvalds1da177e2005-04-16 15:20:36 -07001472static ssize_t module_sect_show(struct module_attribute *mattr,
Kay Sievers4befb022011-07-24 22:06:04 +09301473 struct module_kobject *mk, char *buf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001474{
1475 struct module_sect_attr *sattr =
1476 container_of(mattr, struct module_sect_attr, mattr);
Kees Cook9f36e2c2011-03-22 16:34:22 -07001477 return sprintf(buf, "0x%pK\n", (void *)sattr->address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001478}
1479
Ian S. Nelson04b1db92006-09-29 02:01:31 -07001480static void free_sect_attrs(struct module_sect_attrs *sect_attrs)
1481{
Rusty Russella58730c2008-03-13 09:03:44 +00001482 unsigned int section;
Ian S. Nelson04b1db92006-09-29 02:01:31 -07001483
1484 for (section = 0; section < sect_attrs->nsections; section++)
1485 kfree(sect_attrs->attrs[section].name);
1486 kfree(sect_attrs);
1487}
1488
Rusty Russell8f6d0372010-08-05 12:59:09 -06001489static void add_sect_attrs(struct module *mod, const struct load_info *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001490{
1491 unsigned int nloaded = 0, i, size[2];
1492 struct module_sect_attrs *sect_attrs;
1493 struct module_sect_attr *sattr;
1494 struct attribute **gattr;
Daniel Walker22a8bde2007-10-18 03:06:07 -07001495
Linus Torvalds1da177e2005-04-16 15:20:36 -07001496 /* Count loaded sections and allocate structures */
Rusty Russell8f6d0372010-08-05 12:59:09 -06001497 for (i = 0; i < info->hdr->e_shnum; i++)
1498 if (!sect_empty(&info->sechdrs[i]))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499 nloaded++;
1500 size[0] = ALIGN(sizeof(*sect_attrs)
1501 + nloaded * sizeof(sect_attrs->attrs[0]),
1502 sizeof(sect_attrs->grp.attrs[0]));
1503 size[1] = (nloaded + 1) * sizeof(sect_attrs->grp.attrs[0]);
Ian S. Nelson04b1db92006-09-29 02:01:31 -07001504 sect_attrs = kzalloc(size[0] + size[1], GFP_KERNEL);
1505 if (sect_attrs == NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001506 return;
1507
1508 /* Setup section attributes. */
1509 sect_attrs->grp.name = "sections";
1510 sect_attrs->grp.attrs = (void *)sect_attrs + size[0];
1511
Ian S. Nelson04b1db92006-09-29 02:01:31 -07001512 sect_attrs->nsections = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001513 sattr = &sect_attrs->attrs[0];
1514 gattr = &sect_attrs->grp.attrs[0];
Rusty Russell8f6d0372010-08-05 12:59:09 -06001515 for (i = 0; i < info->hdr->e_shnum; i++) {
1516 Elf_Shdr *sec = &info->sechdrs[i];
1517 if (sect_empty(sec))
Helge Deller35dead42009-12-03 00:29:15 +01001518 continue;
Rusty Russell8f6d0372010-08-05 12:59:09 -06001519 sattr->address = sec->sh_addr;
1520 sattr->name = kstrdup(info->secstrings + sec->sh_name,
Ian S. Nelson04b1db92006-09-29 02:01:31 -07001521 GFP_KERNEL);
1522 if (sattr->name == NULL)
1523 goto out;
1524 sect_attrs->nsections++;
Eric W. Biederman361795b2010-02-12 13:41:56 -08001525 sysfs_attr_init(&sattr->mattr.attr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 sattr->mattr.show = module_sect_show;
1527 sattr->mattr.store = NULL;
1528 sattr->mattr.attr.name = sattr->name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529 sattr->mattr.attr.mode = S_IRUGO;
1530 *(gattr++) = &(sattr++)->mattr.attr;
1531 }
1532 *gattr = NULL;
1533
1534 if (sysfs_create_group(&mod->mkobj.kobj, &sect_attrs->grp))
1535 goto out;
1536
1537 mod->sect_attrs = sect_attrs;
1538 return;
1539 out:
Ian S. Nelson04b1db92006-09-29 02:01:31 -07001540 free_sect_attrs(sect_attrs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541}
1542
1543static void remove_sect_attrs(struct module *mod)
1544{
1545 if (mod->sect_attrs) {
1546 sysfs_remove_group(&mod->mkobj.kobj,
1547 &mod->sect_attrs->grp);
1548 /* We are positive that no one is using any sect attrs
1549 * at this point. Deallocate immediately. */
Ian S. Nelson04b1db92006-09-29 02:01:31 -07001550 free_sect_attrs(mod->sect_attrs);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001551 mod->sect_attrs = NULL;
1552 }
1553}
1554
Roland McGrath6d760132007-10-16 23:26:40 -07001555/*
1556 * /sys/module/foo/notes/.section.name gives contents of SHT_NOTE sections.
1557 */
1558
1559struct module_notes_attrs {
1560 struct kobject *dir;
1561 unsigned int notes;
1562 struct bin_attribute attrs[0];
1563};
1564
Chris Wright2c3c8be2010-05-12 18:28:57 -07001565static ssize_t module_notes_read(struct file *filp, struct kobject *kobj,
Roland McGrath6d760132007-10-16 23:26:40 -07001566 struct bin_attribute *bin_attr,
1567 char *buf, loff_t pos, size_t count)
1568{
1569 /*
1570 * The caller checked the pos and count against our size.
1571 */
1572 memcpy(buf, bin_attr->private + pos, count);
1573 return count;
1574}
1575
1576static void free_notes_attrs(struct module_notes_attrs *notes_attrs,
1577 unsigned int i)
1578{
1579 if (notes_attrs->dir) {
1580 while (i-- > 0)
1581 sysfs_remove_bin_file(notes_attrs->dir,
1582 &notes_attrs->attrs[i]);
Alexey Dobriyane9432092008-09-23 23:51:11 +04001583 kobject_put(notes_attrs->dir);
Roland McGrath6d760132007-10-16 23:26:40 -07001584 }
1585 kfree(notes_attrs);
1586}
1587
Rusty Russell8f6d0372010-08-05 12:59:09 -06001588static void add_notes_attrs(struct module *mod, const struct load_info *info)
Roland McGrath6d760132007-10-16 23:26:40 -07001589{
1590 unsigned int notes, loaded, i;
1591 struct module_notes_attrs *notes_attrs;
1592 struct bin_attribute *nattr;
1593
Ingo Molnarea6bff32009-08-28 10:44:56 +02001594 /* failed to create section attributes, so can't create notes */
1595 if (!mod->sect_attrs)
1596 return;
1597
Roland McGrath6d760132007-10-16 23:26:40 -07001598 /* Count notes sections and allocate structures. */
1599 notes = 0;
Rusty Russell8f6d0372010-08-05 12:59:09 -06001600 for (i = 0; i < info->hdr->e_shnum; i++)
1601 if (!sect_empty(&info->sechdrs[i]) &&
1602 (info->sechdrs[i].sh_type == SHT_NOTE))
Roland McGrath6d760132007-10-16 23:26:40 -07001603 ++notes;
1604
1605 if (notes == 0)
1606 return;
1607
1608 notes_attrs = kzalloc(sizeof(*notes_attrs)
1609 + notes * sizeof(notes_attrs->attrs[0]),
1610 GFP_KERNEL);
1611 if (notes_attrs == NULL)
1612 return;
1613
1614 notes_attrs->notes = notes;
1615 nattr = &notes_attrs->attrs[0];
Rusty Russell8f6d0372010-08-05 12:59:09 -06001616 for (loaded = i = 0; i < info->hdr->e_shnum; ++i) {
1617 if (sect_empty(&info->sechdrs[i]))
Roland McGrath6d760132007-10-16 23:26:40 -07001618 continue;
Rusty Russell8f6d0372010-08-05 12:59:09 -06001619 if (info->sechdrs[i].sh_type == SHT_NOTE) {
Eric W. Biederman361795b2010-02-12 13:41:56 -08001620 sysfs_bin_attr_init(nattr);
Roland McGrath6d760132007-10-16 23:26:40 -07001621 nattr->attr.name = mod->sect_attrs->attrs[loaded].name;
1622 nattr->attr.mode = S_IRUGO;
Rusty Russell8f6d0372010-08-05 12:59:09 -06001623 nattr->size = info->sechdrs[i].sh_size;
1624 nattr->private = (void *) info->sechdrs[i].sh_addr;
Roland McGrath6d760132007-10-16 23:26:40 -07001625 nattr->read = module_notes_read;
1626 ++nattr;
1627 }
1628 ++loaded;
1629 }
1630
Greg Kroah-Hartman4ff6abf2007-11-05 22:24:43 -08001631 notes_attrs->dir = kobject_create_and_add("notes", &mod->mkobj.kobj);
Roland McGrath6d760132007-10-16 23:26:40 -07001632 if (!notes_attrs->dir)
1633 goto out;
1634
1635 for (i = 0; i < notes; ++i)
1636 if (sysfs_create_bin_file(notes_attrs->dir,
1637 &notes_attrs->attrs[i]))
1638 goto out;
1639
1640 mod->notes_attrs = notes_attrs;
1641 return;
1642
1643 out:
1644 free_notes_attrs(notes_attrs, i);
1645}
1646
1647static void remove_notes_attrs(struct module *mod)
1648{
1649 if (mod->notes_attrs)
1650 free_notes_attrs(mod->notes_attrs, mod->notes_attrs->notes);
1651}
1652
Linus Torvalds1da177e2005-04-16 15:20:36 -07001653#else
Ian S. Nelson04b1db92006-09-29 02:01:31 -07001654
Rusty Russell8f6d0372010-08-05 12:59:09 -06001655static inline void add_sect_attrs(struct module *mod,
1656 const struct load_info *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001657{
1658}
1659
1660static inline void remove_sect_attrs(struct module *mod)
1661{
1662}
Roland McGrath6d760132007-10-16 23:26:40 -07001663
Rusty Russell8f6d0372010-08-05 12:59:09 -06001664static inline void add_notes_attrs(struct module *mod,
1665 const struct load_info *info)
Roland McGrath6d760132007-10-16 23:26:40 -07001666{
1667}
1668
1669static inline void remove_notes_attrs(struct module *mod)
1670{
1671}
Rusty Russell8f6d0372010-08-05 12:59:09 -06001672#endif /* CONFIG_KALLSYMS */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001673
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001674static void add_usage_links(struct module *mod)
1675{
1676#ifdef CONFIG_MODULE_UNLOAD
1677 struct module_use *use;
1678 int nowarn;
1679
Rusty Russell75676502010-06-05 11:17:36 -06001680 mutex_lock(&module_mutex);
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001681 list_for_each_entry(use, &mod->target_list, target_list) {
1682 nowarn = sysfs_create_link(use->target->holders_dir,
1683 &mod->mkobj.kobj, mod->name);
1684 }
Rusty Russell75676502010-06-05 11:17:36 -06001685 mutex_unlock(&module_mutex);
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001686#endif
1687}
1688
1689static void del_usage_links(struct module *mod)
1690{
1691#ifdef CONFIG_MODULE_UNLOAD
1692 struct module_use *use;
1693
Rusty Russell75676502010-06-05 11:17:36 -06001694 mutex_lock(&module_mutex);
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001695 list_for_each_entry(use, &mod->target_list, target_list)
1696 sysfs_remove_link(use->target->holders_dir, mod->name);
Rusty Russell75676502010-06-05 11:17:36 -06001697 mutex_unlock(&module_mutex);
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001698#endif
1699}
1700
Rusty Russell6407ebb22010-06-05 11:17:36 -06001701static int module_add_modinfo_attrs(struct module *mod)
Matt Domschc988d2b2005-06-23 22:05:15 -07001702{
1703 struct module_attribute *attr;
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001704 struct module_attribute *temp_attr;
Matt Domschc988d2b2005-06-23 22:05:15 -07001705 int error = 0;
1706 int i;
1707
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001708 mod->modinfo_attrs = kzalloc((sizeof(struct module_attribute) *
1709 (ARRAY_SIZE(modinfo_attrs) + 1)),
1710 GFP_KERNEL);
1711 if (!mod->modinfo_attrs)
1712 return -ENOMEM;
1713
1714 temp_attr = mod->modinfo_attrs;
Matt Domschc988d2b2005-06-23 22:05:15 -07001715 for (i = 0; (attr = modinfo_attrs[i]) && !error; i++) {
1716 if (!attr->test ||
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001717 (attr->test && attr->test(mod))) {
1718 memcpy(temp_attr, attr, sizeof(*temp_attr));
Eric W. Biederman361795b2010-02-12 13:41:56 -08001719 sysfs_attr_init(&temp_attr->attr);
Ionut Alexa6da0b562014-11-10 09:31:29 +10301720 error = sysfs_create_file(&mod->mkobj.kobj,
1721 &temp_attr->attr);
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001722 ++temp_attr;
1723 }
Matt Domschc988d2b2005-06-23 22:05:15 -07001724 }
1725 return error;
1726}
1727
Rusty Russell6407ebb22010-06-05 11:17:36 -06001728static void module_remove_modinfo_attrs(struct module *mod)
Matt Domschc988d2b2005-06-23 22:05:15 -07001729{
1730 struct module_attribute *attr;
1731 int i;
1732
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001733 for (i = 0; (attr = &mod->modinfo_attrs[i]); i++) {
1734 /* pick a field to test for end of list */
1735 if (!attr->attr.name)
1736 break;
Ionut Alexa6da0b562014-11-10 09:31:29 +10301737 sysfs_remove_file(&mod->mkobj.kobj, &attr->attr);
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001738 if (attr->free)
1739 attr->free(mod);
Matt Domschc988d2b2005-06-23 22:05:15 -07001740 }
Greg Kroah-Hartman03e88ae12006-02-16 13:50:23 -08001741 kfree(mod->modinfo_attrs);
Matt Domschc988d2b2005-06-23 22:05:15 -07001742}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001743
Li Zhong942e4432013-09-03 16:33:57 +09301744static void mod_kobject_put(struct module *mod)
1745{
1746 DECLARE_COMPLETION_ONSTACK(c);
1747 mod->mkobj.kobj_completion = &c;
1748 kobject_put(&mod->mkobj.kobj);
1749 wait_for_completion(&c);
1750}
1751
Rusty Russell6407ebb22010-06-05 11:17:36 -06001752static int mod_sysfs_init(struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001753{
1754 int err;
Greg Kroah-Hartman6494a932008-01-27 15:38:40 -08001755 struct kobject *kobj;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001756
Greg Kroah-Hartman823bccf2007-04-13 13:15:19 -07001757 if (!module_sysfs_initialized) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -08001758 pr_err("%s: module sysfs not initialized\n", mod->name);
Ed Swierk1cc5f712006-09-25 16:25:36 -07001759 err = -EINVAL;
1760 goto out;
1761 }
Greg Kroah-Hartman6494a932008-01-27 15:38:40 -08001762
1763 kobj = kset_find_obj(module_kset, mod->name);
1764 if (kobj) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -08001765 pr_err("%s: module is already loaded\n", mod->name);
Greg Kroah-Hartman6494a932008-01-27 15:38:40 -08001766 kobject_put(kobj);
1767 err = -EINVAL;
1768 goto out;
1769 }
1770
Linus Torvalds1da177e2005-04-16 15:20:36 -07001771 mod->mkobj.mod = mod;
Kay Sieverse17e0f52006-11-24 12:15:25 +01001772
Greg Kroah-Hartmanac3c8142007-12-17 23:05:35 -07001773 memset(&mod->mkobj.kobj, 0, sizeof(mod->mkobj.kobj));
1774 mod->mkobj.kobj.kset = module_kset;
1775 err = kobject_init_and_add(&mod->mkobj.kobj, &module_ktype, NULL,
1776 "%s", mod->name);
1777 if (err)
Li Zhong942e4432013-09-03 16:33:57 +09301778 mod_kobject_put(mod);
Kay Sievers270a6c42007-01-18 13:26:15 +01001779
Kay Sievers97c146e2007-11-29 23:46:11 +01001780 /* delay uevent until full sysfs population */
Kay Sievers270a6c42007-01-18 13:26:15 +01001781out:
1782 return err;
1783}
1784
Rusty Russell6407ebb22010-06-05 11:17:36 -06001785static int mod_sysfs_setup(struct module *mod,
Rusty Russell8f6d0372010-08-05 12:59:09 -06001786 const struct load_info *info,
Kay Sievers270a6c42007-01-18 13:26:15 +01001787 struct kernel_param *kparam,
1788 unsigned int num_params)
1789{
1790 int err;
1791
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001792 err = mod_sysfs_init(mod);
1793 if (err)
1794 goto out;
1795
Greg Kroah-Hartman4ff6abf2007-11-05 22:24:43 -08001796 mod->holders_dir = kobject_create_and_add("holders", &mod->mkobj.kobj);
Akinobu Mita240936e2007-04-26 00:12:09 -07001797 if (!mod->holders_dir) {
1798 err = -ENOMEM;
Kay Sievers270a6c42007-01-18 13:26:15 +01001799 goto out_unreg;
Akinobu Mita240936e2007-04-26 00:12:09 -07001800 }
Kay Sievers270a6c42007-01-18 13:26:15 +01001801
Linus Torvalds1da177e2005-04-16 15:20:36 -07001802 err = module_param_sysfs_setup(mod, kparam, num_params);
1803 if (err)
Kay Sievers270a6c42007-01-18 13:26:15 +01001804 goto out_unreg_holders;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001805
Matt Domschc988d2b2005-06-23 22:05:15 -07001806 err = module_add_modinfo_attrs(mod);
1807 if (err)
Kay Sieverse17e0f52006-11-24 12:15:25 +01001808 goto out_unreg_param;
Matt Domschc988d2b2005-06-23 22:05:15 -07001809
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001810 add_usage_links(mod);
Rusty Russell8f6d0372010-08-05 12:59:09 -06001811 add_sect_attrs(mod, info);
1812 add_notes_attrs(mod, info);
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001813
Kay Sieverse17e0f52006-11-24 12:15:25 +01001814 kobject_uevent(&mod->mkobj.kobj, KOBJ_ADD);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001815 return 0;
1816
Kay Sieverse17e0f52006-11-24 12:15:25 +01001817out_unreg_param:
1818 module_param_sysfs_remove(mod);
Kay Sievers270a6c42007-01-18 13:26:15 +01001819out_unreg_holders:
Greg Kroah-Hartman78a2d902007-12-20 08:13:05 -08001820 kobject_put(mod->holders_dir);
Kay Sievers270a6c42007-01-18 13:26:15 +01001821out_unreg:
Li Zhong942e4432013-09-03 16:33:57 +09301822 mod_kobject_put(mod);
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001823out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001824 return err;
1825}
Denis V. Lunev34e4e2f2008-05-20 13:59:48 +04001826
1827static void mod_sysfs_fini(struct module *mod)
1828{
Rusty Russell8f6d0372010-08-05 12:59:09 -06001829 remove_notes_attrs(mod);
1830 remove_sect_attrs(mod);
Li Zhong942e4432013-09-03 16:33:57 +09301831 mod_kobject_put(mod);
Denis V. Lunev34e4e2f2008-05-20 13:59:48 +04001832}
1833
Rusty Russellcf2fde72015-06-26 06:44:38 +09301834static void init_param_lock(struct module *mod)
1835{
1836 mutex_init(&mod->param_lock);
1837}
Rusty Russell8f6d0372010-08-05 12:59:09 -06001838#else /* !CONFIG_SYSFS */
Denis V. Lunev34e4e2f2008-05-20 13:59:48 +04001839
Rusty Russell8f6d0372010-08-05 12:59:09 -06001840static int mod_sysfs_setup(struct module *mod,
1841 const struct load_info *info,
Rusty Russell6407ebb22010-06-05 11:17:36 -06001842 struct kernel_param *kparam,
1843 unsigned int num_params)
1844{
1845 return 0;
1846}
1847
Denis V. Lunev34e4e2f2008-05-20 13:59:48 +04001848static void mod_sysfs_fini(struct module *mod)
1849{
1850}
1851
Rusty Russell36b03602010-08-05 12:59:09 -06001852static void module_remove_modinfo_attrs(struct module *mod)
1853{
1854}
1855
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001856static void del_usage_links(struct module *mod)
1857{
1858}
1859
Rusty Russellcf2fde72015-06-26 06:44:38 +09301860static void init_param_lock(struct module *mod)
1861{
1862}
Denis V. Lunev34e4e2f2008-05-20 13:59:48 +04001863#endif /* CONFIG_SYSFS */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001864
Rusty Russell36b03602010-08-05 12:59:09 -06001865static void mod_sysfs_teardown(struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001866{
Rusty Russell80a3d1b2010-06-05 11:17:36 -06001867 del_usage_links(mod);
Matt Domschc988d2b2005-06-23 22:05:15 -07001868 module_remove_modinfo_attrs(mod);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001869 module_param_sysfs_remove(mod);
Greg Kroah-Hartman78a2d902007-12-20 08:13:05 -08001870 kobject_put(mod->mkobj.drivers_dir);
1871 kobject_put(mod->holders_dir);
Denis V. Lunev34e4e2f2008-05-20 13:59:48 +04001872 mod_sysfs_fini(mod);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001873}
1874
matthieu castet84e1c6b2010-11-16 22:35:16 +01001875#ifdef CONFIG_DEBUG_SET_MODULE_RONX
1876/*
1877 * LKM RO/NX protection: protect module's text/ro-data
1878 * from modification and any data from execution.
1879 */
1880void set_page_attributes(void *start, void *end, int (*set)(unsigned long start, int num_pages))
1881{
1882 unsigned long begin_pfn = PFN_DOWN((unsigned long)start);
1883 unsigned long end_pfn = PFN_DOWN((unsigned long)end);
1884
1885 if (end_pfn > begin_pfn)
1886 set(begin_pfn << PAGE_SHIFT, end_pfn - begin_pfn);
1887}
1888
1889static void set_section_ro_nx(void *base,
1890 unsigned long text_size,
1891 unsigned long ro_size,
1892 unsigned long total_size)
1893{
1894 /* begin and end PFNs of the current subsection */
1895 unsigned long begin_pfn;
1896 unsigned long end_pfn;
1897
1898 /*
1899 * Set RO for module text and RO-data:
1900 * - Always protect first page.
1901 * - Do not protect last partial page.
1902 */
1903 if (ro_size > 0)
1904 set_page_attributes(base, base + ro_size, set_memory_ro);
1905
1906 /*
1907 * Set NX permissions for module data:
1908 * - Do not protect first partial page.
1909 * - Always protect last page.
1910 */
1911 if (total_size > text_size) {
1912 begin_pfn = PFN_UP((unsigned long)base + text_size);
1913 end_pfn = PFN_UP((unsigned long)base + total_size);
1914 if (end_pfn > begin_pfn)
1915 set_memory_nx(begin_pfn << PAGE_SHIFT, end_pfn - begin_pfn);
1916 }
1917}
1918
Jan Glauber01526ed2011-05-19 16:55:26 -06001919static void unset_module_core_ro_nx(struct module *mod)
matthieu castet84e1c6b2010-11-16 22:35:16 +01001920{
Jan Glauber01526ed2011-05-19 16:55:26 -06001921 set_page_attributes(mod->module_core + mod->core_text_size,
1922 mod->module_core + mod->core_size,
1923 set_memory_x);
1924 set_page_attributes(mod->module_core,
1925 mod->module_core + mod->core_ro_size,
1926 set_memory_rw);
1927}
1928
1929static void unset_module_init_ro_nx(struct module *mod)
1930{
1931 set_page_attributes(mod->module_init + mod->init_text_size,
1932 mod->module_init + mod->init_size,
1933 set_memory_x);
1934 set_page_attributes(mod->module_init,
1935 mod->module_init + mod->init_ro_size,
1936 set_memory_rw);
matthieu castet84e1c6b2010-11-16 22:35:16 +01001937}
1938
1939/* Iterate through all modules and set each module's text as RW */
Daniel J Blueman5d05c702011-03-08 22:01:47 +08001940void set_all_modules_text_rw(void)
matthieu castet84e1c6b2010-11-16 22:35:16 +01001941{
1942 struct module *mod;
1943
1944 mutex_lock(&module_mutex);
1945 list_for_each_entry_rcu(mod, &modules, list) {
Rusty Russell0d21b0e2013-01-12 11:38:44 +10301946 if (mod->state == MODULE_STATE_UNFORMED)
1947 continue;
matthieu castet84e1c6b2010-11-16 22:35:16 +01001948 if ((mod->module_core) && (mod->core_text_size)) {
1949 set_page_attributes(mod->module_core,
1950 mod->module_core + mod->core_text_size,
1951 set_memory_rw);
1952 }
1953 if ((mod->module_init) && (mod->init_text_size)) {
1954 set_page_attributes(mod->module_init,
1955 mod->module_init + mod->init_text_size,
1956 set_memory_rw);
1957 }
1958 }
1959 mutex_unlock(&module_mutex);
1960}
1961
1962/* Iterate through all modules and set each module's text as RO */
Daniel J Blueman5d05c702011-03-08 22:01:47 +08001963void set_all_modules_text_ro(void)
matthieu castet84e1c6b2010-11-16 22:35:16 +01001964{
1965 struct module *mod;
1966
1967 mutex_lock(&module_mutex);
1968 list_for_each_entry_rcu(mod, &modules, list) {
Rusty Russell0d21b0e2013-01-12 11:38:44 +10301969 if (mod->state == MODULE_STATE_UNFORMED)
1970 continue;
matthieu castet84e1c6b2010-11-16 22:35:16 +01001971 if ((mod->module_core) && (mod->core_text_size)) {
1972 set_page_attributes(mod->module_core,
1973 mod->module_core + mod->core_text_size,
1974 set_memory_ro);
1975 }
1976 if ((mod->module_init) && (mod->init_text_size)) {
1977 set_page_attributes(mod->module_init,
1978 mod->module_init + mod->init_text_size,
1979 set_memory_ro);
1980 }
1981 }
1982 mutex_unlock(&module_mutex);
1983}
1984#else
1985static inline void set_section_ro_nx(void *base, unsigned long text_size, unsigned long ro_size, unsigned long total_size) { }
Jan Glauber01526ed2011-05-19 16:55:26 -06001986static void unset_module_core_ro_nx(struct module *mod) { }
1987static void unset_module_init_ro_nx(struct module *mod) { }
matthieu castet84e1c6b2010-11-16 22:35:16 +01001988#endif
1989
Rusty Russellbe1f2212015-01-20 09:07:05 +10301990void __weak module_memfree(void *module_region)
Jonas Bonn74e08fc2011-06-30 21:22:11 +02001991{
1992 vfree(module_region);
1993}
1994
1995void __weak module_arch_cleanup(struct module *mod)
1996{
1997}
1998
Rusty Russelld453cde2015-01-20 09:07:04 +10301999void __weak module_arch_freeing_init(struct module *mod)
2000{
2001}
2002
Rusty Russell75676502010-06-05 11:17:36 -06002003/* Free a module, remove from lists, etc. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002004static void free_module(struct module *mod)
2005{
Li Zefan7ead8b82009-08-17 16:56:28 +08002006 trace_module_free(mod);
2007
Rusty Russell36b03602010-08-05 12:59:09 -06002008 mod_sysfs_teardown(mod);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002009
Rusty Russell944a1fa2013-04-17 13:20:03 +09302010 /* We leave it in list to prevent duplicate loads, but make sure
2011 * that noone uses it while it's being deconstructed. */
Prarit Bhargavad3051b42014-10-14 02:51:39 +10302012 mutex_lock(&module_mutex);
Rusty Russell944a1fa2013-04-17 13:20:03 +09302013 mod->state = MODULE_STATE_UNFORMED;
Prarit Bhargavad3051b42014-10-14 02:51:39 +10302014 mutex_unlock(&module_mutex);
Rusty Russell944a1fa2013-04-17 13:20:03 +09302015
Jason Baronb82bab4b2010-07-27 13:18:01 -07002016 /* Remove dynamic debug info */
2017 ddebug_remove_module(mod->name);
2018
Linus Torvalds1da177e2005-04-16 15:20:36 -07002019 /* Arch-specific cleanup. */
2020 module_arch_cleanup(mod);
2021
2022 /* Module unload stuff */
2023 module_unload_free(mod);
2024
Rusty Russelle180a6b2009-03-31 13:05:29 -06002025 /* Free any allocated parameters. */
2026 destroy_params(mod->kp, mod->num_kp);
2027
Rusty Russell944a1fa2013-04-17 13:20:03 +09302028 /* Now we can delete it from the lists */
2029 mutex_lock(&module_mutex);
Masami Hiramatsu461e34a2014-11-10 09:27:29 +10302030 /* Unlink carefully: kallsyms could be walking list. */
2031 list_del_rcu(&mod->list);
Peter Zijlstra93c2e102015-05-27 11:09:37 +09302032 mod_tree_remove(mod);
Masami Hiramatsu0286b5e2014-11-10 09:28:29 +10302033 /* Remove this module from bug list, this uses list_del_rcu */
Masami Hiramatsu461e34a2014-11-10 09:27:29 +10302034 module_bug_cleanup(mod);
Peter Zijlstra0be964b2015-05-27 11:09:35 +09302035 /* Wait for RCU-sched synchronizing before releasing mod->list and buglist. */
2036 synchronize_sched();
Rusty Russell944a1fa2013-04-17 13:20:03 +09302037 mutex_unlock(&module_mutex);
2038
Linus Torvalds1da177e2005-04-16 15:20:36 -07002039 /* This may be NULL, but that's OK */
Jan Glauber01526ed2011-05-19 16:55:26 -06002040 unset_module_init_ro_nx(mod);
Rusty Russelld453cde2015-01-20 09:07:04 +10302041 module_arch_freeing_init(mod);
Rusty Russellbe1f2212015-01-20 09:07:05 +10302042 module_memfree(mod->module_init);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002043 kfree(mod->args);
Tejun Heo259354d2010-03-10 18:56:10 +09002044 percpu_modfree(mod);
Rusty Russell9f85a4b2010-08-05 12:59:04 -06002045
Peter Zijlstra35a93932015-02-26 16:23:11 +01002046 /* Free lock-classes; relies on the preceding sync_rcu(). */
Ingo Molnarfbb9ce952006-07-03 00:24:50 -07002047 lockdep_free_key_range(mod->module_core, mod->core_size);
2048
Linus Torvalds1da177e2005-04-16 15:20:36 -07002049 /* Finally, free the core (containing the module structure) */
Jan Glauber01526ed2011-05-19 16:55:26 -06002050 unset_module_core_ro_nx(mod);
Rusty Russellbe1f2212015-01-20 09:07:05 +10302051 module_memfree(mod->module_core);
Bernd Schmidteb8cdec2009-09-21 17:03:57 -07002052
2053#ifdef CONFIG_MPU
2054 update_protections(current->mm);
2055#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002056}
2057
2058void *__symbol_get(const char *symbol)
2059{
2060 struct module *owner;
Tim Abbott414fd312008-12-05 19:03:56 -05002061 const struct kernel_symbol *sym;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002062
Rusty Russell24da1cb2007-07-15 23:41:46 -07002063 preempt_disable();
Tim Abbott414fd312008-12-05 19:03:56 -05002064 sym = find_symbol(symbol, &owner, NULL, true, true);
2065 if (sym && strong_try_module_get(owner))
2066 sym = NULL;
Rusty Russell24da1cb2007-07-15 23:41:46 -07002067 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002068
Tim Abbott414fd312008-12-05 19:03:56 -05002069 return sym ? (void *)sym->value : NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002070}
2071EXPORT_SYMBOL_GPL(__symbol_get);
2072
Ashutosh Naikeea8b542006-01-08 01:04:25 -08002073/*
2074 * Ensure that an exported symbol [global namespace] does not already exist
Robert P. J. Day02a3e592007-05-09 07:26:28 +02002075 * in the kernel or in some other module's exported symbol table.
Rusty Russellbe593f42010-06-05 11:17:37 -06002076 *
2077 * You must hold the module_mutex.
Ashutosh Naikeea8b542006-01-08 01:04:25 -08002078 */
2079static int verify_export_symbols(struct module *mod)
2080{
Rusty Russellb2111042008-05-01 21:15:00 -05002081 unsigned int i;
Ashutosh Naikeea8b542006-01-08 01:04:25 -08002082 struct module *owner;
Rusty Russellb2111042008-05-01 21:15:00 -05002083 const struct kernel_symbol *s;
2084 struct {
2085 const struct kernel_symbol *sym;
2086 unsigned int num;
2087 } arr[] = {
2088 { mod->syms, mod->num_syms },
2089 { mod->gpl_syms, mod->num_gpl_syms },
2090 { mod->gpl_future_syms, mod->num_gpl_future_syms },
Denys Vlasenkof7f5b672008-07-22 19:24:26 -05002091#ifdef CONFIG_UNUSED_SYMBOLS
Rusty Russellb2111042008-05-01 21:15:00 -05002092 { mod->unused_syms, mod->num_unused_syms },
2093 { mod->unused_gpl_syms, mod->num_unused_gpl_syms },
Denys Vlasenkof7f5b672008-07-22 19:24:26 -05002094#endif
Rusty Russellb2111042008-05-01 21:15:00 -05002095 };
Ashutosh Naikeea8b542006-01-08 01:04:25 -08002096
Rusty Russellb2111042008-05-01 21:15:00 -05002097 for (i = 0; i < ARRAY_SIZE(arr); i++) {
2098 for (s = arr[i].sym; s < arr[i].sym + arr[i].num; s++) {
Rusty Russellbe593f42010-06-05 11:17:37 -06002099 if (find_symbol(s->name, &owner, NULL, true, false)) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -08002100 pr_err("%s: exports duplicate symbol %s"
Rusty Russellb2111042008-05-01 21:15:00 -05002101 " (owned by %s)\n",
2102 mod->name, s->name, module_name(owner));
2103 return -ENOEXEC;
2104 }
Ashutosh Naikeea8b542006-01-08 01:04:25 -08002105 }
Rusty Russellb2111042008-05-01 21:15:00 -05002106 }
2107 return 0;
Ashutosh Naikeea8b542006-01-08 01:04:25 -08002108}
2109
Matti Linnanvuori9a4b9702007-11-08 08:37:38 -08002110/* Change all symbols so that st_value encodes the pointer directly. */
Rusty Russell49668682010-08-05 12:59:10 -06002111static int simplify_symbols(struct module *mod, const struct load_info *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002112{
Rusty Russell49668682010-08-05 12:59:10 -06002113 Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
2114 Elf_Sym *sym = (void *)symsec->sh_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002115 unsigned long secbase;
Rusty Russell49668682010-08-05 12:59:10 -06002116 unsigned int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002117 int ret = 0;
Tim Abbott414fd312008-12-05 19:03:56 -05002118 const struct kernel_symbol *ksym;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002119
Rusty Russell49668682010-08-05 12:59:10 -06002120 for (i = 1; i < symsec->sh_size / sizeof(Elf_Sym); i++) {
2121 const char *name = info->strtab + sym[i].st_name;
2122
Linus Torvalds1da177e2005-04-16 15:20:36 -07002123 switch (sym[i].st_shndx) {
2124 case SHN_COMMON:
Joe Mario80375982014-02-08 09:01:09 +01002125 /* Ignore common symbols */
2126 if (!strncmp(name, "__gnu_lto", 9))
2127 break;
2128
Linus Torvalds1da177e2005-04-16 15:20:36 -07002129 /* We compiled with -fno-common. These are not
2130 supposed to happen. */
Jim Cromie5e124162011-12-06 12:11:31 -07002131 pr_debug("Common symbol: %s\n", name);
Ionut Alexa6da0b562014-11-10 09:31:29 +10302132 pr_warn("%s: please compile with -fno-common\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07002133 mod->name);
2134 ret = -ENOEXEC;
2135 break;
2136
2137 case SHN_ABS:
2138 /* Don't need to do anything */
Jim Cromie5e124162011-12-06 12:11:31 -07002139 pr_debug("Absolute symbol: 0x%08lx\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07002140 (long)sym[i].st_value);
2141 break;
2142
2143 case SHN_UNDEF:
Rusty Russell49668682010-08-05 12:59:10 -06002144 ksym = resolve_symbol_wait(mod, info, name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002145 /* Ok if resolved. */
Rusty Russell9bea7f22010-06-05 11:17:37 -06002146 if (ksym && !IS_ERR(ksym)) {
Tim Abbott414fd312008-12-05 19:03:56 -05002147 sym[i].st_value = ksym->value;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002148 break;
Tim Abbott414fd312008-12-05 19:03:56 -05002149 }
2150
Linus Torvalds1da177e2005-04-16 15:20:36 -07002151 /* Ok if weak. */
Rusty Russell9bea7f22010-06-05 11:17:37 -06002152 if (!ksym && ELF_ST_BIND(sym[i].st_info) == STB_WEAK)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002153 break;
2154
Andrew Mortonbddb12b2013-11-12 15:11:28 -08002155 pr_warn("%s: Unknown symbol %s (err %li)\n",
2156 mod->name, name, PTR_ERR(ksym));
Rusty Russell9bea7f22010-06-05 11:17:37 -06002157 ret = PTR_ERR(ksym) ?: -ENOENT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002158 break;
2159
2160 default:
2161 /* Divert to percpu allocation if a percpu var. */
Rusty Russell49668682010-08-05 12:59:10 -06002162 if (sym[i].st_shndx == info->index.pcpu)
Tejun Heo259354d2010-03-10 18:56:10 +09002163 secbase = (unsigned long)mod_percpu(mod);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002164 else
Rusty Russell49668682010-08-05 12:59:10 -06002165 secbase = info->sechdrs[sym[i].st_shndx].sh_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002166 sym[i].st_value += secbase;
2167 break;
2168 }
2169 }
2170
2171 return ret;
2172}
2173
Rusty Russell49668682010-08-05 12:59:10 -06002174static int apply_relocations(struct module *mod, const struct load_info *info)
Rusty Russell22e268e2010-08-05 12:59:05 -06002175{
2176 unsigned int i;
2177 int err = 0;
2178
2179 /* Now do relocations. */
Rusty Russell49668682010-08-05 12:59:10 -06002180 for (i = 1; i < info->hdr->e_shnum; i++) {
2181 unsigned int infosec = info->sechdrs[i].sh_info;
Rusty Russell22e268e2010-08-05 12:59:05 -06002182
2183 /* Not a valid relocation section? */
Rusty Russell49668682010-08-05 12:59:10 -06002184 if (infosec >= info->hdr->e_shnum)
Rusty Russell22e268e2010-08-05 12:59:05 -06002185 continue;
2186
2187 /* Don't bother with non-allocated sections */
Rusty Russell49668682010-08-05 12:59:10 -06002188 if (!(info->sechdrs[infosec].sh_flags & SHF_ALLOC))
Rusty Russell22e268e2010-08-05 12:59:05 -06002189 continue;
2190
Rusty Russell49668682010-08-05 12:59:10 -06002191 if (info->sechdrs[i].sh_type == SHT_REL)
2192 err = apply_relocate(info->sechdrs, info->strtab,
2193 info->index.sym, i, mod);
2194 else if (info->sechdrs[i].sh_type == SHT_RELA)
2195 err = apply_relocate_add(info->sechdrs, info->strtab,
2196 info->index.sym, i, mod);
Rusty Russell22e268e2010-08-05 12:59:05 -06002197 if (err < 0)
2198 break;
2199 }
2200 return err;
2201}
2202
Helge Deller088af9a2008-12-31 12:31:18 +01002203/* Additional bytes needed by arch in front of individual sections */
2204unsigned int __weak arch_mod_section_prepend(struct module *mod,
2205 unsigned int section)
2206{
2207 /* default implementation just returns zero */
2208 return 0;
2209}
2210
Linus Torvalds1da177e2005-04-16 15:20:36 -07002211/* Update size with this section: return offset. */
Helge Deller088af9a2008-12-31 12:31:18 +01002212static long get_offset(struct module *mod, unsigned int *size,
2213 Elf_Shdr *sechdr, unsigned int section)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002214{
2215 long ret;
2216
Helge Deller088af9a2008-12-31 12:31:18 +01002217 *size += arch_mod_section_prepend(mod, section);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002218 ret = ALIGN(*size, sechdr->sh_addralign ?: 1);
2219 *size = ret + sechdr->sh_size;
2220 return ret;
2221}
2222
2223/* Lay out the SHF_ALLOC sections in a way not dissimilar to how ld
2224 might -- code, read-only data, read-write data, small data. Tally
2225 sizes, and place the offsets into sh_entsize fields: high bit means it
2226 belongs in init. */
Rusty Russell49668682010-08-05 12:59:10 -06002227static void layout_sections(struct module *mod, struct load_info *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002228{
2229 static unsigned long const masks[][2] = {
2230 /* NOTE: all executable code must be the first section
2231 * in this array; otherwise modify the text_size
2232 * finder in the two loops below */
2233 { SHF_EXECINSTR | SHF_ALLOC, ARCH_SHF_SMALL },
2234 { SHF_ALLOC, SHF_WRITE | ARCH_SHF_SMALL },
2235 { SHF_WRITE | SHF_ALLOC, ARCH_SHF_SMALL },
2236 { ARCH_SHF_SMALL | SHF_ALLOC, 0 }
2237 };
2238 unsigned int m, i;
2239
Rusty Russell49668682010-08-05 12:59:10 -06002240 for (i = 0; i < info->hdr->e_shnum; i++)
2241 info->sechdrs[i].sh_entsize = ~0UL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002242
Jim Cromie5e124162011-12-06 12:11:31 -07002243 pr_debug("Core section allocation order:\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002244 for (m = 0; m < ARRAY_SIZE(masks); ++m) {
Rusty Russell49668682010-08-05 12:59:10 -06002245 for (i = 0; i < info->hdr->e_shnum; ++i) {
2246 Elf_Shdr *s = &info->sechdrs[i];
2247 const char *sname = info->secstrings + s->sh_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002248
2249 if ((s->sh_flags & masks[m][0]) != masks[m][0]
2250 || (s->sh_flags & masks[m][1])
2251 || s->sh_entsize != ~0UL
Rusty Russell49668682010-08-05 12:59:10 -06002252 || strstarts(sname, ".init"))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002253 continue;
Helge Deller088af9a2008-12-31 12:31:18 +01002254 s->sh_entsize = get_offset(mod, &mod->core_size, s, i);
Jim Cromie5e124162011-12-06 12:11:31 -07002255 pr_debug("\t%s\n", sname);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002256 }
matthieu castet84e1c6b2010-11-16 22:35:16 +01002257 switch (m) {
2258 case 0: /* executable */
2259 mod->core_size = debug_align(mod->core_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002260 mod->core_text_size = mod->core_size;
matthieu castet84e1c6b2010-11-16 22:35:16 +01002261 break;
2262 case 1: /* RO: text and ro-data */
2263 mod->core_size = debug_align(mod->core_size);
2264 mod->core_ro_size = mod->core_size;
2265 break;
2266 case 3: /* whole core */
2267 mod->core_size = debug_align(mod->core_size);
2268 break;
2269 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002270 }
2271
Jim Cromie5e124162011-12-06 12:11:31 -07002272 pr_debug("Init section allocation order:\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07002273 for (m = 0; m < ARRAY_SIZE(masks); ++m) {
Rusty Russell49668682010-08-05 12:59:10 -06002274 for (i = 0; i < info->hdr->e_shnum; ++i) {
2275 Elf_Shdr *s = &info->sechdrs[i];
2276 const char *sname = info->secstrings + s->sh_name;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002277
2278 if ((s->sh_flags & masks[m][0]) != masks[m][0]
2279 || (s->sh_flags & masks[m][1])
2280 || s->sh_entsize != ~0UL
Rusty Russell49668682010-08-05 12:59:10 -06002281 || !strstarts(sname, ".init"))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002282 continue;
Helge Deller088af9a2008-12-31 12:31:18 +01002283 s->sh_entsize = (get_offset(mod, &mod->init_size, s, i)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002284 | INIT_OFFSET_MASK);
Jim Cromie5e124162011-12-06 12:11:31 -07002285 pr_debug("\t%s\n", sname);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002286 }
matthieu castet84e1c6b2010-11-16 22:35:16 +01002287 switch (m) {
2288 case 0: /* executable */
2289 mod->init_size = debug_align(mod->init_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002290 mod->init_text_size = mod->init_size;
matthieu castet84e1c6b2010-11-16 22:35:16 +01002291 break;
2292 case 1: /* RO: text and ro-data */
2293 mod->init_size = debug_align(mod->init_size);
2294 mod->init_ro_size = mod->init_size;
2295 break;
2296 case 3: /* whole init */
2297 mod->init_size = debug_align(mod->init_size);
2298 break;
2299 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002300 }
2301}
2302
Linus Torvalds1da177e2005-04-16 15:20:36 -07002303static void set_license(struct module *mod, const char *license)
2304{
2305 if (!license)
2306 license = "unspecified";
2307
Florin Malitafa3ba2e82006-10-11 01:21:48 -07002308 if (!license_is_gpl_compatible(license)) {
Andi Kleen25ddbb12008-10-15 22:01:41 -07002309 if (!test_taint(TAINT_PROPRIETARY_MODULE))
Andrew Mortonbddb12b2013-11-12 15:11:28 -08002310 pr_warn("%s: module license '%s' taints kernel.\n",
2311 mod->name, license);
Rusty Russell373d4d02013-01-21 17:17:39 +10302312 add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
2313 LOCKDEP_NOW_UNRELIABLE);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002314 }
2315}
2316
2317/* Parse tag=value strings from .modinfo section */
2318static char *next_string(char *string, unsigned long *secsize)
2319{
2320 /* Skip non-zero chars */
2321 while (string[0]) {
2322 string++;
2323 if ((*secsize)-- <= 1)
2324 return NULL;
2325 }
2326
2327 /* Skip any zero padding. */
2328 while (!string[0]) {
2329 string++;
2330 if ((*secsize)-- <= 1)
2331 return NULL;
2332 }
2333 return string;
2334}
2335
Rusty Russell49668682010-08-05 12:59:10 -06002336static char *get_modinfo(struct load_info *info, const char *tag)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002337{
2338 char *p;
2339 unsigned int taglen = strlen(tag);
Rusty Russell49668682010-08-05 12:59:10 -06002340 Elf_Shdr *infosec = &info->sechdrs[info->index.info];
2341 unsigned long size = infosec->sh_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002342
Rusty Russell49668682010-08-05 12:59:10 -06002343 for (p = (char *)infosec->sh_addr; p; p = next_string(p, &size)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002344 if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=')
2345 return p + taglen + 1;
2346 }
2347 return NULL;
2348}
2349
Rusty Russell49668682010-08-05 12:59:10 -06002350static void setup_modinfo(struct module *mod, struct load_info *info)
Matt Domschc988d2b2005-06-23 22:05:15 -07002351{
2352 struct module_attribute *attr;
2353 int i;
2354
2355 for (i = 0; (attr = modinfo_attrs[i]); i++) {
2356 if (attr->setup)
Rusty Russell49668682010-08-05 12:59:10 -06002357 attr->setup(mod, get_modinfo(info, attr->attr.name));
Matt Domschc988d2b2005-06-23 22:05:15 -07002358 }
2359}
Matt Domschc988d2b2005-06-23 22:05:15 -07002360
Rusty Russella263f772009-09-25 00:32:58 -06002361static void free_modinfo(struct module *mod)
2362{
2363 struct module_attribute *attr;
2364 int i;
2365
2366 for (i = 0; (attr = modinfo_attrs[i]); i++) {
2367 if (attr->free)
2368 attr->free(mod);
2369 }
2370}
2371
Linus Torvalds1da177e2005-04-16 15:20:36 -07002372#ifdef CONFIG_KALLSYMS
WANG Cong15bba372008-07-24 15:41:48 +01002373
2374/* lookup symbol in given range of kernel_symbols */
2375static const struct kernel_symbol *lookup_symbol(const char *name,
2376 const struct kernel_symbol *start,
2377 const struct kernel_symbol *stop)
2378{
Alessio Igor Bogani9d634872011-05-18 22:35:59 +02002379 return bsearch(name, start, stop - start,
2380 sizeof(struct kernel_symbol), cmp_name);
WANG Cong15bba372008-07-24 15:41:48 +01002381}
2382
Tim Abbottca4787b2009-01-05 08:40:10 -06002383static int is_exported(const char *name, unsigned long value,
2384 const struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002385{
Tim Abbottca4787b2009-01-05 08:40:10 -06002386 const struct kernel_symbol *ks;
2387 if (!mod)
2388 ks = lookup_symbol(name, __start___ksymtab, __stop___ksymtab);
Sam Ravnborg3fd68052006-02-08 21:16:45 +01002389 else
Tim Abbottca4787b2009-01-05 08:40:10 -06002390 ks = lookup_symbol(name, mod->syms, mod->syms + mod->num_syms);
2391 return ks != NULL && ks->value == value;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002392}
2393
2394/* As per nm */
Rusty Russelleded41c2010-08-05 12:59:07 -06002395static char elf_type(const Elf_Sym *sym, const struct load_info *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002396{
Rusty Russelleded41c2010-08-05 12:59:07 -06002397 const Elf_Shdr *sechdrs = info->sechdrs;
2398
Linus Torvalds1da177e2005-04-16 15:20:36 -07002399 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) {
2400 if (ELF_ST_TYPE(sym->st_info) == STT_OBJECT)
2401 return 'v';
2402 else
2403 return 'w';
2404 }
2405 if (sym->st_shndx == SHN_UNDEF)
2406 return 'U';
2407 if (sym->st_shndx == SHN_ABS)
2408 return 'a';
2409 if (sym->st_shndx >= SHN_LORESERVE)
2410 return '?';
2411 if (sechdrs[sym->st_shndx].sh_flags & SHF_EXECINSTR)
2412 return 't';
2413 if (sechdrs[sym->st_shndx].sh_flags & SHF_ALLOC
2414 && sechdrs[sym->st_shndx].sh_type != SHT_NOBITS) {
2415 if (!(sechdrs[sym->st_shndx].sh_flags & SHF_WRITE))
2416 return 'r';
2417 else if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
2418 return 'g';
2419 else
2420 return 'd';
2421 }
2422 if (sechdrs[sym->st_shndx].sh_type == SHT_NOBITS) {
2423 if (sechdrs[sym->st_shndx].sh_flags & ARCH_SHF_SMALL)
2424 return 's';
2425 else
2426 return 'b';
2427 }
Rusty Russelleded41c2010-08-05 12:59:07 -06002428 if (strstarts(info->secstrings + sechdrs[sym->st_shndx].sh_name,
2429 ".debug")) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430 return 'n';
Rusty Russelleded41c2010-08-05 12:59:07 -06002431 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002432 return '?';
2433}
2434
Jan Beulich4a496222009-07-06 14:50:42 +01002435static bool is_core_symbol(const Elf_Sym *src, const Elf_Shdr *sechdrs,
Ionut Alexa6da0b562014-11-10 09:31:29 +10302436 unsigned int shnum)
Jan Beulich4a496222009-07-06 14:50:42 +01002437{
2438 const Elf_Shdr *sec;
2439
2440 if (src->st_shndx == SHN_UNDEF
2441 || src->st_shndx >= shnum
2442 || !src->st_name)
2443 return false;
2444
2445 sec = sechdrs + src->st_shndx;
2446 if (!(sec->sh_flags & SHF_ALLOC)
2447#ifndef CONFIG_KALLSYMS_ALL
2448 || !(sec->sh_flags & SHF_EXECINSTR)
2449#endif
2450 || (sec->sh_entsize & INIT_OFFSET_MASK))
2451 return false;
2452
2453 return true;
2454}
2455
Kevin Cernekee48fd1182012-01-13 09:32:14 +10302456/*
2457 * We only allocate and copy the strings needed by the parts of symtab
2458 * we keep. This is simple, but has the effect of making multiple
2459 * copies of duplicates. We could be more sophisticated, see
2460 * linux-kernel thread starting with
2461 * <73defb5e4bca04a6431392cc341112b1@localhost>.
2462 */
Rusty Russell49668682010-08-05 12:59:10 -06002463static void layout_symtab(struct module *mod, struct load_info *info)
Jan Beulich4a496222009-07-06 14:50:42 +01002464{
Rusty Russell49668682010-08-05 12:59:10 -06002465 Elf_Shdr *symsect = info->sechdrs + info->index.sym;
2466 Elf_Shdr *strsect = info->sechdrs + info->index.str;
Jan Beulich4a496222009-07-06 14:50:42 +01002467 const Elf_Sym *src;
Satoru Takeuchi54523ec2012-12-05 12:29:04 +10302468 unsigned int i, nsrc, ndst, strtab_size = 0;
Jan Beulich4a496222009-07-06 14:50:42 +01002469
2470 /* Put symbol section at end of init part of module. */
2471 symsect->sh_flags |= SHF_ALLOC;
2472 symsect->sh_entsize = get_offset(mod, &mod->init_size, symsect,
Rusty Russell49668682010-08-05 12:59:10 -06002473 info->index.sym) | INIT_OFFSET_MASK;
Jim Cromie5e124162011-12-06 12:11:31 -07002474 pr_debug("\t%s\n", info->secstrings + symsect->sh_name);
Jan Beulich4a496222009-07-06 14:50:42 +01002475
Rusty Russell49668682010-08-05 12:59:10 -06002476 src = (void *)info->hdr + symsect->sh_offset;
Jan Beulich4a496222009-07-06 14:50:42 +01002477 nsrc = symsect->sh_size / sizeof(*src);
Kevin Cernekee70b1e9162011-11-12 19:08:55 -08002478
Kevin Cernekee48fd1182012-01-13 09:32:14 +10302479 /* Compute total space required for the core symbols' strtab. */
Rusty Russell59ef28b2012-10-25 10:49:25 +10302480 for (ndst = i = 0; i < nsrc; i++) {
2481 if (i == 0 ||
2482 is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum)) {
2483 strtab_size += strlen(&info->strtab[src[i].st_name])+1;
Kevin Cernekee48fd1182012-01-13 09:32:14 +10302484 ndst++;
Jan Beulich554bdfe2009-07-06 14:51:44 +01002485 }
Rusty Russell59ef28b2012-10-25 10:49:25 +10302486 }
Jan Beulich4a496222009-07-06 14:50:42 +01002487
2488 /* Append room for core symbols at end of core part. */
Rusty Russell49668682010-08-05 12:59:10 -06002489 info->symoffs = ALIGN(mod->core_size, symsect->sh_addralign ?: 1);
Kevin Cernekee48fd1182012-01-13 09:32:14 +10302490 info->stroffs = mod->core_size = info->symoffs + ndst * sizeof(Elf_Sym);
2491 mod->core_size += strtab_size;
Laura Abbott168e47f2015-02-25 14:14:57 -08002492 mod->core_size = debug_align(mod->core_size);
Jan Beulich4a496222009-07-06 14:50:42 +01002493
Jan Beulich554bdfe2009-07-06 14:51:44 +01002494 /* Put string table section at end of init part of module. */
2495 strsect->sh_flags |= SHF_ALLOC;
2496 strsect->sh_entsize = get_offset(mod, &mod->init_size, strsect,
Rusty Russell49668682010-08-05 12:59:10 -06002497 info->index.str) | INIT_OFFSET_MASK;
Jim Cromie5e124162011-12-06 12:11:31 -07002498 pr_debug("\t%s\n", info->secstrings + strsect->sh_name);
Rusty Russell610dde52016-02-03 16:55:26 +10302499
2500 /* We'll tack temporary mod_kallsyms on the end. */
2501 mod->init_size = ALIGN(mod->init_size,
2502 __alignof__(struct mod_kallsyms));
2503 info->mod_kallsyms_init_off = mod->init_size;
2504 mod->init_size += sizeof(struct mod_kallsyms);
2505 mod->init_size = debug_align(mod->init_size);
Jan Beulich4a496222009-07-06 14:50:42 +01002506}
2507
Rusty Russell610dde52016-02-03 16:55:26 +10302508/*
2509 * We use the full symtab and strtab which layout_symtab arranged to
2510 * be appended to the init section. Later we switch to the cut-down
2511 * core-only ones.
2512 */
Rusty Russell811d66a2010-08-05 12:59:12 -06002513static void add_kallsyms(struct module *mod, const struct load_info *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002514{
Jan Beulich4a496222009-07-06 14:50:42 +01002515 unsigned int i, ndst;
2516 const Elf_Sym *src;
2517 Elf_Sym *dst;
Jan Beulich554bdfe2009-07-06 14:51:44 +01002518 char *s;
Rusty Russelleded41c2010-08-05 12:59:07 -06002519 Elf_Shdr *symsec = &info->sechdrs[info->index.sym];
Linus Torvalds1da177e2005-04-16 15:20:36 -07002520
Rusty Russell610dde52016-02-03 16:55:26 +10302521 /* Set up to point into init section. */
2522 mod->kallsyms = mod->module_init + info->mod_kallsyms_init_off;
2523
2524 mod->kallsyms->symtab = (void *)symsec->sh_addr;
2525 mod->kallsyms->num_symtab = symsec->sh_size / sizeof(Elf_Sym);
Rusty Russell511ca6a2010-08-05 12:59:08 -06002526 /* Make sure we get permanent strtab: don't use info->strtab. */
Rusty Russell610dde52016-02-03 16:55:26 +10302527 mod->kallsyms->strtab = (void *)info->sechdrs[info->index.str].sh_addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002528
2529 /* Set types up while we still have access to sections. */
Rusty Russell610dde52016-02-03 16:55:26 +10302530 for (i = 0; i < mod->kallsyms->num_symtab; i++)
2531 mod->kallsyms->symtab[i].st_info
2532 = elf_type(&mod->kallsyms->symtab[i], info);
Jan Beulich4a496222009-07-06 14:50:42 +01002533
Rusty Russell610dde52016-02-03 16:55:26 +10302534 /* Now populate the cut down core kallsyms for after init. */
2535 mod->core_kallsyms.symtab = dst = mod->module_core + info->symoffs;
2536 mod->core_kallsyms.strtab = s = mod->module_core + info->stroffs;
2537 src = mod->kallsyms->symtab;
2538 for (ndst = i = 0; i < mod->kallsyms->num_symtab; i++) {
Rusty Russell59ef28b2012-10-25 10:49:25 +10302539 if (i == 0 ||
2540 is_core_symbol(src+i, info->sechdrs, info->hdr->e_shnum)) {
2541 dst[ndst] = src[i];
Rusty Russell610dde52016-02-03 16:55:26 +10302542 dst[ndst++].st_name = s - mod->core_kallsyms.strtab;
2543 s += strlcpy(s, &mod->kallsyms->strtab[src[i].st_name],
Rusty Russell59ef28b2012-10-25 10:49:25 +10302544 KSYM_NAME_LEN) + 1;
2545 }
Jan Beulich4a496222009-07-06 14:50:42 +01002546 }
Rusty Russell610dde52016-02-03 16:55:26 +10302547 mod->core_kallsyms.num_symtab = ndst;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002548}
2549#else
Rusty Russell49668682010-08-05 12:59:10 -06002550static inline void layout_symtab(struct module *mod, struct load_info *info)
Jan Beulich4a496222009-07-06 14:50:42 +01002551{
2552}
Paul Mundt3ae91c22009-10-01 15:43:54 -07002553
Michał Mirosławabbce902010-09-20 01:58:08 +02002554static void add_kallsyms(struct module *mod, const struct load_info *info)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002555{
2556}
2557#endif /* CONFIG_KALLSYMS */
2558
Jason Barone9d376f2009-02-05 11:51:38 -05002559static void dynamic_debug_setup(struct _ddebug *debug, unsigned int num)
Rusty Russell5e458cc2008-10-22 10:00:13 -05002560{
Rusty Russell811d66a2010-08-05 12:59:12 -06002561 if (!debug)
2562 return;
Jason Barone9d376f2009-02-05 11:51:38 -05002563#ifdef CONFIG_DYNAMIC_DEBUG
2564 if (ddebug_add_module(debug, num, debug->modname))
Andrew Mortonbddb12b2013-11-12 15:11:28 -08002565 pr_err("dynamic debug error adding module: %s\n",
2566 debug->modname);
Jason Barone9d376f2009-02-05 11:51:38 -05002567#endif
Rusty Russell5e458cc2008-10-22 10:00:13 -05002568}
Jason Baron346e15b2008-08-12 16:46:19 -04002569
Yehuda Sadehff49d742010-07-03 13:07:35 +10002570static void dynamic_debug_remove(struct _ddebug *debug)
2571{
2572 if (debug)
2573 ddebug_remove_module(debug->modname);
2574}
2575
Jonas Bonn74e08fc2011-06-30 21:22:11 +02002576void * __weak module_alloc(unsigned long size)
2577{
Rusty Russell82fab442012-12-11 09:38:33 +10302578 return vmalloc_exec(size);
Jonas Bonn74e08fc2011-06-30 21:22:11 +02002579}
2580
Catalin Marinas4f2294b2009-06-11 13:23:20 +01002581#ifdef CONFIG_DEBUG_KMEMLEAK
Rusty Russell49668682010-08-05 12:59:10 -06002582static void kmemleak_load_module(const struct module *mod,
2583 const struct load_info *info)
Catalin Marinas4f2294b2009-06-11 13:23:20 +01002584{
2585 unsigned int i;
2586
2587 /* only scan the sections containing data */
Catalin Marinasc017b4b2009-10-28 13:33:09 +00002588 kmemleak_scan_area(mod, sizeof(struct module), GFP_KERNEL);
Catalin Marinas4f2294b2009-06-11 13:23:20 +01002589
Rusty Russell49668682010-08-05 12:59:10 -06002590 for (i = 1; i < info->hdr->e_shnum; i++) {
Steven Rostedt06c94942013-05-15 20:33:01 +01002591 /* Scan all writable sections that's not executable */
2592 if (!(info->sechdrs[i].sh_flags & SHF_ALLOC) ||
2593 !(info->sechdrs[i].sh_flags & SHF_WRITE) ||
2594 (info->sechdrs[i].sh_flags & SHF_EXECINSTR))
Catalin Marinas4f2294b2009-06-11 13:23:20 +01002595 continue;
2596
Rusty Russell49668682010-08-05 12:59:10 -06002597 kmemleak_scan_area((void *)info->sechdrs[i].sh_addr,
2598 info->sechdrs[i].sh_size, GFP_KERNEL);
Catalin Marinas4f2294b2009-06-11 13:23:20 +01002599 }
2600}
2601#else
Rusty Russell49668682010-08-05 12:59:10 -06002602static inline void kmemleak_load_module(const struct module *mod,
2603 const struct load_info *info)
Catalin Marinas4f2294b2009-06-11 13:23:20 +01002604{
2605}
2606#endif
2607
Rusty Russell106a4ee2012-09-26 10:09:40 +01002608#ifdef CONFIG_MODULE_SIG
Ben Hutchings95290bc2016-04-28 09:24:01 +09302609static int module_sig_check(struct load_info *info, int flags)
Rusty Russell106a4ee2012-09-26 10:09:40 +01002610{
2611 int err = -ENOKEY;
Kees Cook34e11692012-10-16 07:31:07 +10302612 const unsigned long markerlen = sizeof(MODULE_SIG_STRING) - 1;
2613 const void *mod = info->hdr;
Rusty Russell106a4ee2012-09-26 10:09:40 +01002614
Ben Hutchings95290bc2016-04-28 09:24:01 +09302615 /*
2616 * Require flags == 0, as a module with version information
2617 * removed is no longer the module that was signed
2618 */
2619 if (flags == 0 &&
2620 info->len > markerlen &&
Kees Cook34e11692012-10-16 07:31:07 +10302621 memcmp(mod + info->len - markerlen, MODULE_SIG_STRING, markerlen) == 0) {
David Howellscaabe242012-10-20 01:19:29 +01002622 /* We truncate the module to discard the signature */
Kees Cook34e11692012-10-16 07:31:07 +10302623 info->len -= markerlen;
2624 err = mod_verify_sig(mod, &info->len);
Rusty Russell106a4ee2012-09-26 10:09:40 +01002625 }
2626
2627 if (!err) {
2628 info->sig_ok = true;
2629 return 0;
2630 }
2631
2632 /* Not having a signature is only an error if we're strict. */
2633 if (err == -ENOKEY && !sig_enforce)
2634 err = 0;
2635
2636 return err;
2637}
2638#else /* !CONFIG_MODULE_SIG */
Ben Hutchings95290bc2016-04-28 09:24:01 +09302639static int module_sig_check(struct load_info *info, int flags)
Rusty Russell106a4ee2012-09-26 10:09:40 +01002640{
2641 return 0;
2642}
2643#endif /* !CONFIG_MODULE_SIG */
2644
Kees Cook34e11692012-10-16 07:31:07 +10302645/* Sanity checks against invalid binaries, wrong arch, weird elf version. */
2646static int elf_header_check(struct load_info *info)
Rusty Russell40dd2562010-08-05 12:59:03 -06002647{
Kees Cook34e11692012-10-16 07:31:07 +10302648 if (info->len < sizeof(*(info->hdr)))
Rusty Russell40dd2562010-08-05 12:59:03 -06002649 return -ENOEXEC;
2650
Kees Cook34e11692012-10-16 07:31:07 +10302651 if (memcmp(info->hdr->e_ident, ELFMAG, SELFMAG) != 0
2652 || info->hdr->e_type != ET_REL
2653 || !elf_check_arch(info->hdr)
2654 || info->hdr->e_shentsize != sizeof(Elf_Shdr))
2655 return -ENOEXEC;
2656
2657 if (info->hdr->e_shoff >= info->len
2658 || (info->hdr->e_shnum * sizeof(Elf_Shdr) >
2659 info->len - info->hdr->e_shoff))
2660 return -ENOEXEC;
2661
2662 return 0;
2663}
2664
Linus Torvalds3afe9f82015-04-07 10:33:49 -07002665#define COPY_CHUNK_SIZE (16*PAGE_SIZE)
2666
2667static int copy_chunked_from_user(void *dst, const void __user *usrc, unsigned long len)
2668{
2669 do {
2670 unsigned long n = min(len, COPY_CHUNK_SIZE);
2671
2672 if (copy_from_user(dst, usrc, n) != 0)
2673 return -EFAULT;
2674 cond_resched();
2675 dst += n;
2676 usrc += n;
2677 len -= n;
2678 } while (len);
2679 return 0;
2680}
2681
Kees Cook34e11692012-10-16 07:31:07 +10302682/* Sets info->hdr and info->len. */
2683static int copy_module_from_user(const void __user *umod, unsigned long len,
2684 struct load_info *info)
2685{
Kees Cook2e72d512012-10-16 07:32:07 +10302686 int err;
2687
Kees Cook34e11692012-10-16 07:31:07 +10302688 info->len = len;
2689 if (info->len < sizeof(*(info->hdr)))
Rusty Russell40dd2562010-08-05 12:59:03 -06002690 return -ENOEXEC;
2691
Kees Cook2e72d512012-10-16 07:32:07 +10302692 err = security_kernel_module_from_file(NULL);
2693 if (err)
2694 return err;
2695
Rusty Russell40dd2562010-08-05 12:59:03 -06002696 /* Suck in entire file: we'll want most of it. */
Kirill A. Shutemovcc9e6052015-03-24 12:31:40 +10302697 info->hdr = __vmalloc(info->len,
2698 GFP_KERNEL | __GFP_HIGHMEM | __GFP_NOWARN, PAGE_KERNEL);
Kees Cook34e11692012-10-16 07:31:07 +10302699 if (!info->hdr)
Rusty Russell40dd2562010-08-05 12:59:03 -06002700 return -ENOMEM;
2701
Linus Torvalds3afe9f82015-04-07 10:33:49 -07002702 if (copy_chunked_from_user(info->hdr, umod, info->len) != 0) {
Kees Cook34e11692012-10-16 07:31:07 +10302703 vfree(info->hdr);
2704 return -EFAULT;
Rusty Russell40dd2562010-08-05 12:59:03 -06002705 }
2706
Rusty Russell40dd2562010-08-05 12:59:03 -06002707 return 0;
Kees Cook34e11692012-10-16 07:31:07 +10302708}
Rusty Russell40dd2562010-08-05 12:59:03 -06002709
Kees Cook34e11692012-10-16 07:31:07 +10302710/* Sets info->hdr and info->len. */
2711static int copy_module_from_fd(int fd, struct load_info *info)
2712{
Al Viroa2e05782013-08-30 12:41:41 -04002713 struct fd f = fdget(fd);
Kees Cook34e11692012-10-16 07:31:07 +10302714 int err;
2715 struct kstat stat;
2716 loff_t pos;
2717 ssize_t bytes = 0;
2718
Al Viroa2e05782013-08-30 12:41:41 -04002719 if (!f.file)
Kees Cook34e11692012-10-16 07:31:07 +10302720 return -ENOEXEC;
2721
Al Viroa2e05782013-08-30 12:41:41 -04002722 err = security_kernel_module_from_file(f.file);
Kees Cook2e72d512012-10-16 07:32:07 +10302723 if (err)
2724 goto out;
2725
Al Viroa2e05782013-08-30 12:41:41 -04002726 err = vfs_getattr(&f.file->f_path, &stat);
Kees Cook34e11692012-10-16 07:31:07 +10302727 if (err)
2728 goto out;
2729
2730 if (stat.size > INT_MAX) {
2731 err = -EFBIG;
2732 goto out;
2733 }
Sasha Levin52441fa2013-01-03 11:09:53 +10302734
2735 /* Don't hand 0 to vmalloc, it whines. */
2736 if (stat.size == 0) {
2737 err = -EINVAL;
2738 goto out;
2739 }
2740
Kees Cook34e11692012-10-16 07:31:07 +10302741 info->hdr = vmalloc(stat.size);
2742 if (!info->hdr) {
2743 err = -ENOMEM;
2744 goto out;
2745 }
2746
2747 pos = 0;
2748 while (pos < stat.size) {
Al Viroa2e05782013-08-30 12:41:41 -04002749 bytes = kernel_read(f.file, pos, (char *)(info->hdr) + pos,
Kees Cook34e11692012-10-16 07:31:07 +10302750 stat.size - pos);
2751 if (bytes < 0) {
2752 vfree(info->hdr);
2753 err = bytes;
2754 goto out;
2755 }
2756 if (bytes == 0)
2757 break;
2758 pos += bytes;
2759 }
2760 info->len = pos;
2761
2762out:
Al Viroa2e05782013-08-30 12:41:41 -04002763 fdput(f);
Rusty Russell40dd2562010-08-05 12:59:03 -06002764 return err;
2765}
2766
Rusty Russelld9131882010-08-05 12:59:08 -06002767static void free_copy(struct load_info *info)
2768{
Rusty Russelld9131882010-08-05 12:59:08 -06002769 vfree(info->hdr);
2770}
2771
Rusty Russell2f3238a2012-10-22 18:09:41 +10302772static int rewrite_section_headers(struct load_info *info, int flags)
Rusty Russell8b5f61a2010-08-05 12:59:06 -06002773{
2774 unsigned int i;
2775
2776 /* This should always be true, but let's be sure. */
2777 info->sechdrs[0].sh_addr = 0;
2778
2779 for (i = 1; i < info->hdr->e_shnum; i++) {
2780 Elf_Shdr *shdr = &info->sechdrs[i];
2781 if (shdr->sh_type != SHT_NOBITS
2782 && info->len < shdr->sh_offset + shdr->sh_size) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -08002783 pr_err("Module len %lu truncated\n", info->len);
Rusty Russell8b5f61a2010-08-05 12:59:06 -06002784 return -ENOEXEC;
2785 }
2786
2787 /* Mark all sections sh_addr with their address in the
2788 temporary image. */
2789 shdr->sh_addr = (size_t)info->hdr + shdr->sh_offset;
2790
2791#ifndef CONFIG_MODULE_UNLOAD
2792 /* Don't load .exit sections */
2793 if (strstarts(info->secstrings+shdr->sh_name, ".exit"))
2794 shdr->sh_flags &= ~(unsigned long)SHF_ALLOC;
2795#endif
Rusty Russell8b5f61a2010-08-05 12:59:06 -06002796 }
Rusty Russelld6df72a2010-08-05 12:59:07 -06002797
2798 /* Track but don't keep modinfo and version sections. */
Rusty Russell2f3238a2012-10-22 18:09:41 +10302799 if (flags & MODULE_INIT_IGNORE_MODVERSIONS)
2800 info->index.vers = 0; /* Pretend no __versions section! */
2801 else
2802 info->index.vers = find_sec(info, "__versions");
Rusty Russell49668682010-08-05 12:59:10 -06002803 info->index.info = find_sec(info, ".modinfo");
Rusty Russelld6df72a2010-08-05 12:59:07 -06002804 info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC;
2805 info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC;
Rusty Russell8b5f61a2010-08-05 12:59:06 -06002806 return 0;
2807}
2808
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002809/*
2810 * Set up our basic convenience variables (pointers to section headers,
2811 * search for module section index etc), and do some basic section
2812 * verification.
2813 *
2814 * Return the temporary module pointer (we'll replace it with the final
2815 * one when we move the module sections around).
2816 */
Rusty Russell2f3238a2012-10-22 18:09:41 +10302817static struct module *setup_load_info(struct load_info *info, int flags)
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002818{
2819 unsigned int i;
Rusty Russell8b5f61a2010-08-05 12:59:06 -06002820 int err;
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002821 struct module *mod;
2822
2823 /* Set up the convenience variables */
2824 info->sechdrs = (void *)info->hdr + info->hdr->e_shoff;
Rusty Russell8b5f61a2010-08-05 12:59:06 -06002825 info->secstrings = (void *)info->hdr
2826 + info->sechdrs[info->hdr->e_shstrndx].sh_offset;
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002827
Rusty Russell2f3238a2012-10-22 18:09:41 +10302828 err = rewrite_section_headers(info, flags);
Rusty Russell8b5f61a2010-08-05 12:59:06 -06002829 if (err)
2830 return ERR_PTR(err);
2831
2832 /* Find internal symbols and strings. */
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002833 for (i = 1; i < info->hdr->e_shnum; i++) {
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002834 if (info->sechdrs[i].sh_type == SHT_SYMTAB) {
2835 info->index.sym = i;
2836 info->index.str = info->sechdrs[i].sh_link;
Rusty Russell8b5f61a2010-08-05 12:59:06 -06002837 info->strtab = (char *)info->hdr
2838 + info->sechdrs[info->index.str].sh_offset;
2839 break;
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002840 }
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002841 }
2842
Rusty Russell49668682010-08-05 12:59:10 -06002843 info->index.mod = find_sec(info, ".gnu.linkonce.this_module");
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002844 if (!info->index.mod) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -08002845 pr_warn("No module found in object\n");
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002846 return ERR_PTR(-ENOEXEC);
2847 }
2848 /* This is temporary: point mod into copy of data. */
2849 mod = (void *)info->sechdrs[info->index.mod].sh_addr;
2850
2851 if (info->index.sym == 0) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -08002852 pr_warn("%s: module has no symbols (stripped?)\n", mod->name);
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002853 return ERR_PTR(-ENOEXEC);
2854 }
2855
Rusty Russell49668682010-08-05 12:59:10 -06002856 info->index.pcpu = find_pcpusec(info);
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002857
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002858 /* Check module struct version now, before we try to use module. */
2859 if (!check_modstruct_version(info->sechdrs, info->index.vers, mod))
2860 return ERR_PTR(-ENOEXEC);
2861
2862 return mod;
Linus Torvalds3264d3f2010-06-02 11:01:06 -07002863}
2864
Rusty Russell2f3238a2012-10-22 18:09:41 +10302865static int check_modinfo(struct module *mod, struct load_info *info, int flags)
Rusty Russell40dd2562010-08-05 12:59:03 -06002866{
Rusty Russell49668682010-08-05 12:59:10 -06002867 const char *modmagic = get_modinfo(info, "vermagic");
Rusty Russell40dd2562010-08-05 12:59:03 -06002868 int err;
2869
Rusty Russell2f3238a2012-10-22 18:09:41 +10302870 if (flags & MODULE_INIT_IGNORE_VERMAGIC)
2871 modmagic = NULL;
2872
Rusty Russell40dd2562010-08-05 12:59:03 -06002873 /* This is allowed: modprobe --force will invalidate it. */
2874 if (!modmagic) {
2875 err = try_to_force_load(mod, "bad vermagic");
2876 if (err)
2877 return err;
Rusty Russell49668682010-08-05 12:59:10 -06002878 } else if (!same_magic(modmagic, vermagic, info->index.vers)) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -08002879 pr_err("%s: version magic '%s' should be '%s'\n",
Rusty Russell40dd2562010-08-05 12:59:03 -06002880 mod->name, modmagic, vermagic);
2881 return -ENOEXEC;
2882 }
2883
Ben Hutchings2449b8b2011-10-24 15:12:28 +02002884 if (!get_modinfo(info, "intree"))
Rusty Russell373d4d02013-01-21 17:17:39 +10302885 add_taint_module(mod, TAINT_OOT_MODULE, LOCKDEP_STILL_OK);
Ben Hutchings2449b8b2011-10-24 15:12:28 +02002886
Rusty Russell49668682010-08-05 12:59:10 -06002887 if (get_modinfo(info, "staging")) {
Rusty Russell373d4d02013-01-21 17:17:39 +10302888 add_taint_module(mod, TAINT_CRAP, LOCKDEP_STILL_OK);
Andrew Mortonbddb12b2013-11-12 15:11:28 -08002889 pr_warn("%s: module is from the staging directory, the quality "
2890 "is unknown, you have been warned.\n", mod->name);
Rusty Russell40dd2562010-08-05 12:59:03 -06002891 }
Rusty Russell22e268e2010-08-05 12:59:05 -06002892
2893 /* Set up license info based on the info section */
Rusty Russell49668682010-08-05 12:59:10 -06002894 set_license(mod, get_modinfo(info, "license"));
Rusty Russell22e268e2010-08-05 12:59:05 -06002895
Rusty Russell40dd2562010-08-05 12:59:03 -06002896 return 0;
2897}
2898
Frantisek Hrbataeb3057d2013-10-14 18:08:46 +10302899static int find_module_sections(struct module *mod, struct load_info *info)
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002900{
Rusty Russell49668682010-08-05 12:59:10 -06002901 mod->kp = section_objs(info, "__param",
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002902 sizeof(*mod->kp), &mod->num_kp);
Rusty Russell49668682010-08-05 12:59:10 -06002903 mod->syms = section_objs(info, "__ksymtab",
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002904 sizeof(*mod->syms), &mod->num_syms);
Rusty Russell49668682010-08-05 12:59:10 -06002905 mod->crcs = section_addr(info, "__kcrctab");
2906 mod->gpl_syms = section_objs(info, "__ksymtab_gpl",
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002907 sizeof(*mod->gpl_syms),
2908 &mod->num_gpl_syms);
Rusty Russell49668682010-08-05 12:59:10 -06002909 mod->gpl_crcs = section_addr(info, "__kcrctab_gpl");
2910 mod->gpl_future_syms = section_objs(info,
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002911 "__ksymtab_gpl_future",
2912 sizeof(*mod->gpl_future_syms),
2913 &mod->num_gpl_future_syms);
Rusty Russell49668682010-08-05 12:59:10 -06002914 mod->gpl_future_crcs = section_addr(info, "__kcrctab_gpl_future");
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002915
2916#ifdef CONFIG_UNUSED_SYMBOLS
Rusty Russell49668682010-08-05 12:59:10 -06002917 mod->unused_syms = section_objs(info, "__ksymtab_unused",
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002918 sizeof(*mod->unused_syms),
2919 &mod->num_unused_syms);
Rusty Russell49668682010-08-05 12:59:10 -06002920 mod->unused_crcs = section_addr(info, "__kcrctab_unused");
2921 mod->unused_gpl_syms = section_objs(info, "__ksymtab_unused_gpl",
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002922 sizeof(*mod->unused_gpl_syms),
2923 &mod->num_unused_gpl_syms);
Rusty Russell49668682010-08-05 12:59:10 -06002924 mod->unused_gpl_crcs = section_addr(info, "__kcrctab_unused_gpl");
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002925#endif
2926#ifdef CONFIG_CONSTRUCTORS
Rusty Russell49668682010-08-05 12:59:10 -06002927 mod->ctors = section_objs(info, ".ctors",
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002928 sizeof(*mod->ctors), &mod->num_ctors);
Frantisek Hrbataeb3057d2013-10-14 18:08:46 +10302929 if (!mod->ctors)
2930 mod->ctors = section_objs(info, ".init_array",
2931 sizeof(*mod->ctors), &mod->num_ctors);
2932 else if (find_sec(info, ".init_array")) {
2933 /*
2934 * This shouldn't happen with same compiler and binutils
2935 * building all parts of the module.
2936 */
Ionut Alexa6da0b562014-11-10 09:31:29 +10302937 pr_warn("%s: has both .ctors and .init_array.\n",
Frantisek Hrbataeb3057d2013-10-14 18:08:46 +10302938 mod->name);
2939 return -EINVAL;
2940 }
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002941#endif
2942
2943#ifdef CONFIG_TRACEPOINTS
Mathieu Desnoyers65498642011-01-26 17:26:22 -05002944 mod->tracepoints_ptrs = section_objs(info, "__tracepoints_ptrs",
2945 sizeof(*mod->tracepoints_ptrs),
2946 &mod->num_tracepoints);
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002947#endif
Jason Baronbf5438fc2010-09-17 11:09:00 -04002948#ifdef HAVE_JUMP_LABEL
2949 mod->jump_entries = section_objs(info, "__jump_table",
2950 sizeof(*mod->jump_entries),
2951 &mod->num_jump_entries);
2952#endif
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002953#ifdef CONFIG_EVENT_TRACING
Rusty Russell49668682010-08-05 12:59:10 -06002954 mod->trace_events = section_objs(info, "_ftrace_events",
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002955 sizeof(*mod->trace_events),
2956 &mod->num_trace_events);
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04002957 mod->trace_enums = section_objs(info, "_ftrace_enum_map",
2958 sizeof(*mod->trace_enums),
2959 &mod->num_trace_enums);
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002960#endif
Steven Rostedt13b9b6e2010-11-10 22:19:24 -05002961#ifdef CONFIG_TRACING
2962 mod->trace_bprintk_fmt_start = section_objs(info, "__trace_printk_fmt",
2963 sizeof(*mod->trace_bprintk_fmt_start),
2964 &mod->num_trace_bprintk_fmt);
Steven Rostedt13b9b6e2010-11-10 22:19:24 -05002965#endif
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002966#ifdef CONFIG_FTRACE_MCOUNT_RECORD
2967 /* sechdrs[0].sh_size is always zero */
Rusty Russell49668682010-08-05 12:59:10 -06002968 mod->ftrace_callsites = section_objs(info, "__mcount_loc",
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002969 sizeof(*mod->ftrace_callsites),
2970 &mod->num_ftrace_callsites);
2971#endif
Rusty Russell22e268e2010-08-05 12:59:05 -06002972
Rusty Russell811d66a2010-08-05 12:59:12 -06002973 mod->extable = section_objs(info, "__ex_table",
2974 sizeof(*mod->extable), &mod->num_exentries);
2975
Rusty Russell49668682010-08-05 12:59:10 -06002976 if (section_addr(info, "__obsparm"))
Andrew Mortonbddb12b2013-11-12 15:11:28 -08002977 pr_warn("%s: Ignoring obsolete parameters\n", mod->name);
Rusty Russell811d66a2010-08-05 12:59:12 -06002978
2979 info->debug = section_objs(info, "__verbose",
2980 sizeof(*info->debug), &info->num_debug);
Frantisek Hrbataeb3057d2013-10-14 18:08:46 +10302981
2982 return 0;
Linus Torvaldsf91a13b2010-08-05 12:59:02 -06002983}
2984
Rusty Russell49668682010-08-05 12:59:10 -06002985static int move_module(struct module *mod, struct load_info *info)
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06002986{
2987 int i;
2988 void *ptr;
2989
2990 /* Do the allocs. */
Peter Zijlstra4f6665462015-05-27 11:09:38 +09302991 ptr = module_alloc(mod->core_size);
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06002992 /*
2993 * The pointer to this block is stored in the module structure
2994 * which is inside the block. Just mark it as not being a
2995 * leak.
2996 */
2997 kmemleak_not_leak(ptr);
2998 if (!ptr)
Rusty Russelld9131882010-08-05 12:59:08 -06002999 return -ENOMEM;
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003000
3001 memset(ptr, 0, mod->core_size);
3002 mod->module_core = ptr;
3003
Rusty Russell82fab442012-12-11 09:38:33 +10303004 if (mod->init_size) {
Peter Zijlstra4f6665462015-05-27 11:09:38 +09303005 ptr = module_alloc(mod->init_size);
Rusty Russell82fab442012-12-11 09:38:33 +10303006 /*
3007 * The pointer to this block is stored in the module structure
3008 * which is inside the block. This block doesn't need to be
3009 * scanned as it contains data and code that will be freed
3010 * after the module is initialized.
3011 */
3012 kmemleak_ignore(ptr);
3013 if (!ptr) {
Rusty Russellbe1f2212015-01-20 09:07:05 +10303014 module_memfree(mod->module_core);
Rusty Russell82fab442012-12-11 09:38:33 +10303015 return -ENOMEM;
3016 }
3017 memset(ptr, 0, mod->init_size);
3018 mod->module_init = ptr;
3019 } else
3020 mod->module_init = NULL;
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003021
3022 /* Transfer each section which specifies SHF_ALLOC */
Jim Cromie5e124162011-12-06 12:11:31 -07003023 pr_debug("final section addresses:\n");
Rusty Russell49668682010-08-05 12:59:10 -06003024 for (i = 0; i < info->hdr->e_shnum; i++) {
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003025 void *dest;
Rusty Russell49668682010-08-05 12:59:10 -06003026 Elf_Shdr *shdr = &info->sechdrs[i];
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003027
Rusty Russell49668682010-08-05 12:59:10 -06003028 if (!(shdr->sh_flags & SHF_ALLOC))
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003029 continue;
3030
Rusty Russell49668682010-08-05 12:59:10 -06003031 if (shdr->sh_entsize & INIT_OFFSET_MASK)
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003032 dest = mod->module_init
Rusty Russell49668682010-08-05 12:59:10 -06003033 + (shdr->sh_entsize & ~INIT_OFFSET_MASK);
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003034 else
Rusty Russell49668682010-08-05 12:59:10 -06003035 dest = mod->module_core + shdr->sh_entsize;
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003036
Rusty Russell49668682010-08-05 12:59:10 -06003037 if (shdr->sh_type != SHT_NOBITS)
3038 memcpy(dest, (void *)shdr->sh_addr, shdr->sh_size);
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003039 /* Update sh_addr to point to copy in image. */
Rusty Russell49668682010-08-05 12:59:10 -06003040 shdr->sh_addr = (unsigned long)dest;
Jim Cromie5e124162011-12-06 12:11:31 -07003041 pr_debug("\t0x%lx %s\n",
3042 (long)shdr->sh_addr, info->secstrings + shdr->sh_name);
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003043 }
Rusty Russelld9131882010-08-05 12:59:08 -06003044
3045 return 0;
Linus Torvalds65b8a9b2010-08-05 12:59:02 -06003046}
3047
Rusty Russell49668682010-08-05 12:59:10 -06003048static int check_module_license_and_versions(struct module *mod)
Rusty Russell22e268e2010-08-05 12:59:05 -06003049{
3050 /*
3051 * ndiswrapper is under GPL by itself, but loads proprietary modules.
3052 * Don't use add_taint_module(), as it would prevent ndiswrapper from
3053 * using GPL-only symbols it needs.
3054 */
3055 if (strcmp(mod->name, "ndiswrapper") == 0)
Rusty Russell373d4d02013-01-21 17:17:39 +10303056 add_taint(TAINT_PROPRIETARY_MODULE, LOCKDEP_NOW_UNRELIABLE);
Rusty Russell22e268e2010-08-05 12:59:05 -06003057
3058 /* driverloader was caught wrongly pretending to be under GPL */
3059 if (strcmp(mod->name, "driverloader") == 0)
Rusty Russell373d4d02013-01-21 17:17:39 +10303060 add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
3061 LOCKDEP_NOW_UNRELIABLE);
Rusty Russell22e268e2010-08-05 12:59:05 -06003062
Matthew Garrettc99af372012-06-22 13:49:31 -04003063 /* lve claims to be GPL but upstream won't provide source */
3064 if (strcmp(mod->name, "lve") == 0)
Rusty Russell373d4d02013-01-21 17:17:39 +10303065 add_taint_module(mod, TAINT_PROPRIETARY_MODULE,
3066 LOCKDEP_NOW_UNRELIABLE);
Matthew Garrettc99af372012-06-22 13:49:31 -04003067
Rusty Russell22e268e2010-08-05 12:59:05 -06003068#ifdef CONFIG_MODVERSIONS
3069 if ((mod->num_syms && !mod->crcs)
3070 || (mod->num_gpl_syms && !mod->gpl_crcs)
3071 || (mod->num_gpl_future_syms && !mod->gpl_future_crcs)
3072#ifdef CONFIG_UNUSED_SYMBOLS
3073 || (mod->num_unused_syms && !mod->unused_crcs)
3074 || (mod->num_unused_gpl_syms && !mod->unused_gpl_crcs)
3075#endif
3076 ) {
3077 return try_to_force_load(mod,
3078 "no versions for exported symbols");
3079 }
3080#endif
3081 return 0;
3082}
3083
3084static void flush_module_icache(const struct module *mod)
3085{
3086 mm_segment_t old_fs;
3087
3088 /* flush the icache in correct context */
3089 old_fs = get_fs();
3090 set_fs(KERNEL_DS);
3091
3092 /*
3093 * Flush the instruction cache, since we've played with text.
3094 * Do it before processing of module parameters, so the module
3095 * can provide parameter accessor functions of its own.
3096 */
3097 if (mod->module_init)
3098 flush_icache_range((unsigned long)mod->module_init,
3099 (unsigned long)mod->module_init
3100 + mod->init_size);
3101 flush_icache_range((unsigned long)mod->module_core,
3102 (unsigned long)mod->module_core + mod->core_size);
3103
3104 set_fs(old_fs);
3105}
3106
Jonas Bonn74e08fc2011-06-30 21:22:11 +02003107int __weak module_frob_arch_sections(Elf_Ehdr *hdr,
3108 Elf_Shdr *sechdrs,
3109 char *secstrings,
3110 struct module *mod)
3111{
3112 return 0;
3113}
3114
Rusty Russell2f3238a2012-10-22 18:09:41 +10303115static struct module *layout_and_allocate(struct load_info *info, int flags)
Rusty Russelld9131882010-08-05 12:59:08 -06003116{
3117 /* Module within temporary copy. */
3118 struct module *mod;
3119 int err;
3120
Rusty Russell2f3238a2012-10-22 18:09:41 +10303121 mod = setup_load_info(info, flags);
Rusty Russelld9131882010-08-05 12:59:08 -06003122 if (IS_ERR(mod))
3123 return mod;
3124
Rusty Russell2f3238a2012-10-22 18:09:41 +10303125 err = check_modinfo(mod, info, flags);
Rusty Russelld9131882010-08-05 12:59:08 -06003126 if (err)
3127 return ERR_PTR(err);
3128
3129 /* Allow arches to frob section contents and sizes. */
Rusty Russell49668682010-08-05 12:59:10 -06003130 err = module_frob_arch_sections(info->hdr, info->sechdrs,
3131 info->secstrings, mod);
Rusty Russelld9131882010-08-05 12:59:08 -06003132 if (err < 0)
Rusty Russell8d8022e2013-07-03 10:06:28 +09303133 return ERR_PTR(err);
Rusty Russelld9131882010-08-05 12:59:08 -06003134
Rusty Russell8d8022e2013-07-03 10:06:28 +09303135 /* We will do a special allocation for per-cpu sections later. */
3136 info->sechdrs[info->index.pcpu].sh_flags &= ~(unsigned long)SHF_ALLOC;
Rusty Russelld9131882010-08-05 12:59:08 -06003137
3138 /* Determine total sizes, and put offsets in sh_entsize. For now
3139 this is done generically; there doesn't appear to be any
3140 special cases for the architectures. */
Rusty Russell49668682010-08-05 12:59:10 -06003141 layout_sections(mod, info);
Rusty Russell49668682010-08-05 12:59:10 -06003142 layout_symtab(mod, info);
Rusty Russelld9131882010-08-05 12:59:08 -06003143
3144 /* Allocate and move to the final place */
Rusty Russell49668682010-08-05 12:59:10 -06003145 err = move_module(mod, info);
Rusty Russelld9131882010-08-05 12:59:08 -06003146 if (err)
Rusty Russell8d8022e2013-07-03 10:06:28 +09303147 return ERR_PTR(err);
Rusty Russelld9131882010-08-05 12:59:08 -06003148
3149 /* Module has been copied to its final place now: return it. */
3150 mod = (void *)info->sechdrs[info->index.mod].sh_addr;
Rusty Russell49668682010-08-05 12:59:10 -06003151 kmemleak_load_module(mod, info);
Rusty Russelld9131882010-08-05 12:59:08 -06003152 return mod;
Rusty Russelld9131882010-08-05 12:59:08 -06003153}
3154
3155/* mod is no longer valid after this! */
3156static void module_deallocate(struct module *mod, struct load_info *info)
3157{
Rusty Russelld9131882010-08-05 12:59:08 -06003158 percpu_modfree(mod);
Rusty Russelld453cde2015-01-20 09:07:04 +10303159 module_arch_freeing_init(mod);
Rusty Russellbe1f2212015-01-20 09:07:05 +10303160 module_memfree(mod->module_init);
3161 module_memfree(mod->module_core);
Rusty Russelld9131882010-08-05 12:59:08 -06003162}
3163
Jonas Bonn74e08fc2011-06-30 21:22:11 +02003164int __weak module_finalize(const Elf_Ehdr *hdr,
3165 const Elf_Shdr *sechdrs,
3166 struct module *me)
3167{
3168 return 0;
3169}
3170
Rusty Russell811d66a2010-08-05 12:59:12 -06003171static int post_relocation(struct module *mod, const struct load_info *info)
3172{
Rusty Russell51f3d0f2010-08-05 12:59:13 -06003173 /* Sort exception table now relocations are done. */
Rusty Russell811d66a2010-08-05 12:59:12 -06003174 sort_extable(mod->extable, mod->extable + mod->num_exentries);
3175
3176 /* Copy relocated percpu area over. */
3177 percpu_modcopy(mod, (void *)info->sechdrs[info->index.pcpu].sh_addr,
3178 info->sechdrs[info->index.pcpu].sh_size);
3179
Rusty Russell51f3d0f2010-08-05 12:59:13 -06003180 /* Setup kallsyms-specific fields. */
Rusty Russell811d66a2010-08-05 12:59:12 -06003181 add_kallsyms(mod, info);
3182
3183 /* Arch-specific module finalizing. */
3184 return module_finalize(info->hdr, info->sechdrs, mod);
3185}
3186
Rusty Russell9bb9c3b2012-09-28 14:31:03 +09303187/* Is this module of this name done loading? No locks held. */
3188static bool finished_loading(const char *name)
3189{
3190 struct module *mod;
3191 bool ret;
3192
Peter Zijlstra9cc019b2015-02-11 15:01:13 +10303193 /*
3194 * The module_mutex should not be a heavily contended lock;
3195 * if we get the occasional sleep here, we'll go an extra iteration
3196 * in the wait_event_interruptible(), which is harmless.
3197 */
3198 sched_annotate_sleep();
Rusty Russell9bb9c3b2012-09-28 14:31:03 +09303199 mutex_lock(&module_mutex);
Mathias Krause4f6de4d2013-07-02 15:35:11 +09303200 mod = find_module_all(name, strlen(name), true);
Rusty Russell0d21b0e2013-01-12 11:38:44 +10303201 ret = !mod || mod->state == MODULE_STATE_LIVE
3202 || mod->state == MODULE_STATE_GOING;
Rusty Russell9bb9c3b2012-09-28 14:31:03 +09303203 mutex_unlock(&module_mutex);
3204
3205 return ret;
3206}
3207
Peter Oberparleiterb99b87f2009-06-17 16:28:03 -07003208/* Call module constructors. */
3209static void do_mod_ctors(struct module *mod)
3210{
3211#ifdef CONFIG_CONSTRUCTORS
3212 unsigned long i;
3213
3214 for (i = 0; i < mod->num_ctors; i++)
3215 mod->ctors[i]();
3216#endif
3217}
3218
Rusty Russellc7496372015-01-20 09:07:05 +10303219/* For freeing module_init on success, in case kallsyms traversing */
3220struct mod_initfree {
3221 struct rcu_head rcu;
3222 void *module_init;
3223};
3224
3225static void do_free_init(struct rcu_head *head)
3226{
3227 struct mod_initfree *m = container_of(head, struct mod_initfree, rcu);
3228 module_memfree(m->module_init);
3229 kfree(m);
3230}
3231
Jan Kiszkabe02a182015-02-17 13:46:50 -08003232/*
3233 * This is where the real work happens.
3234 *
3235 * Keep it uninlined to provide a reliable breakpoint target, e.g. for the gdb
3236 * helper command 'lx-symbols'.
3237 */
3238static noinline int do_init_module(struct module *mod)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003239{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003240 int ret = 0;
Rusty Russellc7496372015-01-20 09:07:05 +10303241 struct mod_initfree *freeinit;
3242
3243 freeinit = kmalloc(sizeof(*freeinit), GFP_KERNEL);
3244 if (!freeinit) {
3245 ret = -ENOMEM;
3246 goto fail;
3247 }
3248 freeinit->module_init = mod->module_init;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003249
Tejun Heo774a1222013-01-15 18:52:51 -08003250 /*
3251 * We want to find out whether @mod uses async during init. Clear
3252 * PF_USED_ASYNC. async_schedule*() will set it.
3253 */
3254 current->flags &= ~PF_USED_ASYNC;
3255
Peter Oberparleiterb99b87f2009-06-17 16:28:03 -07003256 do_mod_ctors(mod);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003257 /* Start the module */
3258 if (mod->init != NULL)
Arjan van de Ven59f94152008-07-30 12:49:02 -07003259 ret = do_one_initcall(mod->init);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003260 if (ret < 0) {
Rusty Russellc7496372015-01-20 09:07:05 +10303261 goto fail_free_freeinit;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003262 }
Alexey Dobriyane24e2e62008-03-10 11:43:53 -07003263 if (ret > 0) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -08003264 pr_warn("%s: '%s'->init suspiciously returned %d, it should "
3265 "follow 0/-E convention\n"
3266 "%s: loading module anyway...\n",
3267 __func__, mod->name, ret, __func__);
Alexey Dobriyane24e2e62008-03-10 11:43:53 -07003268 dump_stack();
3269 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003270
Rusty Russell6f139092012-09-28 14:31:03 +09303271 /* Now it's a first class citizen! */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003272 mod->state = MODULE_STATE_LIVE;
Masami Hiramatsu0deddf42009-01-06 14:41:54 -08003273 blocking_notifier_call_chain(&module_notify_list,
3274 MODULE_STATE_LIVE, mod);
Rusty Russell6c5db222008-03-10 11:43:52 -07003275
Tejun Heo774a1222013-01-15 18:52:51 -08003276 /*
3277 * We need to finish all async code before the module init sequence
3278 * is done. This has potential to deadlock. For example, a newly
3279 * detected block device can trigger request_module() of the
3280 * default iosched from async probing task. Once userland helper
3281 * reaches here, async_synchronize_full() will wait on the async
3282 * task waiting on request_module() and deadlock.
3283 *
3284 * This deadlock is avoided by perfomring async_synchronize_full()
3285 * iff module init queued any async jobs. This isn't a full
3286 * solution as it will deadlock the same if module loading from
3287 * async jobs nests more than once; however, due to the various
3288 * constraints, this hack seems to be the best option for now.
3289 * Please refer to the following thread for details.
3290 *
3291 * http://thread.gmane.org/gmane.linux.kernel/1420814
3292 */
Luis R. Rodriguezf2411da2015-03-30 16:20:05 -07003293 if (!mod->async_probe_requested && (current->flags & PF_USED_ASYNC))
Tejun Heo774a1222013-01-15 18:52:51 -08003294 async_synchronize_full();
Linus Torvaldsd6de2c82009-04-10 12:17:41 -07003295
Rusty Russell6c5db222008-03-10 11:43:52 -07003296 mutex_lock(&module_mutex);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003297 /* Drop initial reference. */
3298 module_put(mod);
Rusty Russellad6561d2009-06-12 21:47:03 -06003299 trim_init_extable(mod);
Jan Beulich4a496222009-07-06 14:50:42 +01003300#ifdef CONFIG_KALLSYMS
Rusty Russell610dde52016-02-03 16:55:26 +10303301 /* Switch to core kallsyms now init is done: kallsyms may be walking! */
3302 rcu_assign_pointer(mod->kallsyms, &mod->core_kallsyms);
Jan Beulich4a496222009-07-06 14:50:42 +01003303#endif
Peter Zijlstra93c2e102015-05-27 11:09:37 +09303304 mod_tree_remove_init(mod);
Jan Glauber01526ed2011-05-19 16:55:26 -06003305 unset_module_init_ro_nx(mod);
Rusty Russelld453cde2015-01-20 09:07:04 +10303306 module_arch_freeing_init(mod);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003307 mod->module_init = NULL;
3308 mod->init_size = 0;
Jan Glauber4d103802011-05-19 16:55:25 -06003309 mod->init_ro_size = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003310 mod->init_text_size = 0;
Rusty Russellc7496372015-01-20 09:07:05 +10303311 /*
3312 * We want to free module_init, but be aware that kallsyms may be
Peter Zijlstra0be964b2015-05-27 11:09:35 +09303313 * walking this with preempt disabled. In all the failure paths, we
3314 * call synchronize_sched(), but we don't want to slow down the success
3315 * path, so use actual RCU here.
Rusty Russellc7496372015-01-20 09:07:05 +10303316 */
Peter Zijlstra0be964b2015-05-27 11:09:35 +09303317 call_rcu_sched(&freeinit->rcu, do_free_init);
Ashutosh Naik6389a382006-03-23 03:00:46 -08003318 mutex_unlock(&module_mutex);
Rusty Russell6f139092012-09-28 14:31:03 +09303319 wake_up_all(&module_wq);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003320
3321 return 0;
Rusty Russellc7496372015-01-20 09:07:05 +10303322
3323fail_free_freeinit:
3324 kfree(freeinit);
3325fail:
3326 /* Try to protect us from buggy refcounters. */
3327 mod->state = MODULE_STATE_GOING;
3328 synchronize_sched();
3329 module_put(mod);
3330 blocking_notifier_call_chain(&module_notify_list,
3331 MODULE_STATE_GOING, mod);
3332 free_module(mod);
3333 wake_up_all(&module_wq);
3334 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003335}
3336
Kees Cook34e11692012-10-16 07:31:07 +10303337static int may_init_module(void)
3338{
3339 if (!capable(CAP_SYS_MODULE) || modules_disabled)
3340 return -EPERM;
3341
3342 return 0;
3343}
3344
Rusty Russella3535c72013-01-21 17:18:59 +10303345/*
3346 * We try to place it in the list now to make sure it's unique before
3347 * we dedicate too many resources. In particular, temporary percpu
3348 * memory exhaustion.
3349 */
3350static int add_unformed_module(struct module *mod)
3351{
3352 int err;
3353 struct module *old;
3354
3355 mod->state = MODULE_STATE_UNFORMED;
3356
3357again:
3358 mutex_lock(&module_mutex);
Mathias Krause4f6de4d2013-07-02 15:35:11 +09303359 old = find_module_all(mod->name, strlen(mod->name), true);
3360 if (old != NULL) {
Rusty Russella3535c72013-01-21 17:18:59 +10303361 if (old->state == MODULE_STATE_COMING
3362 || old->state == MODULE_STATE_UNFORMED) {
3363 /* Wait in case it fails to load. */
3364 mutex_unlock(&module_mutex);
Peter Zijlstra9cc019b2015-02-11 15:01:13 +10303365 err = wait_event_interruptible(module_wq,
3366 finished_loading(mod->name));
Rusty Russella3535c72013-01-21 17:18:59 +10303367 if (err)
3368 goto out_unlocked;
3369 goto again;
3370 }
3371 err = -EEXIST;
3372 goto out;
3373 }
Peter Zijlstra4f6665462015-05-27 11:09:38 +09303374 mod_update_bounds(mod);
Rusty Russella3535c72013-01-21 17:18:59 +10303375 list_add_rcu(&mod->list, &modules);
Peter Zijlstra93c2e102015-05-27 11:09:37 +09303376 mod_tree_insert(mod);
Rusty Russella3535c72013-01-21 17:18:59 +10303377 err = 0;
3378
3379out:
3380 mutex_unlock(&module_mutex);
3381out_unlocked:
3382 return err;
3383}
3384
3385static int complete_formation(struct module *mod, struct load_info *info)
3386{
3387 int err;
3388
3389 mutex_lock(&module_mutex);
3390
3391 /* Find duplicate symbols (must be called under lock). */
3392 err = verify_export_symbols(mod);
3393 if (err < 0)
3394 goto out;
3395
3396 /* This relies on module_mutex for list integrity. */
3397 module_bug_finalize(info->hdr, info->sechdrs, mod);
3398
Rusty Russell49822232014-05-14 10:54:19 +09303399 /* Set RO and NX regions for core */
3400 set_section_ro_nx(mod->module_core,
3401 mod->core_text_size,
3402 mod->core_ro_size,
3403 mod->core_size);
3404
3405 /* Set RO and NX regions for init */
3406 set_section_ro_nx(mod->module_init,
3407 mod->init_text_size,
3408 mod->init_ro_size,
3409 mod->init_size);
3410
Rusty Russella3535c72013-01-21 17:18:59 +10303411 /* Mark state as coming so strong_try_module_get() ignores us,
3412 * but kallsyms etc. can see us. */
3413 mod->state = MODULE_STATE_COMING;
Rusty Russell49822232014-05-14 10:54:19 +09303414 mutex_unlock(&module_mutex);
3415
3416 blocking_notifier_call_chain(&module_notify_list,
3417 MODULE_STATE_COMING, mod);
3418 return 0;
Rusty Russella3535c72013-01-21 17:18:59 +10303419
3420out:
3421 mutex_unlock(&module_mutex);
3422 return err;
3423}
3424
Luis R. Rodriguezecc86172015-03-30 16:20:03 -07003425static int unknown_module_param_cb(char *param, char *val, const char *modname,
3426 void *arg)
Rusty Russell54041d82013-07-02 15:35:12 +09303427{
Luis R. Rodriguezf2411da2015-03-30 16:20:05 -07003428 struct module *mod = arg;
3429 int ret;
3430
3431 if (strcmp(param, "async_probe") == 0) {
3432 mod->async_probe_requested = true;
3433 return 0;
3434 }
3435
Ionut Alexa6da0b562014-11-10 09:31:29 +10303436 /* Check for magic 'dyndbg' arg */
Luis R. Rodriguezf2411da2015-03-30 16:20:05 -07003437 ret = ddebug_dyndbg_module_param_cb(param, val, modname);
Andrew Mortonbddb12b2013-11-12 15:11:28 -08003438 if (ret != 0)
3439 pr_warn("%s: unknown parameter '%s' ignored\n", modname, param);
Rusty Russell54041d82013-07-02 15:35:12 +09303440 return 0;
3441}
3442
Kees Cook34e11692012-10-16 07:31:07 +10303443/* Allocate and load the module: note that size of section 0 is always
3444 zero, and we rely on this for optional sections. */
Rusty Russell2f3238a2012-10-22 18:09:41 +10303445static int load_module(struct load_info *info, const char __user *uargs,
3446 int flags)
Kees Cook34e11692012-10-16 07:31:07 +10303447{
Rusty Russella3535c72013-01-21 17:18:59 +10303448 struct module *mod;
Kees Cook34e11692012-10-16 07:31:07 +10303449 long err;
Rusty Russell51e158c2014-04-28 11:34:33 +09303450 char *after_dashes;
Kees Cook34e11692012-10-16 07:31:07 +10303451
Ben Hutchings95290bc2016-04-28 09:24:01 +09303452 err = module_sig_check(info, flags);
Kees Cook34e11692012-10-16 07:31:07 +10303453 if (err)
3454 goto free_copy;
3455
3456 err = elf_header_check(info);
3457 if (err)
3458 goto free_copy;
3459
3460 /* Figure out module layout, and allocate all the memory. */
Rusty Russell2f3238a2012-10-22 18:09:41 +10303461 mod = layout_and_allocate(info, flags);
Kees Cook34e11692012-10-16 07:31:07 +10303462 if (IS_ERR(mod)) {
3463 err = PTR_ERR(mod);
3464 goto free_copy;
3465 }
3466
Rusty Russella3535c72013-01-21 17:18:59 +10303467 /* Reserve our place in the list. */
3468 err = add_unformed_module(mod);
3469 if (err)
Rusty Russell1fb93412013-01-12 13:27:34 +10303470 goto free_module;
Rusty Russell1fb93412013-01-12 13:27:34 +10303471
Kees Cook34e11692012-10-16 07:31:07 +10303472#ifdef CONFIG_MODULE_SIG
3473 mod->sig_ok = info->sig_ok;
Rusty Russell64748a22013-01-21 17:03:02 +10303474 if (!mod->sig_ok) {
Andrew Mortonbddb12b2013-11-12 15:11:28 -08003475 pr_notice_once("%s: module verification failed: signature "
Marcel Holtmannab92ebb2015-02-06 15:09:57 +10303476 "and/or required key missing - tainting "
Andrew Mortonbddb12b2013-11-12 15:11:28 -08003477 "kernel\n", mod->name);
Mathieu Desnoyers66cc69e2014-03-13 12:11:30 +10303478 add_taint_module(mod, TAINT_UNSIGNED_MODULE, LOCKDEP_STILL_OK);
Rusty Russell64748a22013-01-21 17:03:02 +10303479 }
Kees Cook34e11692012-10-16 07:31:07 +10303480#endif
3481
Rusty Russell8d8022e2013-07-03 10:06:28 +09303482 /* To avoid stressing percpu allocator, do this once we're unique. */
Rusty Russell9eb76d72013-07-03 10:06:29 +09303483 err = percpu_modalloc(mod, info);
Rusty Russell8d8022e2013-07-03 10:06:28 +09303484 if (err)
3485 goto unlink_mod;
3486
Kees Cook34e11692012-10-16 07:31:07 +10303487 /* Now module is in final location, initialize linked lists, etc. */
3488 err = module_unload_init(mod);
3489 if (err)
Rusty Russell1fb93412013-01-12 13:27:34 +10303490 goto unlink_mod;
Kees Cook34e11692012-10-16 07:31:07 +10303491
Rusty Russellcf2fde72015-06-26 06:44:38 +09303492 init_param_lock(mod);
Dan Streetmanb51d23e2015-06-17 06:18:52 +09303493
Kees Cook34e11692012-10-16 07:31:07 +10303494 /* Now we've got everything in the final locations, we can
3495 * find optional sections. */
Frantisek Hrbataeb3057d2013-10-14 18:08:46 +10303496 err = find_module_sections(mod, info);
3497 if (err)
3498 goto free_unload;
Kees Cook34e11692012-10-16 07:31:07 +10303499
3500 err = check_module_license_and_versions(mod);
3501 if (err)
3502 goto free_unload;
3503
3504 /* Set up MODINFO_ATTR fields */
3505 setup_modinfo(mod, info);
3506
3507 /* Fix up syms, so that st_value is a pointer to location. */
3508 err = simplify_symbols(mod, info);
3509 if (err < 0)
3510 goto free_modinfo;
3511
3512 err = apply_relocations(mod, info);
3513 if (err < 0)
3514 goto free_modinfo;
3515
3516 err = post_relocation(mod, info);
3517 if (err < 0)
3518 goto free_modinfo;
3519
3520 flush_module_icache(mod);
3521
3522 /* Now copy in args */
3523 mod->args = strndup_user(uargs, ~0UL >> 1);
3524 if (IS_ERR(mod->args)) {
3525 err = PTR_ERR(mod->args);
3526 goto free_arch_cleanup;
3527 }
3528
Kees Cook34e11692012-10-16 07:31:07 +10303529 dynamic_debug_setup(info->debug, info->num_debug);
3530
Steven Rostedt (Red Hat)a949ae52014-04-24 10:40:12 -04003531 /* Ftrace init must be called in the MODULE_STATE_UNFORMED state */
3532 ftrace_module_init(mod);
3533
Rusty Russella3535c72013-01-21 17:18:59 +10303534 /* Finally it's fully formed, ready to start executing. */
3535 err = complete_formation(mod, info);
3536 if (err)
Rusty Russell1fb93412013-01-12 13:27:34 +10303537 goto ddebug_cleanup;
Kees Cook34e11692012-10-16 07:31:07 +10303538
Kees Cook34e11692012-10-16 07:31:07 +10303539 /* Module is ready to execute: parsing args may do that. */
Rusty Russell51e158c2014-04-28 11:34:33 +09303540 after_dashes = parse_args(mod->name, mod->args, mod->kp, mod->num_kp,
Luis R. Rodrigueze2f712d2016-02-03 16:55:26 +10303541 -32768, 32767, mod,
Luis R. Rodriguezecc86172015-03-30 16:20:03 -07003542 unknown_module_param_cb);
Rusty Russell51e158c2014-04-28 11:34:33 +09303543 if (IS_ERR(after_dashes)) {
3544 err = PTR_ERR(after_dashes);
Rusty Russell1fb93412013-01-12 13:27:34 +10303545 goto bug_cleanup;
Rusty Russell51e158c2014-04-28 11:34:33 +09303546 } else if (after_dashes) {
3547 pr_warn("%s: parameters '%s' after `--' ignored\n",
3548 mod->name, after_dashes);
3549 }
Kees Cook34e11692012-10-16 07:31:07 +10303550
3551 /* Link in to syfs. */
3552 err = mod_sysfs_setup(mod, info, mod->kp, mod->num_kp);
3553 if (err < 0)
Rusty Russell1fb93412013-01-12 13:27:34 +10303554 goto bug_cleanup;
Kees Cook34e11692012-10-16 07:31:07 +10303555
3556 /* Get rid of temporary copy. */
3557 free_copy(info);
3558
3559 /* Done! */
3560 trace_module_load(mod);
3561
3562 return do_init_module(mod);
3563
Rusty Russell1fb93412013-01-12 13:27:34 +10303564 bug_cleanup:
3565 /* module_bug_cleanup needs module_mutex protection */
Kees Cook34e11692012-10-16 07:31:07 +10303566 mutex_lock(&module_mutex);
Kees Cook34e11692012-10-16 07:31:07 +10303567 module_bug_cleanup(mod);
Linus Torvaldsee61abb2013-01-20 20:22:58 -08003568 mutex_unlock(&module_mutex);
Andy Lutomirskiff7e0052014-08-16 04:13:37 +09303569
Steven Rostedt37815bf2015-05-09 03:06:23 +09303570 blocking_notifier_call_chain(&module_notify_list,
3571 MODULE_STATE_GOING, mod);
3572
Andy Lutomirskiff7e0052014-08-16 04:13:37 +09303573 /* we can't deallocate the module until we clear memory protection */
3574 unset_module_init_ro_nx(mod);
3575 unset_module_core_ro_nx(mod);
3576
Rusty Russella3535c72013-01-21 17:18:59 +10303577 ddebug_cleanup:
Rusty Russell1fb93412013-01-12 13:27:34 +10303578 dynamic_debug_remove(info->debug);
Kees Cook34e11692012-10-16 07:31:07 +10303579 synchronize_sched();
3580 kfree(mod->args);
3581 free_arch_cleanup:
3582 module_arch_cleanup(mod);
3583 free_modinfo:
3584 free_modinfo(mod);
3585 free_unload:
3586 module_unload_free(mod);
Rusty Russell1fb93412013-01-12 13:27:34 +10303587 unlink_mod:
3588 mutex_lock(&module_mutex);
3589 /* Unlink carefully: kallsyms could be walking list. */
3590 list_del_rcu(&mod->list);
Peter Zijlstra758556b2015-07-09 06:48:06 +09303591 mod_tree_remove(mod);
Rusty Russell1fb93412013-01-12 13:27:34 +10303592 wake_up_all(&module_wq);
Peter Zijlstra0be964b2015-05-27 11:09:35 +09303593 /* Wait for RCU-sched synchronizing before releasing mod->list. */
3594 synchronize_sched();
Rusty Russell1fb93412013-01-12 13:27:34 +10303595 mutex_unlock(&module_mutex);
Kees Cook34e11692012-10-16 07:31:07 +10303596 free_module:
Steven Rostedt (Red Hat)049fb9b2016-01-05 20:32:47 -05003597 /*
3598 * Ftrace needs to clean up what it initialized.
3599 * This does nothing if ftrace_module_init() wasn't called,
3600 * but it must be called outside of module_mutex.
3601 */
3602 ftrace_release_mod(mod);
Peter Zijlstra35a93932015-02-26 16:23:11 +01003603 /* Free lock-classes; relies on the preceding sync_rcu() */
3604 lockdep_free_key_range(mod->module_core, mod->core_size);
3605
Kees Cook34e11692012-10-16 07:31:07 +10303606 module_deallocate(mod, info);
3607 free_copy:
3608 free_copy(info);
3609 return err;
3610}
3611
3612SYSCALL_DEFINE3(init_module, void __user *, umod,
3613 unsigned long, len, const char __user *, uargs)
3614{
3615 int err;
3616 struct load_info info = { };
3617
3618 err = may_init_module();
3619 if (err)
3620 return err;
3621
3622 pr_debug("init_module: umod=%p, len=%lu, uargs=%p\n",
3623 umod, len, uargs);
3624
3625 err = copy_module_from_user(umod, len, &info);
3626 if (err)
3627 return err;
3628
Rusty Russell2f3238a2012-10-22 18:09:41 +10303629 return load_module(&info, uargs, 0);
Kees Cook34e11692012-10-16 07:31:07 +10303630}
3631
Rusty Russell2f3238a2012-10-22 18:09:41 +10303632SYSCALL_DEFINE3(finit_module, int, fd, const char __user *, uargs, int, flags)
Kees Cook34e11692012-10-16 07:31:07 +10303633{
3634 int err;
3635 struct load_info info = { };
3636
3637 err = may_init_module();
3638 if (err)
3639 return err;
3640
Rusty Russell2f3238a2012-10-22 18:09:41 +10303641 pr_debug("finit_module: fd=%d, uargs=%p, flags=%i\n", fd, uargs, flags);
3642
3643 if (flags & ~(MODULE_INIT_IGNORE_MODVERSIONS
3644 |MODULE_INIT_IGNORE_VERMAGIC))
3645 return -EINVAL;
Kees Cook34e11692012-10-16 07:31:07 +10303646
3647 err = copy_module_from_fd(fd, &info);
3648 if (err)
3649 return err;
3650
Rusty Russell2f3238a2012-10-22 18:09:41 +10303651 return load_module(&info, uargs, flags);
Kees Cook34e11692012-10-16 07:31:07 +10303652}
3653
Linus Torvalds1da177e2005-04-16 15:20:36 -07003654static inline int within(unsigned long addr, void *start, unsigned long size)
3655{
3656 return ((void *)addr >= start && (void *)addr < start + size);
3657}
3658
3659#ifdef CONFIG_KALLSYMS
3660/*
3661 * This ignores the intensely annoying "mapping symbols" found
3662 * in ARM ELF files: $a, $t and $d.
3663 */
3664static inline int is_arm_mapping_symbol(const char *str)
3665{
Russell King2e3a10a2014-07-27 07:29:01 +09303666 if (str[0] == '.' && str[1] == 'L')
3667 return true;
Kyle McMartin6c34f1f2014-09-16 22:37:18 +01003668 return str[0] == '$' && strchr("axtd", str[1])
Linus Torvalds1da177e2005-04-16 15:20:36 -07003669 && (str[2] == '\0' || str[2] == '.');
3670}
3671
Rusty Russell610dde52016-02-03 16:55:26 +10303672static const char *symname(struct mod_kallsyms *kallsyms, unsigned int symnum)
Rusty Russella24d9a22016-02-03 16:55:26 +10303673{
Rusty Russell610dde52016-02-03 16:55:26 +10303674 return kallsyms->strtab + kallsyms->symtab[symnum].st_name;
Rusty Russella24d9a22016-02-03 16:55:26 +10303675}
3676
Linus Torvalds1da177e2005-04-16 15:20:36 -07003677static const char *get_ksymbol(struct module *mod,
3678 unsigned long addr,
3679 unsigned long *size,
3680 unsigned long *offset)
3681{
3682 unsigned int i, best = 0;
3683 unsigned long nextval;
Rusty Russell610dde52016-02-03 16:55:26 +10303684 struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003685
3686 /* At worse, next value is at end of module */
Masami Hiramatsua06f6212009-01-06 14:41:49 -08003687 if (within_module_init(addr, mod))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003688 nextval = (unsigned long)mod->module_init+mod->init_text_size;
Daniel Walker22a8bde2007-10-18 03:06:07 -07003689 else
Linus Torvalds1da177e2005-04-16 15:20:36 -07003690 nextval = (unsigned long)mod->module_core+mod->core_text_size;
3691
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003692 /* Scan for closest preceding symbol, and next symbol. (ELF
Daniel Walker22a8bde2007-10-18 03:06:07 -07003693 starts real symbols at 1). */
Rusty Russell610dde52016-02-03 16:55:26 +10303694 for (i = 1; i < kallsyms->num_symtab; i++) {
3695 if (kallsyms->symtab[i].st_shndx == SHN_UNDEF)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003696 continue;
3697
3698 /* We ignore unnamed symbols: they're uninformative
3699 * and inserted at a whim. */
Rusty Russell610dde52016-02-03 16:55:26 +10303700 if (*symname(kallsyms, i) == '\0'
3701 || is_arm_mapping_symbol(symname(kallsyms, i)))
Rusty Russella24d9a22016-02-03 16:55:26 +10303702 continue;
3703
Rusty Russell610dde52016-02-03 16:55:26 +10303704 if (kallsyms->symtab[i].st_value <= addr
3705 && kallsyms->symtab[i].st_value > kallsyms->symtab[best].st_value)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003706 best = i;
Rusty Russell610dde52016-02-03 16:55:26 +10303707 if (kallsyms->symtab[i].st_value > addr
3708 && kallsyms->symtab[i].st_value < nextval)
3709 nextval = kallsyms->symtab[i].st_value;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003710 }
3711
3712 if (!best)
3713 return NULL;
3714
Alexey Dobriyanffb45122007-05-08 00:28:41 -07003715 if (size)
Rusty Russell610dde52016-02-03 16:55:26 +10303716 *size = nextval - kallsyms->symtab[best].st_value;
Alexey Dobriyanffb45122007-05-08 00:28:41 -07003717 if (offset)
Rusty Russell610dde52016-02-03 16:55:26 +10303718 *offset = addr - kallsyms->symtab[best].st_value;
3719 return symname(kallsyms, best);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003720}
3721
Rusty Russell6dd06c92008-01-29 17:13:22 -05003722/* For kallsyms to ask for address resolution. NULL means not found. Careful
3723 * not to lock to avoid deadlock on oopses, simply disable preemption. */
Andrew Morton92dfc9d2008-02-08 04:18:43 -08003724const char *module_address_lookup(unsigned long addr,
Rusty Russell6dd06c92008-01-29 17:13:22 -05003725 unsigned long *size,
3726 unsigned long *offset,
3727 char **modname,
3728 char *namebuf)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003729{
Rusty Russellcb2a5202008-01-14 00:55:03 -08003730 const char *ret = NULL;
Peter Zijlstrab7df4d12015-05-27 11:09:37 +09303731 struct module *mod;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003732
Rusty Russellcb2a5202008-01-14 00:55:03 -08003733 preempt_disable();
Peter Zijlstrab7df4d12015-05-27 11:09:37 +09303734 mod = __module_address(addr);
3735 if (mod) {
3736 if (modname)
3737 *modname = mod->name;
3738 ret = get_ksymbol(mod, addr, size, offset);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003739 }
Rusty Russell6dd06c92008-01-29 17:13:22 -05003740 /* Make a copy in here where it's safe */
3741 if (ret) {
3742 strncpy(namebuf, ret, KSYM_NAME_LEN - 1);
3743 ret = namebuf;
3744 }
Rusty Russellcb2a5202008-01-14 00:55:03 -08003745 preempt_enable();
Peter Zijlstrab7df4d12015-05-27 11:09:37 +09303746
Andrew Morton92dfc9d2008-02-08 04:18:43 -08003747 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003748}
3749
Alexey Dobriyan9d65cb42007-05-08 00:28:43 -07003750int lookup_module_symbol_name(unsigned long addr, char *symname)
3751{
3752 struct module *mod;
3753
Rusty Russellcb2a5202008-01-14 00:55:03 -08003754 preempt_disable();
Andi Kleend72b3752008-08-30 10:09:00 +02003755 list_for_each_entry_rcu(mod, &modules, list) {
Rusty Russell0d21b0e2013-01-12 11:38:44 +10303756 if (mod->state == MODULE_STATE_UNFORMED)
3757 continue;
Petr Mladek9b20a352014-07-27 07:24:01 +09303758 if (within_module(addr, mod)) {
Alexey Dobriyan9d65cb42007-05-08 00:28:43 -07003759 const char *sym;
3760
3761 sym = get_ksymbol(mod, addr, NULL, NULL);
3762 if (!sym)
3763 goto out;
Tejun Heo9281ace2007-07-17 04:03:51 -07003764 strlcpy(symname, sym, KSYM_NAME_LEN);
Rusty Russellcb2a5202008-01-14 00:55:03 -08003765 preempt_enable();
Alexey Dobriyan9d65cb42007-05-08 00:28:43 -07003766 return 0;
3767 }
3768 }
3769out:
Rusty Russellcb2a5202008-01-14 00:55:03 -08003770 preempt_enable();
Alexey Dobriyan9d65cb42007-05-08 00:28:43 -07003771 return -ERANGE;
3772}
3773
Alexey Dobriyana5c43da2007-05-08 00:28:47 -07003774int lookup_module_symbol_attrs(unsigned long addr, unsigned long *size,
3775 unsigned long *offset, char *modname, char *name)
3776{
3777 struct module *mod;
3778
Rusty Russellcb2a5202008-01-14 00:55:03 -08003779 preempt_disable();
Andi Kleend72b3752008-08-30 10:09:00 +02003780 list_for_each_entry_rcu(mod, &modules, list) {
Rusty Russell0d21b0e2013-01-12 11:38:44 +10303781 if (mod->state == MODULE_STATE_UNFORMED)
3782 continue;
Petr Mladek9b20a352014-07-27 07:24:01 +09303783 if (within_module(addr, mod)) {
Alexey Dobriyana5c43da2007-05-08 00:28:47 -07003784 const char *sym;
3785
3786 sym = get_ksymbol(mod, addr, size, offset);
3787 if (!sym)
3788 goto out;
3789 if (modname)
Tejun Heo9281ace2007-07-17 04:03:51 -07003790 strlcpy(modname, mod->name, MODULE_NAME_LEN);
Alexey Dobriyana5c43da2007-05-08 00:28:47 -07003791 if (name)
Tejun Heo9281ace2007-07-17 04:03:51 -07003792 strlcpy(name, sym, KSYM_NAME_LEN);
Rusty Russellcb2a5202008-01-14 00:55:03 -08003793 preempt_enable();
Alexey Dobriyana5c43da2007-05-08 00:28:47 -07003794 return 0;
3795 }
3796 }
3797out:
Rusty Russellcb2a5202008-01-14 00:55:03 -08003798 preempt_enable();
Alexey Dobriyana5c43da2007-05-08 00:28:47 -07003799 return -ERANGE;
3800}
3801
Alexey Dobriyanea078902007-05-08 00:28:39 -07003802int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
3803 char *name, char *module_name, int *exported)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003804{
3805 struct module *mod;
3806
Rusty Russellcb2a5202008-01-14 00:55:03 -08003807 preempt_disable();
Andi Kleend72b3752008-08-30 10:09:00 +02003808 list_for_each_entry_rcu(mod, &modules, list) {
Rusty Russell610dde52016-02-03 16:55:26 +10303809 struct mod_kallsyms *kallsyms;
3810
Rusty Russell0d21b0e2013-01-12 11:38:44 +10303811 if (mod->state == MODULE_STATE_UNFORMED)
3812 continue;
Rusty Russell610dde52016-02-03 16:55:26 +10303813 kallsyms = rcu_dereference_sched(mod->kallsyms);
3814 if (symnum < kallsyms->num_symtab) {
3815 *value = kallsyms->symtab[symnum].st_value;
3816 *type = kallsyms->symtab[symnum].st_info;
3817 strlcpy(name, symname(kallsyms, symnum), KSYM_NAME_LEN);
Tejun Heo9281ace2007-07-17 04:03:51 -07003818 strlcpy(module_name, mod->name, MODULE_NAME_LEN);
Tim Abbottca4787b2009-01-05 08:40:10 -06003819 *exported = is_exported(name, *value, mod);
Rusty Russellcb2a5202008-01-14 00:55:03 -08003820 preempt_enable();
Alexey Dobriyanea078902007-05-08 00:28:39 -07003821 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003822 }
Rusty Russell610dde52016-02-03 16:55:26 +10303823 symnum -= kallsyms->num_symtab;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003824 }
Rusty Russellcb2a5202008-01-14 00:55:03 -08003825 preempt_enable();
Alexey Dobriyanea078902007-05-08 00:28:39 -07003826 return -ERANGE;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003827}
3828
3829static unsigned long mod_find_symname(struct module *mod, const char *name)
3830{
3831 unsigned int i;
Rusty Russell610dde52016-02-03 16:55:26 +10303832 struct mod_kallsyms *kallsyms = rcu_dereference_sched(mod->kallsyms);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003833
Rusty Russell610dde52016-02-03 16:55:26 +10303834 for (i = 0; i < kallsyms->num_symtab; i++)
3835 if (strcmp(name, symname(kallsyms, i)) == 0 &&
3836 kallsyms->symtab[i].st_info != 'U')
3837 return kallsyms->symtab[i].st_value;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003838 return 0;
3839}
3840
3841/* Look for this name: can be of form module:name. */
3842unsigned long module_kallsyms_lookup_name(const char *name)
3843{
3844 struct module *mod;
3845 char *colon;
3846 unsigned long ret = 0;
3847
3848 /* Don't lock: we're in enough trouble already. */
Rusty Russellcb2a5202008-01-14 00:55:03 -08003849 preempt_disable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003850 if ((colon = strchr(name, ':')) != NULL) {
Mathias Krause4f6de4d2013-07-02 15:35:11 +09303851 if ((mod = find_module_all(name, colon - name, false)) != NULL)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003852 ret = mod_find_symname(mod, colon+1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003853 } else {
Rusty Russell0d21b0e2013-01-12 11:38:44 +10303854 list_for_each_entry_rcu(mod, &modules, list) {
3855 if (mod->state == MODULE_STATE_UNFORMED)
3856 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003857 if ((ret = mod_find_symname(mod, name)) != 0)
3858 break;
Rusty Russell0d21b0e2013-01-12 11:38:44 +10303859 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003860 }
Rusty Russellcb2a5202008-01-14 00:55:03 -08003861 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003862 return ret;
3863}
Anders Kaseorg75a66612008-12-05 19:03:58 -05003864
3865int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *,
3866 struct module *, unsigned long),
3867 void *data)
3868{
3869 struct module *mod;
3870 unsigned int i;
3871 int ret;
3872
Peter Zijlstra0be964b2015-05-27 11:09:35 +09303873 module_assert_mutex();
3874
Anders Kaseorg75a66612008-12-05 19:03:58 -05003875 list_for_each_entry(mod, &modules, list) {
Rusty Russell610dde52016-02-03 16:55:26 +10303876 /* We hold module_mutex: no need for rcu_dereference_sched */
3877 struct mod_kallsyms *kallsyms = mod->kallsyms;
3878
Rusty Russell0d21b0e2013-01-12 11:38:44 +10303879 if (mod->state == MODULE_STATE_UNFORMED)
3880 continue;
Rusty Russell610dde52016-02-03 16:55:26 +10303881 for (i = 0; i < kallsyms->num_symtab; i++) {
3882 ret = fn(data, symname(kallsyms, i),
3883 mod, kallsyms->symtab[i].st_value);
Anders Kaseorg75a66612008-12-05 19:03:58 -05003884 if (ret != 0)
3885 return ret;
3886 }
3887 }
3888 return 0;
3889}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003890#endif /* CONFIG_KALLSYMS */
3891
Arjan van de Ven21aa9282008-01-25 21:08:33 +01003892static char *module_flags(struct module *mod, char *buf)
Florin Malitafa3ba2e82006-10-11 01:21:48 -07003893{
3894 int bx = 0;
3895
Rusty Russell0d21b0e2013-01-12 11:38:44 +10303896 BUG_ON(mod->state == MODULE_STATE_UNFORMED);
Arjan van de Ven21aa9282008-01-25 21:08:33 +01003897 if (mod->taints ||
3898 mod->state == MODULE_STATE_GOING ||
3899 mod->state == MODULE_STATE_COMING) {
Florin Malitafa3ba2e82006-10-11 01:21:48 -07003900 buf[bx++] = '(';
Kay Sieverscca3e702012-01-13 09:32:15 +10303901 bx += module_flags_taint(mod, buf + bx);
Arjan van de Ven21aa9282008-01-25 21:08:33 +01003902 /* Show a - for module-is-being-unloaded */
3903 if (mod->state == MODULE_STATE_GOING)
3904 buf[bx++] = '-';
3905 /* Show a + for module-is-being-loaded */
3906 if (mod->state == MODULE_STATE_COMING)
3907 buf[bx++] = '+';
Florin Malitafa3ba2e82006-10-11 01:21:48 -07003908 buf[bx++] = ')';
3909 }
3910 buf[bx] = '\0';
3911
3912 return buf;
3913}
3914
Alexey Dobriyan3b5d5c62008-10-06 13:19:27 +04003915#ifdef CONFIG_PROC_FS
3916/* Called by the /proc file system to return a list of modules. */
3917static void *m_start(struct seq_file *m, loff_t *pos)
3918{
3919 mutex_lock(&module_mutex);
3920 return seq_list_start(&modules, *pos);
3921}
3922
3923static void *m_next(struct seq_file *m, void *p, loff_t *pos)
3924{
3925 return seq_list_next(p, &modules, pos);
3926}
3927
3928static void m_stop(struct seq_file *m, void *p)
3929{
3930 mutex_unlock(&module_mutex);
3931}
3932
Linus Torvalds1da177e2005-04-16 15:20:36 -07003933static int m_show(struct seq_file *m, void *p)
3934{
3935 struct module *mod = list_entry(p, struct module, list);
Florin Malitafa3ba2e82006-10-11 01:21:48 -07003936 char buf[8];
3937
Rusty Russell0d21b0e2013-01-12 11:38:44 +10303938 /* We always ignore unformed modules. */
3939 if (mod->state == MODULE_STATE_UNFORMED)
3940 return 0;
3941
Denys Vlasenko2f0f2a32008-07-22 19:24:27 -05003942 seq_printf(m, "%s %u",
Linus Torvalds1da177e2005-04-16 15:20:36 -07003943 mod->name, mod->init_size + mod->core_size);
3944 print_unload_info(m, mod);
3945
3946 /* Informative for users. */
3947 seq_printf(m, " %s",
Ionut Alexa6da0b562014-11-10 09:31:29 +10303948 mod->state == MODULE_STATE_GOING ? "Unloading" :
3949 mod->state == MODULE_STATE_COMING ? "Loading" :
Linus Torvalds1da177e2005-04-16 15:20:36 -07003950 "Live");
3951 /* Used by oprofile and other similar tools. */
Kees Cook9f36e2c2011-03-22 16:34:22 -07003952 seq_printf(m, " 0x%pK", mod->module_core);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003953
Florin Malitafa3ba2e82006-10-11 01:21:48 -07003954 /* Taints info */
3955 if (mod->taints)
Arjan van de Ven21aa9282008-01-25 21:08:33 +01003956 seq_printf(m, " %s", module_flags(mod, buf));
Florin Malitafa3ba2e82006-10-11 01:21:48 -07003957
Ionut Alexa6da0b562014-11-10 09:31:29 +10303958 seq_puts(m, "\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07003959 return 0;
3960}
3961
3962/* Format: modulename size refcount deps address
3963
3964 Where refcount is a number or -, and deps is a comma-separated list
3965 of depends or -.
3966*/
Alexey Dobriyan3b5d5c62008-10-06 13:19:27 +04003967static const struct seq_operations modules_op = {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003968 .start = m_start,
3969 .next = m_next,
3970 .stop = m_stop,
3971 .show = m_show
3972};
3973
Alexey Dobriyan3b5d5c62008-10-06 13:19:27 +04003974static int modules_open(struct inode *inode, struct file *file)
3975{
3976 return seq_open(file, &modules_op);
3977}
3978
3979static const struct file_operations proc_modules_operations = {
3980 .open = modules_open,
3981 .read = seq_read,
3982 .llseek = seq_lseek,
3983 .release = seq_release,
3984};
3985
3986static int __init proc_modules_init(void)
3987{
3988 proc_create("modules", 0, NULL, &proc_modules_operations);
3989 return 0;
3990}
3991module_init(proc_modules_init);
3992#endif
3993
Linus Torvalds1da177e2005-04-16 15:20:36 -07003994/* Given an address, look for it in the module exception tables. */
3995const struct exception_table_entry *search_module_extables(unsigned long addr)
3996{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003997 const struct exception_table_entry *e = NULL;
3998 struct module *mod;
3999
Rusty Russell24da1cb2007-07-15 23:41:46 -07004000 preempt_disable();
Andi Kleend72b3752008-08-30 10:09:00 +02004001 list_for_each_entry_rcu(mod, &modules, list) {
Rusty Russell0d21b0e2013-01-12 11:38:44 +10304002 if (mod->state == MODULE_STATE_UNFORMED)
4003 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004004 if (mod->num_exentries == 0)
4005 continue;
Daniel Walker22a8bde2007-10-18 03:06:07 -07004006
Linus Torvalds1da177e2005-04-16 15:20:36 -07004007 e = search_extable(mod->extable,
4008 mod->extable + mod->num_exentries - 1,
4009 addr);
4010 if (e)
4011 break;
4012 }
Rusty Russell24da1cb2007-07-15 23:41:46 -07004013 preempt_enable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07004014
4015 /* Now, if we found one, we are running inside it now, hence
Daniel Walker22a8bde2007-10-18 03:06:07 -07004016 we cannot unload the module, hence no refcnt needed. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004017 return e;
4018}
4019
Ingo Molnar4d435f92006-07-03 00:24:24 -07004020/*
Rusty Russelle6104992009-03-31 13:05:31 -06004021 * is_module_address - is this address inside a module?
4022 * @addr: the address to check.
4023 *
4024 * See is_module_text_address() if you simply want to see if the address
4025 * is code (not data).
Ingo Molnar4d435f92006-07-03 00:24:24 -07004026 */
Rusty Russelle6104992009-03-31 13:05:31 -06004027bool is_module_address(unsigned long addr)
Ingo Molnar4d435f92006-07-03 00:24:24 -07004028{
Rusty Russelle6104992009-03-31 13:05:31 -06004029 bool ret;
Ingo Molnar4d435f92006-07-03 00:24:24 -07004030
Rusty Russell24da1cb2007-07-15 23:41:46 -07004031 preempt_disable();
Rusty Russelle6104992009-03-31 13:05:31 -06004032 ret = __module_address(addr) != NULL;
Rusty Russell24da1cb2007-07-15 23:41:46 -07004033 preempt_enable();
Ingo Molnar4d435f92006-07-03 00:24:24 -07004034
Rusty Russelle6104992009-03-31 13:05:31 -06004035 return ret;
Ingo Molnar4d435f92006-07-03 00:24:24 -07004036}
4037
Rusty Russelle6104992009-03-31 13:05:31 -06004038/*
4039 * __module_address - get the module which contains an address.
4040 * @addr: the address.
4041 *
4042 * Must be called with preempt disabled or module mutex held so that
4043 * module doesn't get freed during this.
4044 */
Linus Torvalds714f83d2009-04-05 11:04:19 -07004045struct module *__module_address(unsigned long addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004046{
4047 struct module *mod;
4048
Rusty Russell3a642e92008-07-22 19:24:28 -05004049 if (addr < module_addr_min || addr > module_addr_max)
4050 return NULL;
4051
Peter Zijlstra0be964b2015-05-27 11:09:35 +09304052 module_assert_mutex_or_preempt();
4053
Peter Zijlstra6c9692e2015-05-27 11:09:37 +09304054 mod = mod_find(addr);
Peter Zijlstra93c2e102015-05-27 11:09:37 +09304055 if (mod) {
4056 BUG_ON(!within_module(addr, mod));
Rusty Russell0d21b0e2013-01-12 11:38:44 +10304057 if (mod->state == MODULE_STATE_UNFORMED)
Peter Zijlstra93c2e102015-05-27 11:09:37 +09304058 mod = NULL;
Rusty Russell0d21b0e2013-01-12 11:38:44 +10304059 }
Peter Zijlstra93c2e102015-05-27 11:09:37 +09304060 return mod;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004061}
Tim Abbottc6b37802008-12-05 19:03:59 -05004062EXPORT_SYMBOL_GPL(__module_address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004063
Rusty Russelle6104992009-03-31 13:05:31 -06004064/*
4065 * is_module_text_address - is this address inside module code?
4066 * @addr: the address to check.
4067 *
4068 * See is_module_address() if you simply want to see if the address is
4069 * anywhere in a module. See kernel_text_address() for testing if an
4070 * address corresponds to kernel or module code.
4071 */
4072bool is_module_text_address(unsigned long addr)
4073{
4074 bool ret;
4075
4076 preempt_disable();
4077 ret = __module_text_address(addr) != NULL;
4078 preempt_enable();
4079
4080 return ret;
4081}
4082
4083/*
4084 * __module_text_address - get the module whose code contains an address.
4085 * @addr: the address.
4086 *
4087 * Must be called with preempt disabled or module mutex held so that
4088 * module doesn't get freed during this.
4089 */
4090struct module *__module_text_address(unsigned long addr)
4091{
4092 struct module *mod = __module_address(addr);
4093 if (mod) {
4094 /* Make sure it's within the text section. */
4095 if (!within(addr, mod->module_init, mod->init_text_size)
4096 && !within(addr, mod->module_core, mod->core_text_size))
4097 mod = NULL;
4098 }
4099 return mod;
4100}
Tim Abbottc6b37802008-12-05 19:03:59 -05004101EXPORT_SYMBOL_GPL(__module_text_address);
Rusty Russelle6104992009-03-31 13:05:31 -06004102
Linus Torvalds1da177e2005-04-16 15:20:36 -07004103/* Don't grab lock, we're oopsing. */
4104void print_modules(void)
4105{
4106 struct module *mod;
Randy Dunlap2bc2d612006-10-02 02:17:02 -07004107 char buf[8];
Linus Torvalds1da177e2005-04-16 15:20:36 -07004108
Linus Torvaldsb2311252009-06-16 11:07:14 -07004109 printk(KERN_DEFAULT "Modules linked in:");
Andi Kleend72b3752008-08-30 10:09:00 +02004110 /* Most callers should already have preempt disabled, but make sure */
4111 preempt_disable();
Rusty Russell0d21b0e2013-01-12 11:38:44 +10304112 list_for_each_entry_rcu(mod, &modules, list) {
4113 if (mod->state == MODULE_STATE_UNFORMED)
4114 continue;
Jiri Slaby27bba4d2014-02-03 11:13:13 +10304115 pr_cont(" %s%s", mod->name, module_flags(mod, buf));
Rusty Russell0d21b0e2013-01-12 11:38:44 +10304116 }
Andi Kleend72b3752008-08-30 10:09:00 +02004117 preempt_enable();
Arjan van de Vene14af7e2008-01-25 21:08:33 +01004118 if (last_unloaded_module[0])
Jiri Slaby27bba4d2014-02-03 11:13:13 +10304119 pr_cont(" [last unloaded: %s]", last_unloaded_module);
4120 pr_cont("\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -07004121}
4122
Linus Torvalds1da177e2005-04-16 15:20:36 -07004123#ifdef CONFIG_MODVERSIONS
Rusty Russell8c8ef422009-03-31 13:05:34 -06004124/* Generate the signature for all relevant module structures here.
4125 * If these change, we don't want to try to parse the module. */
4126void module_layout(struct module *mod,
4127 struct modversion_info *ver,
4128 struct kernel_param *kp,
4129 struct kernel_symbol *ks,
Mathieu Desnoyers65498642011-01-26 17:26:22 -05004130 struct tracepoint * const *tp)
Rusty Russell8c8ef422009-03-31 13:05:34 -06004131{
4132}
4133EXPORT_SYMBOL(module_layout);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004134#endif