blob: 63c342e5e6c3481540d7f1e6b271d2e6fffbf1f2 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * Kernel Probes (KProbes)
3 * kernel/kprobes.c
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * Copyright (C) IBM Corporation, 2002, 2004
20 *
21 * 2002-Oct Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22 * Probes initial implementation (includes suggestions from
23 * Rusty Russell).
24 * 2004-Aug Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25 * hlists and exceptions notifier as suggested by Andi Kleen.
26 * 2004-July Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27 * interface to access function arguments.
28 * 2004-Sep Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29 * exceptions notifier to be first on the priority list.
Hien Nguyenb94cce92005-06-23 00:09:19 -070030 * 2005-May Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31 * <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32 * <prasanna@in.ibm.com> added function-return probes.
Linus Torvalds1da177e2005-04-16 15:20:36 -070033 */
34#include <linux/kprobes.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <linux/hash.h>
36#include <linux/init.h>
Tim Schmielau4e57b682005-10-30 15:03:48 -080037#include <linux/slab.h>
Randy Dunlape3869792007-05-08 00:27:01 -070038#include <linux/stddef.h>
Paul Gortmaker9984de12011-05-23 14:51:41 -040039#include <linux/export.h>
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070040#include <linux/moduleloader.h>
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -070041#include <linux/kallsyms.h>
Masami Hiramatsub4c6c342006-12-06 20:38:11 -080042#include <linux/freezer.h>
Srinivasa Ds346fd592007-02-20 13:57:54 -080043#include <linux/seq_file.h>
44#include <linux/debugfs.h>
Masami Hiramatsub2be84d2010-02-25 08:34:15 -050045#include <linux/sysctl.h>
Christoph Hellwig1eeb66a2007-05-08 00:27:03 -070046#include <linux/kdebug.h>
Mathieu Desnoyers4460fda2009-03-06 10:36:38 -050047#include <linux/memory.h>
Masami Hiramatsu4554dbc2010-02-02 16:49:18 -050048#include <linux/ftrace.h>
Masami Hiramatsuafd66252010-02-25 08:34:07 -050049#include <linux/cpu.h>
Jason Baronbf5438fc2010-09-17 11:09:00 -040050#include <linux/jump_label.h>
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070051
Christoph Hellwigbfd45be2016-10-11 13:52:22 -070052#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070053#include <asm/cacheflush.h>
54#include <asm/errno.h>
Linus Torvalds7c0f6ba2016-12-24 11:46:01 -080055#include <linux/uaccess.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070056
57#define KPROBE_HASH_BITS 6
58#define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
59
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -070060
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070061static int kprobes_initialized;
Linus Torvalds1da177e2005-04-16 15:20:36 -070062static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
Hien Nguyenb94cce92005-06-23 00:09:19 -070063static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
Linus Torvalds1da177e2005-04-16 15:20:36 -070064
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070065/* NOTE: change this value only with kprobe_mutex held */
Masami Hiramatsue579abe2009-04-06 19:01:01 -070066static bool kprobes_all_disarmed;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -070067
Masami Hiramatsu43948f52010-10-25 22:18:01 +090068/* This protects kprobe_table and optimizing_list */
69static DEFINE_MUTEX(kprobe_mutex);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -080070static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070071static struct {
Thomas Gleixnerec4846082009-07-25 16:09:17 +020072 raw_spinlock_t lock ____cacheline_aligned_in_smp;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070073} kretprobe_table_locks[KPROBE_TABLE_SIZE];
74
Naveen N. Rao290e3072017-04-19 18:21:01 +053075kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
76 unsigned int __unused)
Naveen N. Rao49e0b462017-04-19 18:21:00 +053077{
78 return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
79}
80
Thomas Gleixnerec4846082009-07-25 16:09:17 +020081static raw_spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -070082{
83 return &(kretprobe_table_locks[hash].lock);
84}
Linus Torvalds1da177e2005-04-16 15:20:36 -070085
Masami Hiramatsu376e2422014-04-17 17:17:05 +090086/* Blacklist -- list of struct kprobe_blacklist_entry */
87static LIST_HEAD(kprobe_blacklist);
Srinivasa Ds3d8d9962008-04-28 02:14:26 -070088
Anil S Keshavamurthy2d14e392006-01-09 20:52:41 -080089#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070090/*
91 * kprobe->ainsn.insn points to the copy of the instruction to be
92 * single-stepped. x86_64, POWER4 and above have no-exec support and
93 * stepping on the instruction on a vmalloced/kmalloced/data page
94 * is a recipe for disaster
95 */
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070096struct kprobe_insn_page {
Masami Hiramatsuc5cb5a22009-06-30 17:08:14 -040097 struct list_head list;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -070098 kprobe_opcode_t *insns; /* Page of instruction slots */
Heiko Carstensaf963972013-09-11 14:24:13 -070099 struct kprobe_insn_cache *cache;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700100 int nused;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800101 int ngarbage;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500102 char slot_used[];
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700103};
104
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500105#define KPROBE_INSN_PAGE_SIZE(slots) \
106 (offsetof(struct kprobe_insn_page, slot_used) + \
107 (sizeof(char) * (slots)))
108
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500109static int slots_per_page(struct kprobe_insn_cache *c)
110{
111 return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
112}
113
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800114enum kprobe_slot_state {
115 SLOT_CLEAN = 0,
116 SLOT_DIRTY = 1,
117 SLOT_USED = 2,
118};
119
Masami Hiramatsu63fef142017-08-18 17:24:00 +0900120void __weak *alloc_insn_page(void)
Heiko Carstensaf963972013-09-11 14:24:13 -0700121{
122 return module_alloc(PAGE_SIZE);
123}
124
Masami Hiramatsuc93f5cf2017-05-25 19:38:17 +0900125void __weak free_insn_page(void *page)
Heiko Carstensaf963972013-09-11 14:24:13 -0700126{
Rusty Russellbe1f2212015-01-20 09:07:05 +1030127 module_memfree(page);
Heiko Carstensaf963972013-09-11 14:24:13 -0700128}
129
Heiko Carstensc802d642013-09-11 14:24:11 -0700130struct kprobe_insn_cache kprobe_insn_slots = {
131 .mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
Heiko Carstensaf963972013-09-11 14:24:13 -0700132 .alloc = alloc_insn_page,
133 .free = free_insn_page,
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500134 .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
135 .insn_size = MAX_INSN_SIZE,
136 .nr_garbage = 0,
137};
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900138static int collect_garbage_slots(struct kprobe_insn_cache *c);
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800139
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700140/**
Masami Hiramatsu12941562009-01-06 14:41:50 -0800141 * __get_insn_slot() - Find a slot on an executable page for an instruction.
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700142 * We allocate an executable page if there's no room on existing ones.
143 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900144kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c)
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700145{
146 struct kprobe_insn_page *kip;
Heiko Carstensc802d642013-09-11 14:24:11 -0700147 kprobe_opcode_t *slot = NULL;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700148
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900149 /* Since the slot array is not protected by rcu, we need a mutex */
Heiko Carstensc802d642013-09-11 14:24:11 -0700150 mutex_lock(&c->mutex);
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700151 retry:
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900152 rcu_read_lock();
153 list_for_each_entry_rcu(kip, &c->pages, list) {
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500154 if (kip->nused < slots_per_page(c)) {
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700155 int i;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500156 for (i = 0; i < slots_per_page(c); i++) {
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800157 if (kip->slot_used[i] == SLOT_CLEAN) {
158 kip->slot_used[i] = SLOT_USED;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700159 kip->nused++;
Heiko Carstensc802d642013-09-11 14:24:11 -0700160 slot = kip->insns + (i * c->insn_size);
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900161 rcu_read_unlock();
Heiko Carstensc802d642013-09-11 14:24:11 -0700162 goto out;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700163 }
164 }
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500165 /* kip->nused is broken. Fix it. */
166 kip->nused = slots_per_page(c);
167 WARN_ON(1);
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700168 }
169 }
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900170 rcu_read_unlock();
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700171
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800172 /* If there are any garbage slots, collect it and try again. */
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500173 if (c->nr_garbage && collect_garbage_slots(c) == 0)
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800174 goto retry;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500175
176 /* All out of space. Need to allocate a new page. */
177 kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
Christoph Hellwig6f716ac2007-05-08 00:34:13 -0700178 if (!kip)
Heiko Carstensc802d642013-09-11 14:24:11 -0700179 goto out;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700180
181 /*
182 * Use module_alloc so this page is within +/- 2GB of where the
183 * kernel image and loaded module images reside. This is required
184 * so x86_64 can correctly handle the %rip-relative fixups.
185 */
Heiko Carstensaf963972013-09-11 14:24:13 -0700186 kip->insns = c->alloc();
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700187 if (!kip->insns) {
188 kfree(kip);
Heiko Carstensc802d642013-09-11 14:24:11 -0700189 goto out;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700190 }
Masami Hiramatsuc5cb5a22009-06-30 17:08:14 -0400191 INIT_LIST_HEAD(&kip->list);
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500192 memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800193 kip->slot_used[0] = SLOT_USED;
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700194 kip->nused = 1;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800195 kip->ngarbage = 0;
Heiko Carstensaf963972013-09-11 14:24:13 -0700196 kip->cache = c;
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900197 list_add_rcu(&kip->list, &c->pages);
Heiko Carstensc802d642013-09-11 14:24:11 -0700198 slot = kip->insns;
199out:
200 mutex_unlock(&c->mutex);
201 return slot;
Masami Hiramatsu12941562009-01-06 14:41:50 -0800202}
203
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800204/* Return 1 if all garbages are collected, otherwise 0. */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900205static int collect_one_slot(struct kprobe_insn_page *kip, int idx)
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800206{
Masami Hiramatsuab40c5c2007-01-30 14:36:06 -0800207 kip->slot_used[idx] = SLOT_CLEAN;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800208 kip->nused--;
209 if (kip->nused == 0) {
210 /*
211 * Page is no longer in use. Free it unless
212 * it's the last one. We keep the last one
213 * so as not to have to set it up again the
214 * next time somebody inserts a probe.
215 */
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500216 if (!list_is_singular(&kip->list)) {
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900217 list_del_rcu(&kip->list);
218 synchronize_rcu();
Heiko Carstensaf963972013-09-11 14:24:13 -0700219 kip->cache->free(kip->insns);
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800220 kfree(kip);
221 }
222 return 1;
223 }
224 return 0;
225}
226
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900227static int collect_garbage_slots(struct kprobe_insn_cache *c)
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800228{
Masami Hiramatsuc5cb5a22009-06-30 17:08:14 -0400229 struct kprobe_insn_page *kip, *next;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800230
Masami Hiramatsu615d0eb2010-02-02 16:49:04 -0500231 /* Ensure no-one is interrupted on the garbages */
232 synchronize_sched();
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800233
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500234 list_for_each_entry_safe(kip, next, &c->pages, list) {
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800235 int i;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800236 if (kip->ngarbage == 0)
237 continue;
238 kip->ngarbage = 0; /* we will collect all garbages */
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500239 for (i = 0; i < slots_per_page(c); i++) {
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900240 if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800241 break;
242 }
243 }
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500244 c->nr_garbage = 0;
Masami Hiramatsub4c6c342006-12-06 20:38:11 -0800245 return 0;
246}
247
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900248void __free_insn_slot(struct kprobe_insn_cache *c,
249 kprobe_opcode_t *slot, int dirty)
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500250{
251 struct kprobe_insn_page *kip;
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900252 long idx;
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500253
Heiko Carstensc802d642013-09-11 14:24:11 -0700254 mutex_lock(&c->mutex);
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900255 rcu_read_lock();
256 list_for_each_entry_rcu(kip, &c->pages, list) {
257 idx = ((long)slot - (long)kip->insns) /
258 (c->insn_size * sizeof(kprobe_opcode_t));
259 if (idx >= 0 && idx < slots_per_page(c))
Heiko Carstensc802d642013-09-11 14:24:11 -0700260 goto out;
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900261 }
262 /* Could not find this slot. */
263 WARN_ON(1);
264 kip = NULL;
265out:
266 rcu_read_unlock();
267 /* Mark and sweep: this may sleep */
268 if (kip) {
269 /* Check double free */
270 WARN_ON(kip->slot_used[idx] != SLOT_USED);
271 if (dirty) {
272 kip->slot_used[idx] = SLOT_DIRTY;
273 kip->ngarbage++;
274 if (++c->nr_garbage > slots_per_page(c))
275 collect_garbage_slots(c);
276 } else {
277 collect_one_slot(kip, idx);
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500278 }
279 }
Heiko Carstensc802d642013-09-11 14:24:11 -0700280 mutex_unlock(&c->mutex);
Masami Hiramatsu4610ee12010-02-25 08:33:59 -0500281}
282
Masami Hiramatsu5b485622017-01-08 23:58:09 +0900283/*
284 * Check given address is on the page of kprobe instruction slots.
285 * This will be used for checking whether the address on a stack
286 * is on a text area or not.
287 */
288bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
289{
290 struct kprobe_insn_page *kip;
291 bool ret = false;
292
293 rcu_read_lock();
294 list_for_each_entry_rcu(kip, &c->pages, list) {
295 if (addr >= (unsigned long)kip->insns &&
296 addr < (unsigned long)kip->insns + PAGE_SIZE) {
297 ret = true;
298 break;
299 }
300 }
301 rcu_read_unlock();
302
303 return ret;
304}
305
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500306#ifdef CONFIG_OPTPROBES
307/* For optimized_kprobe buffer */
Heiko Carstensc802d642013-09-11 14:24:11 -0700308struct kprobe_insn_cache kprobe_optinsn_slots = {
309 .mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
Heiko Carstensaf963972013-09-11 14:24:13 -0700310 .alloc = alloc_insn_page,
311 .free = free_insn_page,
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500312 .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
313 /* .insn_size is initialized later */
314 .nr_garbage = 0,
315};
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500316#endif
Anil S Keshavamurthy2d14e392006-01-09 20:52:41 -0800317#endif
Ananth N Mavinakayanahalli9ec4b1f2005-06-27 15:17:01 -0700318
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800319/* We have preemption disabled.. so it is safe to use __ versions */
320static inline void set_kprobe_instance(struct kprobe *kp)
321{
Christoph Lameterb76834b2010-12-06 11:16:25 -0600322 __this_cpu_write(kprobe_instance, kp);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800323}
324
325static inline void reset_kprobe_instance(void)
326{
Christoph Lameterb76834b2010-12-06 11:16:25 -0600327 __this_cpu_write(kprobe_instance, NULL);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -0800328}
329
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800330/*
331 * This routine is called either:
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -0800332 * - under the kprobe_mutex - during kprobe_[un]register()
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800333 * OR
Ananth N Mavinakayanahallid217d542005-11-07 01:00:14 -0800334 * - with preemption disabled - from arch/xxx/kernel/kprobes.c
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800335 */
Masami Hiramatsu820aede2014-04-17 17:18:21 +0900336struct kprobe *get_kprobe(void *addr)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337{
338 struct hlist_head *head;
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -0800339 struct kprobe *p;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340
341 head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
Sasha Levinb67bfe02013-02-27 17:06:00 -0800342 hlist_for_each_entry_rcu(p, head, hlist) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700343 if (p->addr == addr)
344 return p;
345 }
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500346
Linus Torvalds1da177e2005-04-16 15:20:36 -0700347 return NULL;
348}
Masami Hiramatsu820aede2014-04-17 17:18:21 +0900349NOKPROBE_SYMBOL(get_kprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350
Masami Hiramatsu820aede2014-04-17 17:18:21 +0900351static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500352
353/* Return true if the kprobe is an aggregator */
354static inline int kprobe_aggrprobe(struct kprobe *p)
355{
356 return p->pre_handler == aggr_pre_handler;
357}
358
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900359/* Return true(!0) if the kprobe is unused */
360static inline int kprobe_unused(struct kprobe *p)
361{
362 return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
363 list_empty(&p->list);
364}
365
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500366/*
367 * Keep all fields in the kprobe consistent
368 */
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900369static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500370{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900371 memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
372 memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500373}
374
375#ifdef CONFIG_OPTPROBES
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500376/* NOTE: change this value only with kprobe_mutex held */
377static bool kprobes_allow_optimization;
378
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500379/*
380 * Call all pre_handler on the list, but ignores its return value.
381 * This must be called from arch-dep optimized caller.
382 */
Masami Hiramatsu820aede2014-04-17 17:18:21 +0900383void opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500384{
385 struct kprobe *kp;
386
387 list_for_each_entry_rcu(kp, &p->list, list) {
388 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
389 set_kprobe_instance(kp);
Naveen N. Rao4f3a8712017-10-17 13:48:34 +0530390 kp->pre_handler(kp, regs);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500391 }
392 reset_kprobe_instance();
393 }
394}
Masami Hiramatsu820aede2014-04-17 17:18:21 +0900395NOKPROBE_SYMBOL(opt_pre_handler);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500396
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900397/* Free optimized instructions and optimized_kprobe */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900398static void free_aggr_kprobe(struct kprobe *p)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900399{
400 struct optimized_kprobe *op;
401
402 op = container_of(p, struct optimized_kprobe, kp);
403 arch_remove_optimized_kprobe(op);
404 arch_remove_kprobe(p);
405 kfree(op);
406}
407
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500408/* Return true(!0) if the kprobe is ready for optimization. */
409static inline int kprobe_optready(struct kprobe *p)
410{
411 struct optimized_kprobe *op;
412
413 if (kprobe_aggrprobe(p)) {
414 op = container_of(p, struct optimized_kprobe, kp);
415 return arch_prepared_optinsn(&op->optinsn);
416 }
417
418 return 0;
419}
420
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900421/* Return true(!0) if the kprobe is disarmed. Note: p must be on hash list */
422static inline int kprobe_disarmed(struct kprobe *p)
423{
424 struct optimized_kprobe *op;
425
426 /* If kprobe is not aggr/opt probe, just return kprobe is disabled */
427 if (!kprobe_aggrprobe(p))
428 return kprobe_disabled(p);
429
430 op = container_of(p, struct optimized_kprobe, kp);
431
432 return kprobe_disabled(p) && list_empty(&op->list);
433}
434
435/* Return true(!0) if the probe is queued on (un)optimizing lists */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900436static int kprobe_queued(struct kprobe *p)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900437{
438 struct optimized_kprobe *op;
439
440 if (kprobe_aggrprobe(p)) {
441 op = container_of(p, struct optimized_kprobe, kp);
442 if (!list_empty(&op->list))
443 return 1;
444 }
445 return 0;
446}
447
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500448/*
449 * Return an optimized kprobe whose optimizing code replaces
450 * instructions including addr (exclude breakpoint).
451 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900452static struct kprobe *get_optimized_kprobe(unsigned long addr)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500453{
454 int i;
455 struct kprobe *p = NULL;
456 struct optimized_kprobe *op;
457
458 /* Don't check i == 0, since that is a breakpoint case. */
459 for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
460 p = get_kprobe((void *)(addr - i));
461
462 if (p && kprobe_optready(p)) {
463 op = container_of(p, struct optimized_kprobe, kp);
464 if (arch_within_optimized_kprobe(op, addr))
465 return p;
466 }
467
468 return NULL;
469}
470
471/* Optimization staging list, protected by kprobe_mutex */
472static LIST_HEAD(optimizing_list);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900473static LIST_HEAD(unoptimizing_list);
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900474static LIST_HEAD(freeing_list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500475
476static void kprobe_optimizer(struct work_struct *work);
477static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
478#define OPTIMIZE_DELAY 5
479
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900480/*
481 * Optimize (replace a breakpoint with a jump) kprobes listed on
482 * optimizing_list.
483 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900484static void do_optimize_kprobes(void)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500485{
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500486 /*
487 * The optimization/unoptimization refers online_cpus via
488 * stop_machine() and cpu-hotplug modifies online_cpus.
489 * And same time, text_mutex will be held in cpu-hotplug and here.
490 * This combination can cause a deadlock (cpu-hotplug try to lock
491 * text_mutex but stop_machine can not be done because online_cpus
492 * has been changed)
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200493 * To avoid this deadlock, caller must have locked cpu hotplug
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500494 * for preventing cpu-hotplug outside of text_mutex locking.
495 */
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200496 lockdep_assert_cpus_held();
497
498 /* Optimization never be done when disarmed */
499 if (kprobes_all_disarmed || !kprobes_allow_optimization ||
500 list_empty(&optimizing_list))
501 return;
502
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500503 mutex_lock(&text_mutex);
Masami Hiramatsucd7ebe22010-12-03 18:54:28 +0900504 arch_optimize_kprobes(&optimizing_list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500505 mutex_unlock(&text_mutex);
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900506}
507
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900508/*
509 * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
510 * if need) kprobes listed on unoptimizing_list.
511 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900512static void do_unoptimize_kprobes(void)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900513{
514 struct optimized_kprobe *op, *tmp;
515
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200516 /* See comment in do_optimize_kprobes() */
517 lockdep_assert_cpus_held();
518
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900519 /* Unoptimization must be done anytime */
520 if (list_empty(&unoptimizing_list))
521 return;
522
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900523 mutex_lock(&text_mutex);
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900524 arch_unoptimize_kprobes(&unoptimizing_list, &freeing_list);
Masami Hiramatsuf984ba42010-12-03 18:54:34 +0900525 /* Loop free_list for disarming */
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900526 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900527 /* Disarm probes if marked disabled */
528 if (kprobe_disabled(&op->kp))
529 arch_disarm_kprobe(&op->kp);
530 if (kprobe_unused(&op->kp)) {
531 /*
532 * Remove unused probes from hash list. After waiting
533 * for synchronization, these probes are reclaimed.
534 * (reclaiming is done by do_free_cleaned_kprobes.)
535 */
536 hlist_del_rcu(&op->kp.hlist);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900537 } else
538 list_del_init(&op->list);
539 }
540 mutex_unlock(&text_mutex);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900541}
542
543/* Reclaim all kprobes on the free_list */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900544static void do_free_cleaned_kprobes(void)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900545{
546 struct optimized_kprobe *op, *tmp;
547
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900548 list_for_each_entry_safe(op, tmp, &freeing_list, list) {
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900549 BUG_ON(!kprobe_unused(&op->kp));
550 list_del_init(&op->list);
551 free_aggr_kprobe(&op->kp);
552 }
553}
554
555/* Start optimizer after OPTIMIZE_DELAY passed */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900556static void kick_kprobe_optimizer(void)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900557{
Tejun Heoad72b3b2012-12-21 17:57:00 -0800558 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900559}
560
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900561/* Kprobe jump optimizer */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900562static void kprobe_optimizer(struct work_struct *work)
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900563{
Steven Rostedt72ef3792012-06-05 19:28:14 +0900564 mutex_lock(&kprobe_mutex);
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200565 cpus_read_lock();
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900566 /* Lock modules while optimizing kprobes */
567 mutex_lock(&module_mutex);
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900568
569 /*
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900570 * Step 1: Unoptimize kprobes and collect cleaned (unused and disarmed)
571 * kprobes before waiting for quiesence period.
572 */
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900573 do_unoptimize_kprobes();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900574
575 /*
Masami Hiramatsua30b85d2017-10-20 08:43:39 +0900576 * Step 2: Wait for quiesence period to ensure all potentially
577 * preempted tasks to have normally scheduled. Because optprobe
578 * may modify multiple instructions, there is a chance that Nth
579 * instruction is preempted. In that case, such tasks can return
580 * to 2nd-Nth byte of jump instruction. This wait is for avoiding it.
581 * Note that on non-preemptive kernel, this is transparently converted
582 * to synchronoze_sched() to wait for all interrupts to have completed.
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900583 */
Masami Hiramatsua30b85d2017-10-20 08:43:39 +0900584 synchronize_rcu_tasks();
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900585
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900586 /* Step 3: Optimize kprobes after quiesence period */
Masami Hiramatsu61f4e132010-12-03 18:54:03 +0900587 do_optimize_kprobes();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900588
589 /* Step 4: Free cleaned kprobes after quiesence period */
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900590 do_free_cleaned_kprobes();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900591
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500592 mutex_unlock(&module_mutex);
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200593 cpus_read_unlock();
Steven Rostedt72ef3792012-06-05 19:28:14 +0900594 mutex_unlock(&kprobe_mutex);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900595
Masami Hiramatsucd7ebe22010-12-03 18:54:28 +0900596 /* Step 5: Kick optimizer again if needed */
Masami Hiramatsuf984ba42010-12-03 18:54:34 +0900597 if (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list))
Masami Hiramatsucd7ebe22010-12-03 18:54:28 +0900598 kick_kprobe_optimizer();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900599}
600
601/* Wait for completing optimization and unoptimization */
Thomas Gleixner30e7d892017-05-17 10:19:49 +0200602void wait_for_kprobe_optimizer(void)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900603{
Tejun Heoad72b3b2012-12-21 17:57:00 -0800604 mutex_lock(&kprobe_mutex);
605
606 while (!list_empty(&optimizing_list) || !list_empty(&unoptimizing_list)) {
607 mutex_unlock(&kprobe_mutex);
608
609 /* this will also make optimizing_work execute immmediately */
610 flush_delayed_work(&optimizing_work);
611 /* @optimizing_work might not have been queued yet, relax */
612 cpu_relax();
613
614 mutex_lock(&kprobe_mutex);
615 }
616
617 mutex_unlock(&kprobe_mutex);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500618}
619
620/* Optimize kprobe if p is ready to be optimized */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900621static void optimize_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500622{
623 struct optimized_kprobe *op;
624
625 /* Check if the kprobe is disabled or not ready for optimization. */
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500626 if (!kprobe_optready(p) || !kprobes_allow_optimization ||
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500627 (kprobe_disabled(p) || kprobes_all_disarmed))
628 return;
629
Masami Hiramatsu059053a2018-06-20 01:10:27 +0900630 /* kprobes with post_handler can not be optimized */
631 if (p->post_handler)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500632 return;
633
634 op = container_of(p, struct optimized_kprobe, kp);
635
636 /* Check there is no other kprobes at the optimized instructions */
637 if (arch_check_optimized_kprobe(op) < 0)
638 return;
639
640 /* Check if it is already optimized. */
641 if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
642 return;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500643 op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900644
645 if (!list_empty(&op->list))
646 /* This is under unoptimizing. Just dequeue the probe */
647 list_del_init(&op->list);
648 else {
649 list_add(&op->list, &optimizing_list);
650 kick_kprobe_optimizer();
651 }
652}
653
654/* Short cut to direct unoptimizing */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900655static void force_unoptimize_kprobe(struct optimized_kprobe *op)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900656{
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200657 lockdep_assert_cpus_held();
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900658 arch_unoptimize_kprobe(op);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900659 if (kprobe_disabled(&op->kp))
660 arch_disarm_kprobe(&op->kp);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500661}
662
663/* Unoptimize a kprobe if p is optimized */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900664static void unoptimize_kprobe(struct kprobe *p, bool force)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500665{
666 struct optimized_kprobe *op;
667
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900668 if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
669 return; /* This is not an optprobe nor optimized */
670
671 op = container_of(p, struct optimized_kprobe, kp);
672 if (!kprobe_optimized(p)) {
673 /* Unoptimized or unoptimizing case */
674 if (force && !list_empty(&op->list)) {
675 /*
676 * Only if this is unoptimizing kprobe and forced,
677 * forcibly unoptimize it. (No need to unoptimize
678 * unoptimized kprobe again :)
679 */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500680 list_del_init(&op->list);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900681 force_unoptimize_kprobe(op);
682 }
683 return;
684 }
685
686 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
687 if (!list_empty(&op->list)) {
688 /* Dequeue from the optimization queue */
689 list_del_init(&op->list);
690 return;
691 }
692 /* Optimized kprobe case */
693 if (force)
694 /* Forcibly update the code: this is a special case */
695 force_unoptimize_kprobe(op);
696 else {
697 list_add(&op->list, &unoptimizing_list);
698 kick_kprobe_optimizer();
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500699 }
700}
701
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900702/* Cancel unoptimizing for reusing */
Masami Hiramatsu819319f2018-09-11 19:20:40 +0900703static int reuse_unused_kprobe(struct kprobe *ap)
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900704{
705 struct optimized_kprobe *op;
Masami Hiramatsu819319f2018-09-11 19:20:40 +0900706 int ret;
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900707
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900708 /*
709 * Unused kprobe MUST be on the way of delayed unoptimizing (means
710 * there is still a relative jump) and disabled.
711 */
712 op = container_of(ap, struct optimized_kprobe, kp);
Masami Hiramatsu44585152018-04-28 21:36:33 +0900713 WARN_ON_ONCE(list_empty(&op->list));
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900714 /* Enable the probe again */
715 ap->flags &= ~KPROBE_FLAG_DISABLED;
716 /* Optimize it again (remove from op->list) */
Masami Hiramatsu819319f2018-09-11 19:20:40 +0900717 ret = kprobe_optready(ap);
718 if (ret)
719 return ret;
720
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900721 optimize_kprobe(ap);
Masami Hiramatsu819319f2018-09-11 19:20:40 +0900722 return 0;
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900723}
724
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500725/* Remove optimized instructions */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900726static void kill_optimized_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500727{
728 struct optimized_kprobe *op;
729
730 op = container_of(p, struct optimized_kprobe, kp);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900731 if (!list_empty(&op->list))
732 /* Dequeue from the (un)optimization queue */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500733 list_del_init(&op->list);
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900734 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
Masami Hiramatsu7b959fc2013-05-22 18:34:09 +0900735
736 if (kprobe_unused(p)) {
737 /* Enqueue if it is unused */
738 list_add(&op->list, &freeing_list);
739 /*
740 * Remove unused probes from the hash list. After waiting
741 * for synchronization, this probe is reclaimed.
742 * (reclaiming is done by do_free_cleaned_kprobes().)
743 */
744 hlist_del_rcu(&op->kp.hlist);
745 }
746
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900747 /* Don't touch the code, because it is already freed. */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500748 arch_remove_optimized_kprobe(op);
749}
750
Masami Hiramatsua4602462017-04-19 18:22:25 +0530751static inline
752void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
753{
754 if (!kprobe_ftrace(p))
755 arch_prepare_optimized_kprobe(op, p);
756}
757
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500758/* Try to prepare optimized instructions */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900759static void prepare_optimized_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500760{
761 struct optimized_kprobe *op;
762
763 op = container_of(p, struct optimized_kprobe, kp);
Masami Hiramatsua4602462017-04-19 18:22:25 +0530764 __prepare_optimized_kprobe(op, p);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500765}
766
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500767/* Allocate new optimized_kprobe and try to prepare optimized instructions */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900768static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500769{
770 struct optimized_kprobe *op;
771
772 op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
773 if (!op)
774 return NULL;
775
776 INIT_LIST_HEAD(&op->list);
777 op->kp.addr = p->addr;
Masami Hiramatsua4602462017-04-19 18:22:25 +0530778 __prepare_optimized_kprobe(op, p);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500779
780 return &op->kp;
781}
782
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900783static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500784
785/*
786 * Prepare an optimized_kprobe and optimize it
787 * NOTE: p must be a normal registered kprobe
788 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900789static void try_to_optimize_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500790{
791 struct kprobe *ap;
792 struct optimized_kprobe *op;
793
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900794 /* Impossible to optimize ftrace-based kprobe */
795 if (kprobe_ftrace(p))
796 return;
797
Masami Hiramatsu25764282012-06-05 19:28:26 +0900798 /* For preparing optimization, jump_label_text_reserved() is called */
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200799 cpus_read_lock();
Masami Hiramatsu25764282012-06-05 19:28:26 +0900800 jump_label_lock();
801 mutex_lock(&text_mutex);
802
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500803 ap = alloc_aggr_kprobe(p);
804 if (!ap)
Masami Hiramatsu25764282012-06-05 19:28:26 +0900805 goto out;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500806
807 op = container_of(ap, struct optimized_kprobe, kp);
808 if (!arch_prepared_optinsn(&op->optinsn)) {
809 /* If failed to setup optimizing, fallback to kprobe */
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900810 arch_remove_optimized_kprobe(op);
811 kfree(op);
Masami Hiramatsu25764282012-06-05 19:28:26 +0900812 goto out;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500813 }
814
815 init_aggr_kprobe(ap, p);
Masami Hiramatsu25764282012-06-05 19:28:26 +0900816 optimize_kprobe(ap); /* This just kicks optimizer thread */
817
818out:
819 mutex_unlock(&text_mutex);
820 jump_label_unlock();
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200821 cpus_read_unlock();
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500822}
823
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500824#ifdef CONFIG_SYSCTL
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900825static void optimize_all_kprobes(void)
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500826{
827 struct hlist_head *head;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500828 struct kprobe *p;
829 unsigned int i;
830
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900831 mutex_lock(&kprobe_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500832 /* If optimization is already allowed, just return */
833 if (kprobes_allow_optimization)
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900834 goto out;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500835
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200836 cpus_read_lock();
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500837 kprobes_allow_optimization = true;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500838 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
839 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -0800840 hlist_for_each_entry_rcu(p, head, hlist)
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500841 if (!kprobe_disabled(p))
842 optimize_kprobe(p);
843 }
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200844 cpus_read_unlock();
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500845 printk(KERN_INFO "Kprobes globally optimized\n");
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900846out:
847 mutex_unlock(&kprobe_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500848}
849
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900850static void unoptimize_all_kprobes(void)
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500851{
852 struct hlist_head *head;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500853 struct kprobe *p;
854 unsigned int i;
855
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900856 mutex_lock(&kprobe_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500857 /* If optimization is already prohibited, just return */
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900858 if (!kprobes_allow_optimization) {
859 mutex_unlock(&kprobe_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500860 return;
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900861 }
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500862
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200863 cpus_read_lock();
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500864 kprobes_allow_optimization = false;
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500865 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
866 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -0800867 hlist_for_each_entry_rcu(p, head, hlist) {
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500868 if (!kprobe_disabled(p))
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900869 unoptimize_kprobe(p, false);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500870 }
871 }
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +0200872 cpus_read_unlock();
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900873 mutex_unlock(&kprobe_mutex);
874
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900875 /* Wait for unoptimizing completion */
876 wait_for_kprobe_optimizer();
877 printk(KERN_INFO "Kprobes globally unoptimized\n");
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500878}
879
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900880static DEFINE_MUTEX(kprobe_sysctl_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500881int sysctl_kprobes_optimization;
882int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
883 void __user *buffer, size_t *length,
884 loff_t *ppos)
885{
886 int ret;
887
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900888 mutex_lock(&kprobe_sysctl_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500889 sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
890 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
891
892 if (sysctl_kprobes_optimization)
893 optimize_all_kprobes();
894 else
895 unoptimize_all_kprobes();
Masami Hiramatsu5c515432013-04-18 18:33:18 +0900896 mutex_unlock(&kprobe_sysctl_mutex);
Masami Hiramatsub2be84d2010-02-25 08:34:15 -0500897
898 return ret;
899}
900#endif /* CONFIG_SYSCTL */
901
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900902/* Put a breakpoint for a probe. Must be called with text_mutex locked */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900903static void __arm_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500904{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900905 struct kprobe *_p;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500906
907 /* Check collision with other optimized kprobes */
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900908 _p = get_optimized_kprobe((unsigned long)p->addr);
909 if (unlikely(_p))
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900910 /* Fallback to unoptimized kprobe */
911 unoptimize_kprobe(_p, true);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500912
913 arch_arm_kprobe(p);
914 optimize_kprobe(p); /* Try to optimize (add kprobe to a list) */
915}
916
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900917/* Remove the breakpoint of a probe. Must be called with text_mutex locked */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900918static void __disarm_kprobe(struct kprobe *p, bool reopt)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500919{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +0900920 struct kprobe *_p;
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500921
Wang Nan69d54b92015-02-13 14:40:26 -0800922 /* Try to unoptimize */
923 unoptimize_kprobe(p, kprobes_all_disarmed);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500924
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900925 if (!kprobe_queued(p)) {
926 arch_disarm_kprobe(p);
927 /* If another kprobe was blocked, optimize it. */
928 _p = get_optimized_kprobe((unsigned long)p->addr);
929 if (unlikely(_p) && reopt)
930 optimize_kprobe(_p);
931 }
932 /* TODO: reoptimize others after unoptimized this probe */
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500933}
934
935#else /* !CONFIG_OPTPROBES */
936
937#define optimize_kprobe(p) do {} while (0)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900938#define unoptimize_kprobe(p, f) do {} while (0)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500939#define kill_optimized_kprobe(p) do {} while (0)
940#define prepare_optimized_kprobe(p) do {} while (0)
941#define try_to_optimize_kprobe(p) do {} while (0)
942#define __arm_kprobe(p) arch_arm_kprobe(p)
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900943#define __disarm_kprobe(p, o) arch_disarm_kprobe(p)
944#define kprobe_disarmed(p) kprobe_disabled(p)
945#define wait_for_kprobe_optimizer() do {} while (0)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500946
Masami Hiramatsu819319f2018-09-11 19:20:40 +0900947static int reuse_unused_kprobe(struct kprobe *ap)
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900948{
Masami Hiramatsu819319f2018-09-11 19:20:40 +0900949 /*
950 * If the optimized kprobe is NOT supported, the aggr kprobe is
951 * released at the same time that the last aggregated kprobe is
952 * unregistered.
953 * Thus there should be no chance to reuse unused kprobe.
954 */
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900955 printk(KERN_ERR "Error: There should be no unused kprobe here.\n");
Masami Hiramatsu819319f2018-09-11 19:20:40 +0900956 return -EINVAL;
Masami Hiramatsu0490cd12010-12-03 18:54:16 +0900957}
958
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900959static void free_aggr_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500960{
Masami Hiramatsu6274de42010-12-03 18:54:09 +0900961 arch_remove_kprobe(p);
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500962 kfree(p);
963}
964
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900965static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
Masami Hiramatsuafd66252010-02-25 08:34:07 -0500966{
967 return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
968}
969#endif /* CONFIG_OPTPROBES */
970
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +0900971#ifdef CONFIG_KPROBES_ON_FTRACE
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900972static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
Masami Hiramatsue5253892012-06-05 19:28:38 +0900973 .func = kprobe_ftrace_handler,
Masami Hiramatsu1d70be32014-11-21 05:25:23 -0500974 .flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900975};
976static int kprobe_ftrace_enabled;
977
978/* Must ensure p->addr is really on ftrace */
Masami Hiramatsu55479f62014-04-17 17:17:54 +0900979static int prepare_kprobe(struct kprobe *p)
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900980{
981 if (!kprobe_ftrace(p))
982 return arch_prepare_kprobe(p);
983
984 return arch_prepare_kprobe_ftrace(p);
985}
986
987/* Caller must lock kprobe_mutex */
Jessica Yu12310e342018-01-10 00:51:23 +0100988static int arm_kprobe_ftrace(struct kprobe *p)
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900989{
Jessica Yu12310e342018-01-10 00:51:23 +0100990 int ret = 0;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900991
992 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
993 (unsigned long)p->addr, 0, 0);
Jessica Yu12310e342018-01-10 00:51:23 +0100994 if (ret) {
Masami Hiramatsu44585152018-04-28 21:36:33 +0900995 pr_debug("Failed to arm kprobe-ftrace at %pS (%d)\n",
996 p->addr, ret);
Jessica Yu12310e342018-01-10 00:51:23 +0100997 return ret;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +0900998 }
Jessica Yu12310e342018-01-10 00:51:23 +0100999
1000 if (kprobe_ftrace_enabled == 0) {
1001 ret = register_ftrace_function(&kprobe_ftrace_ops);
1002 if (ret) {
1003 pr_debug("Failed to init kprobe-ftrace (%d)\n", ret);
1004 goto err_ftrace;
1005 }
1006 }
1007
1008 kprobe_ftrace_enabled++;
1009 return ret;
1010
1011err_ftrace:
1012 /*
1013 * Note: Since kprobe_ftrace_ops has IPMODIFY set, and ftrace requires a
1014 * non-empty filter_hash for IPMODIFY ops, we're safe from an accidental
1015 * empty filter_hash which would undesirably trace all functions.
1016 */
1017 ftrace_set_filter_ip(&kprobe_ftrace_ops, (unsigned long)p->addr, 1, 0);
1018 return ret;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001019}
1020
1021/* Caller must lock kprobe_mutex */
Jessica Yu297f9232018-01-10 00:51:24 +01001022static int disarm_kprobe_ftrace(struct kprobe *p)
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001023{
Jessica Yu297f9232018-01-10 00:51:24 +01001024 int ret = 0;
1025
1026 if (kprobe_ftrace_enabled == 1) {
1027 ret = unregister_ftrace_function(&kprobe_ftrace_ops);
1028 if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (%d)\n", ret))
1029 return ret;
1030 }
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001031
1032 kprobe_ftrace_enabled--;
Jessica Yu297f9232018-01-10 00:51:24 +01001033
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001034 ret = ftrace_set_filter_ip(&kprobe_ftrace_ops,
1035 (unsigned long)p->addr, 1, 0);
Masami Hiramatsu44585152018-04-28 21:36:33 +09001036 WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (%d)\n",
1037 p->addr, ret);
Jessica Yu297f9232018-01-10 00:51:24 +01001038 return ret;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001039}
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +09001040#else /* !CONFIG_KPROBES_ON_FTRACE */
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001041#define prepare_kprobe(p) arch_prepare_kprobe(p)
Jessica Yu12310e342018-01-10 00:51:23 +01001042#define arm_kprobe_ftrace(p) (-ENODEV)
Jessica Yu297f9232018-01-10 00:51:24 +01001043#define disarm_kprobe_ftrace(p) (-ENODEV)
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001044#endif
1045
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001046/* Arm a kprobe with text_mutex */
Jessica Yu12310e342018-01-10 00:51:23 +01001047static int arm_kprobe(struct kprobe *kp)
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001048{
Jessica Yu12310e342018-01-10 00:51:23 +01001049 if (unlikely(kprobe_ftrace(kp)))
1050 return arm_kprobe_ftrace(kp);
1051
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +02001052 cpus_read_lock();
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001053 mutex_lock(&text_mutex);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001054 __arm_kprobe(kp);
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001055 mutex_unlock(&text_mutex);
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +02001056 cpus_read_unlock();
Jessica Yu12310e342018-01-10 00:51:23 +01001057
1058 return 0;
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001059}
1060
1061/* Disarm a kprobe with text_mutex */
Jessica Yu297f9232018-01-10 00:51:24 +01001062static int disarm_kprobe(struct kprobe *kp, bool reopt)
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001063{
Jessica Yu297f9232018-01-10 00:51:24 +01001064 if (unlikely(kprobe_ftrace(kp)))
1065 return disarm_kprobe_ftrace(kp);
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +02001066
1067 cpus_read_lock();
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001068 mutex_lock(&text_mutex);
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001069 __disarm_kprobe(kp, reopt);
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001070 mutex_unlock(&text_mutex);
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +02001071 cpus_read_unlock();
Jessica Yu297f9232018-01-10 00:51:24 +01001072
1073 return 0;
Masami Hiramatsu201517a2009-05-07 16:31:26 -04001074}
1075
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001076/*
1077 * Aggregate handlers for multiple kprobes support - these handlers
1078 * take care of invoking the individual kprobe handlers on p->list
1079 */
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001080static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001081{
1082 struct kprobe *kp;
1083
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001084 list_for_each_entry_rcu(kp, &p->list, list) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001085 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001086 set_kprobe_instance(kp);
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001087 if (kp->pre_handler(kp, regs))
1088 return 1;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001089 }
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001090 reset_kprobe_instance();
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001091 }
1092 return 0;
1093}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001094NOKPROBE_SYMBOL(aggr_pre_handler);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001095
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001096static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1097 unsigned long flags)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001098{
1099 struct kprobe *kp;
1100
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001101 list_for_each_entry_rcu(kp, &p->list, list) {
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001102 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001103 set_kprobe_instance(kp);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001104 kp->post_handler(kp, regs, flags);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001105 reset_kprobe_instance();
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001106 }
1107 }
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001108}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001109NOKPROBE_SYMBOL(aggr_post_handler);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001110
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001111static int aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
1112 int trapnr)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001113{
Christoph Lameterb76834b2010-12-06 11:16:25 -06001114 struct kprobe *cur = __this_cpu_read(kprobe_instance);
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001115
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001116 /*
1117 * if we faulted "during" the execution of a user specified
1118 * probe handler, invoke just that probe's fault handler
1119 */
Ananth N Mavinakayanahallie6584522005-11-07 01:00:07 -08001120 if (cur && cur->fault_handler) {
1121 if (cur->fault_handler(cur, regs, trapnr))
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001122 return 1;
1123 }
1124 return 0;
1125}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001126NOKPROBE_SYMBOL(aggr_fault_handler);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001127
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -08001128/* Walks the list and increments nmissed count for multiprobe case */
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001129void kprobes_inc_nmissed_count(struct kprobe *p)
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -08001130{
1131 struct kprobe *kp;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001132 if (!kprobe_aggrprobe(p)) {
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -08001133 p->nmissed++;
1134 } else {
1135 list_for_each_entry_rcu(kp, &p->list, list)
1136 kp->nmissed++;
1137 }
1138 return;
1139}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001140NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
Keshavamurthy Anil Sbf8d5c52005-12-12 00:37:34 -08001141
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001142void recycle_rp_inst(struct kretprobe_instance *ri,
1143 struct hlist_head *head)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001144{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001145 struct kretprobe *rp = ri->rp;
1146
Hien Nguyenb94cce92005-06-23 00:09:19 -07001147 /* remove rp inst off the rprobe_inst_table */
1148 hlist_del(&ri->hlist);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001149 INIT_HLIST_NODE(&ri->hlist);
1150 if (likely(rp)) {
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001151 raw_spin_lock(&rp->lock);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001152 hlist_add_head(&ri->hlist, &rp->free_instances);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001153 raw_spin_unlock(&rp->lock);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001154 } else
1155 /* Unregistering */
bibo,mao99219a32006-10-02 02:17:35 -07001156 hlist_add_head(&ri->hlist, head);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001157}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001158NOKPROBE_SYMBOL(recycle_rp_inst);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001159
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001160void kretprobe_hash_lock(struct task_struct *tsk,
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001161 struct hlist_head **head, unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001162__acquires(hlist_lock)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001163{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001164 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001165 raw_spinlock_t *hlist_lock;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001166
1167 *head = &kretprobe_inst_table[hash];
1168 hlist_lock = kretprobe_table_lock_ptr(hash);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001169 raw_spin_lock_irqsave(hlist_lock, *flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001170}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001171NOKPROBE_SYMBOL(kretprobe_hash_lock);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001172
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001173static void kretprobe_table_lock(unsigned long hash,
1174 unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001175__acquires(hlist_lock)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001176{
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001177 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1178 raw_spin_lock_irqsave(hlist_lock, *flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001179}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001180NOKPROBE_SYMBOL(kretprobe_table_lock);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001181
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001182void kretprobe_hash_unlock(struct task_struct *tsk,
1183 unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001184__releases(hlist_lock)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001185{
1186 unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001187 raw_spinlock_t *hlist_lock;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001188
1189 hlist_lock = kretprobe_table_lock_ptr(hash);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001190 raw_spin_unlock_irqrestore(hlist_lock, *flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001191}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001192NOKPROBE_SYMBOL(kretprobe_hash_unlock);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001193
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001194static void kretprobe_table_unlock(unsigned long hash,
1195 unsigned long *flags)
Namhyung Kim635c17c2010-09-15 10:04:30 +09001196__releases(hlist_lock)
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001197{
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001198 raw_spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
1199 raw_spin_unlock_irqrestore(hlist_lock, *flags);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001200}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001201NOKPROBE_SYMBOL(kretprobe_table_unlock);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001202
Hien Nguyenb94cce92005-06-23 00:09:19 -07001203/*
bibo maoc6fd91f2006-03-26 01:38:20 -08001204 * This function is called from finish_task_switch when task tk becomes dead,
1205 * so that we can recycle any function-return probe instances associated
1206 * with this task. These left over instances represent probed functions
1207 * that have been called but will never return.
Hien Nguyenb94cce92005-06-23 00:09:19 -07001208 */
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001209void kprobe_flush_task(struct task_struct *tk)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001210{
bibo,mao62c27be2006-10-02 02:17:33 -07001211 struct kretprobe_instance *ri;
bibo,mao99219a32006-10-02 02:17:35 -07001212 struct hlist_head *head, empty_rp;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001213 struct hlist_node *tmp;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001214 unsigned long hash, flags = 0;
Rusty Lynch802eae72005-06-27 15:17:08 -07001215
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001216 if (unlikely(!kprobes_initialized))
1217 /* Early boot. kretprobe_table_locks not yet initialized. */
1218 return;
1219
Ananth N Mavinakayanahallid496aab2012-01-20 14:34:04 -08001220 INIT_HLIST_HEAD(&empty_rp);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001221 hash = hash_ptr(tk, KPROBE_HASH_BITS);
1222 head = &kretprobe_inst_table[hash];
1223 kretprobe_table_lock(hash, &flags);
Sasha Levinb67bfe02013-02-27 17:06:00 -08001224 hlist_for_each_entry_safe(ri, tmp, head, hlist) {
bibo,mao62c27be2006-10-02 02:17:33 -07001225 if (ri->task == tk)
bibo,mao99219a32006-10-02 02:17:35 -07001226 recycle_rp_inst(ri, &empty_rp);
bibo,mao62c27be2006-10-02 02:17:33 -07001227 }
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001228 kretprobe_table_unlock(hash, &flags);
Sasha Levinb67bfe02013-02-27 17:06:00 -08001229 hlist_for_each_entry_safe(ri, tmp, &empty_rp, hlist) {
bibo,mao99219a32006-10-02 02:17:35 -07001230 hlist_del(&ri->hlist);
1231 kfree(ri);
1232 }
Hien Nguyenb94cce92005-06-23 00:09:19 -07001233}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001234NOKPROBE_SYMBOL(kprobe_flush_task);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001235
Hien Nguyenb94cce92005-06-23 00:09:19 -07001236static inline void free_rp_inst(struct kretprobe *rp)
1237{
1238 struct kretprobe_instance *ri;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001239 struct hlist_node *next;
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001240
Sasha Levinb67bfe02013-02-27 17:06:00 -08001241 hlist_for_each_entry_safe(ri, next, &rp->free_instances, hlist) {
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001242 hlist_del(&ri->hlist);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001243 kfree(ri);
1244 }
1245}
1246
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001247static void cleanup_rp_inst(struct kretprobe *rp)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001248{
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001249 unsigned long flags, hash;
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001250 struct kretprobe_instance *ri;
Sasha Levinb67bfe02013-02-27 17:06:00 -08001251 struct hlist_node *next;
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001252 struct hlist_head *head;
1253
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001254 /* No race here */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001255 for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
1256 kretprobe_table_lock(hash, &flags);
1257 head = &kretprobe_inst_table[hash];
Sasha Levinb67bfe02013-02-27 17:06:00 -08001258 hlist_for_each_entry_safe(ri, next, head, hlist) {
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001259 if (ri->rp == rp)
1260 ri->rp = NULL;
1261 }
1262 kretprobe_table_unlock(hash, &flags);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001263 }
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001264 free_rp_inst(rp);
1265}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001266NOKPROBE_SYMBOL(cleanup_rp_inst);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001267
Masami Hiramatsu059053a2018-06-20 01:10:27 +09001268/* Add the new probe to ap->list */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001269static int add_new_kprobe(struct kprobe *ap, struct kprobe *p)
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001270{
Masami Hiramatsu059053a2018-06-20 01:10:27 +09001271 if (p->post_handler)
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001272 unoptimize_kprobe(ap, true); /* Fall back to normal kprobe */
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001273
Masami Hiramatsu059053a2018-06-20 01:10:27 +09001274 list_add_rcu(&p->list, &ap->list);
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001275 if (p->post_handler && !ap->post_handler)
1276 ap->post_handler = aggr_post_handler;
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001277
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001278 return 0;
1279}
1280
1281/*
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001282 * Fill in the required fields of the "manager kprobe". Replace the
1283 * earlier kprobe in the hlist with the manager kprobe
1284 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001285static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001286{
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001287 /* Copy p's insn slot to ap */
Prasanna S Panchamukhi8b0914e2005-06-23 00:09:41 -07001288 copy_kprobe(p, ap);
bibo, maoa9ad9652006-07-30 03:03:26 -07001289 flush_insn_slot(ap);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001290 ap->addr = p->addr;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001291 ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001292 ap->pre_handler = aggr_pre_handler;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001293 ap->fault_handler = aggr_fault_handler;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001294 /* We don't care the kprobe which has gone. */
1295 if (p->post_handler && !kprobe_gone(p))
mao, bibo36721652006-06-26 00:25:22 -07001296 ap->post_handler = aggr_post_handler;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001297
1298 INIT_LIST_HEAD(&ap->list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001299 INIT_HLIST_NODE(&ap->hlist);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001300
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001301 list_add_rcu(&p->list, &ap->list);
Keshavamurthy Anil Sadad0f32005-12-12 00:37:12 -08001302 hlist_replace_rcu(&p->hlist, &ap->hlist);
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001303}
1304
1305/*
1306 * This is the second or subsequent kprobe at the address - handle
1307 * the intricacies
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001308 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001309static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p)
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001310{
1311 int ret = 0;
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001312 struct kprobe *ap = orig_p;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001313
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +02001314 cpus_read_lock();
1315
Masami Hiramatsu25764282012-06-05 19:28:26 +09001316 /* For preparing optimization, jump_label_text_reserved() is called */
1317 jump_label_lock();
Masami Hiramatsu25764282012-06-05 19:28:26 +09001318 mutex_lock(&text_mutex);
1319
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001320 if (!kprobe_aggrprobe(orig_p)) {
1321 /* If orig_p is not an aggr_kprobe, create new aggr_kprobe. */
1322 ap = alloc_aggr_kprobe(orig_p);
Masami Hiramatsu25764282012-06-05 19:28:26 +09001323 if (!ap) {
1324 ret = -ENOMEM;
1325 goto out;
1326 }
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001327 init_aggr_kprobe(ap, orig_p);
Masami Hiramatsu819319f2018-09-11 19:20:40 +09001328 } else if (kprobe_unused(ap)) {
Masami Hiramatsu0490cd12010-12-03 18:54:16 +09001329 /* This probe is going to die. Rescue it */
Masami Hiramatsu819319f2018-09-11 19:20:40 +09001330 ret = reuse_unused_kprobe(ap);
1331 if (ret)
1332 goto out;
1333 }
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001334
1335 if (kprobe_gone(ap)) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001336 /*
1337 * Attempting to insert new probe at the same location that
1338 * had a probe in the module vaddr area which already
1339 * freed. So, the instruction slot has already been
1340 * released. We need a new slot for the new probe.
1341 */
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001342 ret = arch_prepare_kprobe(ap);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001343 if (ret)
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001344 /*
1345 * Even if fail to allocate new slot, don't need to
1346 * free aggr_probe. It will be used next time, or
1347 * freed by unregister_kprobe.
1348 */
Masami Hiramatsu25764282012-06-05 19:28:26 +09001349 goto out;
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001350
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001351 /* Prepare optimized instructions if possible. */
1352 prepare_optimized_kprobe(ap);
1353
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001354 /*
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001355 * Clear gone flag to prevent allocating new slot again, and
1356 * set disabled flag because it is not armed yet.
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001357 */
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001358 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1359 | KPROBE_FLAG_DISABLED;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001360 }
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001361
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001362 /* Copy ap's insn slot to p */
Masami Hiramatsub918e5e2009-04-06 19:00:58 -07001363 copy_kprobe(ap, p);
Masami Hiramatsu25764282012-06-05 19:28:26 +09001364 ret = add_new_kprobe(ap, p);
1365
1366out:
1367 mutex_unlock(&text_mutex);
Masami Hiramatsu25764282012-06-05 19:28:26 +09001368 jump_label_unlock();
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +02001369 cpus_read_unlock();
Masami Hiramatsu25764282012-06-05 19:28:26 +09001370
1371 if (ret == 0 && kprobe_disabled(ap) && !kprobe_disabled(p)) {
1372 ap->flags &= ~KPROBE_FLAG_DISABLED;
Jessica Yu12310e342018-01-10 00:51:23 +01001373 if (!kprobes_all_disarmed) {
Masami Hiramatsu25764282012-06-05 19:28:26 +09001374 /* Arm the breakpoint again. */
Jessica Yu12310e342018-01-10 00:51:23 +01001375 ret = arm_kprobe(ap);
1376 if (ret) {
1377 ap->flags |= KPROBE_FLAG_DISABLED;
1378 list_del_rcu(&p->list);
1379 synchronize_sched();
1380 }
1381 }
Masami Hiramatsu25764282012-06-05 19:28:26 +09001382 }
1383 return ret;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001384}
1385
Masami Hiramatsube8f2742014-04-17 17:16:58 +09001386bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1387{
1388 /* The __kprobes marked functions and entry code must not be probed */
1389 return addr >= (unsigned long)__kprobes_text_start &&
1390 addr < (unsigned long)__kprobes_text_end;
1391}
1392
Andy Lutomirskie5779e82015-07-30 20:32:40 -07001393bool within_kprobe_blacklist(unsigned long addr)
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001394{
Masami Hiramatsu376e2422014-04-17 17:17:05 +09001395 struct kprobe_blacklist_entry *ent;
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07001396
Masami Hiramatsube8f2742014-04-17 17:16:58 +09001397 if (arch_within_kprobe_blacklist(addr))
Masami Hiramatsu376e2422014-04-17 17:17:05 +09001398 return true;
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07001399 /*
1400 * If there exists a kprobe_blacklist, verify and
1401 * fail any probe registration in the prohibited area
1402 */
Masami Hiramatsu376e2422014-04-17 17:17:05 +09001403 list_for_each_entry(ent, &kprobe_blacklist, list) {
1404 if (addr >= ent->start_addr && addr < ent->end_addr)
1405 return true;
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07001406 }
Masami Hiramatsu376e2422014-04-17 17:17:05 +09001407
1408 return false;
Prasanna S Panchamukhid0aaff92005-09-06 15:19:26 -07001409}
1410
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001411/*
1412 * If we have a symbol_name argument, look it up and add the offset field
1413 * to it. This way, we can specify a relative address to a symbol.
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001414 * This returns encoded errors if it fails to look up symbol or invalid
1415 * combination of parameters.
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001416 */
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301417static kprobe_opcode_t *_kprobe_addr(kprobe_opcode_t *addr,
1418 const char *symbol_name, unsigned int offset)
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001419{
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301420 if ((symbol_name && addr) || (!symbol_name && !addr))
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001421 goto invalid;
1422
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301423 if (symbol_name) {
Linus Torvalds7246f602017-05-05 11:36:44 -07001424 addr = kprobe_lookup_name(symbol_name, offset);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001425 if (!addr)
1426 return ERR_PTR(-ENOENT);
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001427 }
1428
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301429 addr = (kprobe_opcode_t *)(((char *)addr) + offset);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001430 if (addr)
1431 return addr;
1432
1433invalid:
1434 return ERR_PTR(-EINVAL);
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001435}
1436
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301437static kprobe_opcode_t *kprobe_addr(struct kprobe *p)
1438{
1439 return _kprobe_addr(p->addr, p->symbol_name, p->offset);
1440}
1441
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301442/* Check passed kprobe is valid and return kprobe in kprobe_table. */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001443static struct kprobe *__get_valid_kprobe(struct kprobe *p)
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301444{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001445 struct kprobe *ap, *list_p;
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301446
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001447 ap = get_kprobe(p->addr);
1448 if (unlikely(!ap))
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301449 return NULL;
1450
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001451 if (p != ap) {
1452 list_for_each_entry_rcu(list_p, &ap->list, list)
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301453 if (list_p == p)
1454 /* kprobe p is a valid probe */
1455 goto valid;
1456 return NULL;
1457 }
1458valid:
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001459 return ap;
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301460}
1461
1462/* Return error if the kprobe is being re-registered */
1463static inline int check_kprobe_rereg(struct kprobe *p)
1464{
1465 int ret = 0;
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301466
1467 mutex_lock(&kprobe_mutex);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001468 if (__get_valid_kprobe(p))
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301469 ret = -EINVAL;
1470 mutex_unlock(&kprobe_mutex);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001471
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301472 return ret;
1473}
1474
Heiko Carstensf7f242f2014-10-15 12:17:34 +02001475int __weak arch_check_ftrace_location(struct kprobe *p)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001476{
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001477 unsigned long ftrace_addr;
1478
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001479 ftrace_addr = ftrace_location((unsigned long)p->addr);
1480 if (ftrace_addr) {
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +09001481#ifdef CONFIG_KPROBES_ON_FTRACE
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001482 /* Given address is not on the instruction boundary */
1483 if ((unsigned long)p->addr != ftrace_addr)
1484 return -EILSEQ;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001485 p->flags |= KPROBE_FLAG_FTRACE;
Masami Hiramatsue7dbfe32012-09-28 17:15:20 +09001486#else /* !CONFIG_KPROBES_ON_FTRACE */
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001487 return -EINVAL;
1488#endif
1489 }
Heiko Carstensf7f242f2014-10-15 12:17:34 +02001490 return 0;
1491}
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001492
Heiko Carstensf7f242f2014-10-15 12:17:34 +02001493static int check_kprobe_address_safe(struct kprobe *p,
1494 struct module **probed_mod)
1495{
1496 int ret;
1497
1498 ret = arch_check_ftrace_location(p);
1499 if (ret)
1500 return ret;
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001501 jump_label_lock();
1502 preempt_disable();
1503
1504 /* Ensure it is not in reserved area nor out of text */
1505 if (!kernel_text_address((unsigned long) p->addr) ||
Masami Hiramatsu376e2422014-04-17 17:17:05 +09001506 within_kprobe_blacklist((unsigned long) p->addr) ||
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001507 jump_label_text_reserved(p->addr, p->addr)) {
1508 ret = -EINVAL;
1509 goto out;
1510 }
1511
1512 /* Check if are we probing a module */
1513 *probed_mod = __module_text_address((unsigned long) p->addr);
1514 if (*probed_mod) {
1515 /*
1516 * We must hold a refcount of the probed module while updating
1517 * its code to prohibit unexpected unloading.
1518 */
1519 if (unlikely(!try_module_get(*probed_mod))) {
1520 ret = -ENOENT;
1521 goto out;
1522 }
1523
1524 /*
1525 * If the module freed .init.text, we couldn't insert
1526 * kprobes in there.
1527 */
1528 if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1529 (*probed_mod)->state != MODULE_STATE_COMING) {
1530 module_put(*probed_mod);
1531 *probed_mod = NULL;
1532 ret = -ENOENT;
1533 }
1534 }
1535out:
1536 preempt_enable();
1537 jump_label_unlock();
1538
1539 return ret;
1540}
1541
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001542int register_kprobe(struct kprobe *p)
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001543{
1544 int ret;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001545 struct kprobe *old_p;
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -08001546 struct module *probed_mod;
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001547 kprobe_opcode_t *addr;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001548
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001549 /* Adjust probe address from symbol */
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001550 addr = kprobe_addr(p);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001551 if (IS_ERR(addr))
1552 return PTR_ERR(addr);
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001553 p->addr = addr;
Ananth N Mavinakayanahalli3a872d82006-10-02 02:17:30 -07001554
Ananth N Mavinakayanahalli1f0ab402009-09-15 10:43:07 +05301555 ret = check_kprobe_rereg(p);
1556 if (ret)
1557 return ret;
1558
Masami Hiramatsude5bd882009-04-06 19:01:02 -07001559 /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1560 p->flags &= KPROBE_FLAG_DISABLED;
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001561 p->nmissed = 0;
Masami Hiramatsu98616682008-04-28 02:14:28 -07001562 INIT_LIST_HEAD(&p->list);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001563
Masami Hiramatsuf7fa6ef02012-06-05 19:28:20 +09001564 ret = check_kprobe_address_safe(p, &probed_mod);
1565 if (ret)
1566 return ret;
1567
1568 mutex_lock(&kprobe_mutex);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001569
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001570 old_p = get_kprobe(p->addr);
1571 if (old_p) {
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001572 /* Since this may unoptimize old_p, locking text_mutex. */
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001573 ret = register_aggr_kprobe(old_p, p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001574 goto out;
1575 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001576
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +02001577 cpus_read_lock();
1578 /* Prevent text modification */
1579 mutex_lock(&text_mutex);
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09001580 ret = prepare_kprobe(p);
Masami Hiramatsu25764282012-06-05 19:28:26 +09001581 mutex_unlock(&text_mutex);
Thomas Gleixner2d1e38f2017-05-24 10:15:36 +02001582 cpus_read_unlock();
Christoph Hellwig6f716ac2007-05-08 00:34:13 -07001583 if (ret)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001584 goto out;
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001585
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001586 INIT_HLIST_NODE(&p->hlist);
Ananth N Mavinakayanahalli3516a462005-11-07 01:00:13 -08001587 hlist_add_head_rcu(&p->hlist,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588 &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1589
Jessica Yu12310e342018-01-10 00:51:23 +01001590 if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
1591 ret = arm_kprobe(p);
1592 if (ret) {
1593 hlist_del_rcu(&p->hlist);
1594 synchronize_sched();
1595 goto out;
1596 }
1597 }
Christoph Hellwig74a0b572007-10-16 01:24:07 -07001598
Masami Hiramatsuafd66252010-02-25 08:34:07 -05001599 /* Try to optimize kprobe */
1600 try_to_optimize_kprobe(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001601out:
Ingo Molnar7a7d1cf2006-03-23 03:00:35 -08001602 mutex_unlock(&kprobe_mutex);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001603
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001604 if (probed_mod)
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -08001605 module_put(probed_mod);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001606
Linus Torvalds1da177e2005-04-16 15:20:36 -07001607 return ret;
1608}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001609EXPORT_SYMBOL_GPL(register_kprobe);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001610
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001611/* Check if all probes on the aggrprobe are disabled */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001612static int aggr_kprobe_disabled(struct kprobe *ap)
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001613{
1614 struct kprobe *kp;
1615
1616 list_for_each_entry_rcu(kp, &ap->list, list)
1617 if (!kprobe_disabled(kp))
1618 /*
1619 * There is an active probe on the list.
1620 * We can't disable this ap.
1621 */
1622 return 0;
1623
1624 return 1;
1625}
1626
1627/* Disable one kprobe: Make sure called under kprobe_mutex is locked */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001628static struct kprobe *__disable_kprobe(struct kprobe *p)
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001629{
1630 struct kprobe *orig_p;
Jessica Yu297f9232018-01-10 00:51:24 +01001631 int ret;
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001632
1633 /* Get an original kprobe for return */
1634 orig_p = __get_valid_kprobe(p);
1635 if (unlikely(orig_p == NULL))
Jessica Yu297f9232018-01-10 00:51:24 +01001636 return ERR_PTR(-EINVAL);
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001637
1638 if (!kprobe_disabled(p)) {
1639 /* Disable probe if it is a child probe */
1640 if (p != orig_p)
1641 p->flags |= KPROBE_FLAG_DISABLED;
1642
1643 /* Try to disarm and disable this/parent probe */
1644 if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
Wang Nan69d54b92015-02-13 14:40:26 -08001645 /*
1646 * If kprobes_all_disarmed is set, orig_p
1647 * should have already been disarmed, so
1648 * skip unneed disarming process.
1649 */
Jessica Yu297f9232018-01-10 00:51:24 +01001650 if (!kprobes_all_disarmed) {
1651 ret = disarm_kprobe(orig_p, true);
1652 if (ret) {
1653 p->flags &= ~KPROBE_FLAG_DISABLED;
1654 return ERR_PTR(ret);
1655 }
1656 }
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001657 orig_p->flags |= KPROBE_FLAG_DISABLED;
1658 }
1659 }
1660
1661 return orig_p;
1662}
1663
Masami Hiramatsu98616682008-04-28 02:14:28 -07001664/*
1665 * Unregister a kprobe without a scheduler synchronization.
1666 */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001667static int __unregister_kprobe_top(struct kprobe *p)
Keshavamurthy Anil Sdf019b12006-01-11 12:17:41 -08001668{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001669 struct kprobe *ap, *list_p;
Ananth N Mavinakayanahalli64f562c2005-05-05 16:15:42 -07001670
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001671 /* Disable kprobe. This will disarm it if needed. */
1672 ap = __disable_kprobe(p);
Jessica Yu297f9232018-01-10 00:51:24 +01001673 if (IS_ERR(ap))
1674 return PTR_ERR(ap);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001675
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001676 if (ap == p)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001677 /*
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001678 * This probe is an independent(and non-optimized) kprobe
1679 * (not an aggrprobe). Remove from the hash list.
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07001680 */
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001681 goto disarmed;
1682
1683 /* Following process expects this probe is an aggrprobe */
1684 WARN_ON(!kprobe_aggrprobe(ap));
1685
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001686 if (list_is_singular(&ap->list) && kprobe_disarmed(ap))
1687 /*
1688 * !disarmed could be happen if the probe is under delayed
1689 * unoptimizing.
1690 */
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001691 goto disarmed;
1692 else {
1693 /* If disabling probe has special handlers, update aggrprobe */
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001694 if (p->post_handler && !kprobe_gone(p)) {
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001695 list_for_each_entry_rcu(list_p, &ap->list, list) {
Masami Hiramatsu98616682008-04-28 02:14:28 -07001696 if ((list_p != p) && (list_p->post_handler))
1697 goto noclean;
1698 }
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001699 ap->post_handler = NULL;
Masami Hiramatsu98616682008-04-28 02:14:28 -07001700 }
1701noclean:
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001702 /*
1703 * Remove from the aggrprobe: this path will do nothing in
1704 * __unregister_kprobe_bottom().
1705 */
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001706 list_del_rcu(&p->list);
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001707 if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
1708 /*
1709 * Try to optimize this probe again, because post
1710 * handler may have been changed.
1711 */
1712 optimize_kprobe(ap);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001713 }
Masami Hiramatsu98616682008-04-28 02:14:28 -07001714 return 0;
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09001715
1716disarmed:
1717 hlist_del_rcu(&ap->hlist);
1718 return 0;
Masami Hiramatsu98616682008-04-28 02:14:28 -07001719}
Mao, Bibob3e55c72005-12-12 00:37:00 -08001720
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001721static void __unregister_kprobe_bottom(struct kprobe *p)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001722{
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001723 struct kprobe *ap;
Mao, Bibob3e55c72005-12-12 00:37:00 -08001724
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001725 if (list_empty(&p->list))
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001726 /* This is an independent kprobe */
Ananth N Mavinakayanahalli0498b632006-01-09 20:52:46 -08001727 arch_remove_kprobe(p);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001728 else if (list_is_singular(&p->list)) {
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001729 /* This is the last child of an aggrprobe */
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001730 ap = list_entry(p->list.next, struct kprobe, list);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08001731 list_del(&p->list);
Masami Hiramatsu6d8e40a2010-12-03 18:53:50 +09001732 free_aggr_kprobe(ap);
Anil S Keshavamurthy49a2a1b2006-01-09 20:52:43 -08001733 }
Masami Hiramatsu6274de42010-12-03 18:54:09 +09001734 /* Otherwise, do nothing. */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001735}
1736
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001737int register_kprobes(struct kprobe **kps, int num)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001738{
1739 int i, ret = 0;
1740
1741 if (num <= 0)
1742 return -EINVAL;
1743 for (i = 0; i < num; i++) {
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001744 ret = register_kprobe(kps[i]);
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -07001745 if (ret < 0) {
1746 if (i > 0)
1747 unregister_kprobes(kps, i);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001748 break;
1749 }
1750 }
1751 return ret;
1752}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001753EXPORT_SYMBOL_GPL(register_kprobes);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001754
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001755void unregister_kprobe(struct kprobe *p)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001756{
1757 unregister_kprobes(&p, 1);
1758}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001759EXPORT_SYMBOL_GPL(unregister_kprobe);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001760
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001761void unregister_kprobes(struct kprobe **kps, int num)
Masami Hiramatsu98616682008-04-28 02:14:28 -07001762{
1763 int i;
1764
1765 if (num <= 0)
1766 return;
1767 mutex_lock(&kprobe_mutex);
1768 for (i = 0; i < num; i++)
1769 if (__unregister_kprobe_top(kps[i]) < 0)
1770 kps[i]->addr = NULL;
1771 mutex_unlock(&kprobe_mutex);
1772
1773 synchronize_sched();
1774 for (i = 0; i < num; i++)
1775 if (kps[i]->addr)
1776 __unregister_kprobe_bottom(kps[i]);
1777}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001778EXPORT_SYMBOL_GPL(unregister_kprobes);
Masami Hiramatsu98616682008-04-28 02:14:28 -07001779
Naveen N. Rao5f6bee32017-03-08 22:34:15 +05301780int __weak kprobe_exceptions_notify(struct notifier_block *self,
1781 unsigned long val, void *data)
Naveen N. Raofc62d022017-02-08 01:24:14 +05301782{
1783 return NOTIFY_DONE;
1784}
Naveen N. Rao5f6bee32017-03-08 22:34:15 +05301785NOKPROBE_SYMBOL(kprobe_exceptions_notify);
Naveen N. Raofc62d022017-02-08 01:24:14 +05301786
Linus Torvalds1da177e2005-04-16 15:20:36 -07001787static struct notifier_block kprobe_exceptions_nb = {
1788 .notifier_call = kprobe_exceptions_notify,
Anil S Keshavamurthy3d5631e2006-06-26 00:25:28 -07001789 .priority = 0x7fffffff /* we need to be notified first */
1790};
1791
Michael Ellerman3d7e3382007-07-19 01:48:11 -07001792unsigned long __weak arch_deref_entry_point(void *entry)
1793{
1794 return (unsigned long)entry;
1795}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08001797#ifdef CONFIG_KRETPROBES
Adrian Bunke65cefe2006-02-03 03:03:42 -08001798/*
1799 * This kprobe pre_handler is registered with every kretprobe. When probe
1800 * hits it will set up the return probe.
1801 */
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001802static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
Adrian Bunke65cefe2006-02-03 03:03:42 -08001803{
1804 struct kretprobe *rp = container_of(p, struct kretprobe, kp);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001805 unsigned long hash, flags = 0;
1806 struct kretprobe_instance *ri;
Adrian Bunke65cefe2006-02-03 03:03:42 -08001807
Masami Hiramatsuf96f5672014-08-04 03:10:16 +00001808 /*
1809 * To avoid deadlocks, prohibit return probing in NMI contexts,
1810 * just skip the probe and increase the (inexact) 'nmissed'
1811 * statistical counter, so that the user is informed that
1812 * something happened:
1813 */
1814 if (unlikely(in_nmi())) {
1815 rp->nmissed++;
1816 return 0;
1817 }
1818
1819 /* TODO: consider to only swap the RA after the last pre_handler fired */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001820 hash = hash_ptr(current, KPROBE_HASH_BITS);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001821 raw_spin_lock_irqsave(&rp->lock, flags);
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001822 if (!hlist_empty(&rp->free_instances)) {
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001823 ri = hlist_entry(rp->free_instances.first,
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001824 struct kretprobe_instance, hlist);
1825 hlist_del(&ri->hlist);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001826 raw_spin_unlock_irqrestore(&rp->lock, flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001827
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001828 ri->rp = rp;
1829 ri->task = current;
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001830
Jiang Liu55ca6142012-02-03 15:37:16 -08001831 if (rp->entry_handler && rp->entry_handler(ri, regs)) {
1832 raw_spin_lock_irqsave(&rp->lock, flags);
1833 hlist_add_head(&ri->hlist, &rp->free_instances);
1834 raw_spin_unlock_irqrestore(&rp->lock, flags);
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001835 return 0;
Jiang Liu55ca6142012-02-03 15:37:16 -08001836 }
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001837
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001838 arch_prepare_kretprobe(ri, regs);
1839
1840 /* XXX(hch): why is there no hlist_move_head? */
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001841 INIT_HLIST_NODE(&ri->hlist);
1842 kretprobe_table_lock(hash, &flags);
1843 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1844 kretprobe_table_unlock(hash, &flags);
1845 } else {
Christoph Hellwig4c4308c2007-05-08 00:34:14 -07001846 rp->nmissed++;
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001847 raw_spin_unlock_irqrestore(&rp->lock, flags);
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001848 }
Adrian Bunke65cefe2006-02-03 03:03:42 -08001849 return 0;
1850}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001851NOKPROBE_SYMBOL(pre_handler_kretprobe);
Adrian Bunke65cefe2006-02-03 03:03:42 -08001852
Naveen N. Rao659b9572017-07-07 22:37:24 +05301853bool __weak arch_kprobe_on_func_entry(unsigned long offset)
Naveen N. Rao90ec5e82017-02-22 19:23:37 +05301854{
1855 return !offset;
1856}
1857
Naveen N. Rao659b9572017-07-07 22:37:24 +05301858bool kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301859{
1860 kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset);
1861
1862 if (IS_ERR(kp_addr))
1863 return false;
1864
1865 if (!kallsyms_lookup_size_offset((unsigned long)kp_addr, NULL, &offset) ||
Naveen N. Rao659b9572017-07-07 22:37:24 +05301866 !arch_kprobe_on_func_entry(offset))
Naveen N. Rao1d585e72017-03-08 13:56:06 +05301867 return false;
1868
1869 return true;
1870}
1871
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001872int register_kretprobe(struct kretprobe *rp)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001873{
1874 int ret = 0;
1875 struct kretprobe_instance *inst;
1876 int i;
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001877 void *addr;
Naveen N. Rao90ec5e82017-02-22 19:23:37 +05301878
Naveen N. Rao659b9572017-07-07 22:37:24 +05301879 if (!kprobe_on_func_entry(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset))
Naveen N. Rao90ec5e82017-02-22 19:23:37 +05301880 return -EINVAL;
Masami Hiramatsuf438d912007-10-16 01:27:49 -07001881
1882 if (kretprobe_blacklist_size) {
Masami Hiramatsub2a5cd62008-03-04 14:29:44 -08001883 addr = kprobe_addr(&rp->kp);
Masami Hiramatsubc81d482011-06-27 16:26:50 +09001884 if (IS_ERR(addr))
1885 return PTR_ERR(addr);
Masami Hiramatsuf438d912007-10-16 01:27:49 -07001886
1887 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1888 if (kretprobe_blacklist[i].addr == addr)
1889 return -EINVAL;
1890 }
1891 }
Hien Nguyenb94cce92005-06-23 00:09:19 -07001892
1893 rp->kp.pre_handler = pre_handler_kretprobe;
Ananth N Mavinakayanahalli7522a842006-04-20 02:43:11 -07001894 rp->kp.post_handler = NULL;
1895 rp->kp.fault_handler = NULL;
Hien Nguyenb94cce92005-06-23 00:09:19 -07001896
1897 /* Pre-allocate memory for max kretprobe instances */
1898 if (rp->maxactive <= 0) {
1899#ifdef CONFIG_PREEMPT
Heiko Carstensc2ef6662009-12-21 13:02:24 +01001900 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
Hien Nguyenb94cce92005-06-23 00:09:19 -07001901#else
Ananth N Mavinakayanahalli4dae5602009-10-30 19:23:10 +05301902 rp->maxactive = num_possible_cpus();
Hien Nguyenb94cce92005-06-23 00:09:19 -07001903#endif
1904 }
Thomas Gleixnerec4846082009-07-25 16:09:17 +02001905 raw_spin_lock_init(&rp->lock);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001906 INIT_HLIST_HEAD(&rp->free_instances);
1907 for (i = 0; i < rp->maxactive; i++) {
Abhishek Sagarf47cd9b2008-02-06 01:38:22 -08001908 inst = kmalloc(sizeof(struct kretprobe_instance) +
1909 rp->data_size, GFP_KERNEL);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001910 if (inst == NULL) {
1911 free_rp_inst(rp);
1912 return -ENOMEM;
1913 }
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07001914 INIT_HLIST_NODE(&inst->hlist);
1915 hlist_add_head(&inst->hlist, &rp->free_instances);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001916 }
1917
1918 rp->nmissed = 0;
1919 /* Establish function entry probe point */
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001920 ret = register_kprobe(&rp->kp);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001921 if (ret != 0)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001922 free_rp_inst(rp);
1923 return ret;
1924}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001925EXPORT_SYMBOL_GPL(register_kretprobe);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001926
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001927int register_kretprobes(struct kretprobe **rps, int num)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001928{
1929 int ret = 0, i;
1930
1931 if (num <= 0)
1932 return -EINVAL;
1933 for (i = 0; i < num; i++) {
Masami Hiramatsu49ad2fd2009-01-06 14:41:53 -08001934 ret = register_kretprobe(rps[i]);
Masami Hiramatsu67dddaa2008-06-12 15:21:35 -07001935 if (ret < 0) {
1936 if (i > 0)
1937 unregister_kretprobes(rps, i);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001938 break;
1939 }
1940 }
1941 return ret;
1942}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001943EXPORT_SYMBOL_GPL(register_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001944
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001945void unregister_kretprobe(struct kretprobe *rp)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001946{
1947 unregister_kretprobes(&rp, 1);
1948}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001949EXPORT_SYMBOL_GPL(unregister_kretprobe);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001950
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001951void unregister_kretprobes(struct kretprobe **rps, int num)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001952{
1953 int i;
1954
1955 if (num <= 0)
1956 return;
1957 mutex_lock(&kprobe_mutex);
1958 for (i = 0; i < num; i++)
1959 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1960 rps[i]->kp.addr = NULL;
1961 mutex_unlock(&kprobe_mutex);
1962
1963 synchronize_sched();
1964 for (i = 0; i < num; i++) {
1965 if (rps[i]->kp.addr) {
1966 __unregister_kprobe_bottom(&rps[i]->kp);
1967 cleanup_rp_inst(rps[i]);
1968 }
1969 }
1970}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001971EXPORT_SYMBOL_GPL(unregister_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001972
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08001973#else /* CONFIG_KRETPROBES */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001974int register_kretprobe(struct kretprobe *rp)
Hien Nguyenb94cce92005-06-23 00:09:19 -07001975{
1976 return -ENOSYS;
1977}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001978EXPORT_SYMBOL_GPL(register_kretprobe);
Hien Nguyenb94cce92005-06-23 00:09:19 -07001979
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001980int register_kretprobes(struct kretprobe **rps, int num)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001981{
1982 return -ENOSYS;
1983}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001984EXPORT_SYMBOL_GPL(register_kretprobes);
1985
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001986void unregister_kretprobe(struct kretprobe *rp)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001987{
1988}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001989EXPORT_SYMBOL_GPL(unregister_kretprobe);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001990
Masami Hiramatsu55479f62014-04-17 17:17:54 +09001991void unregister_kretprobes(struct kretprobe **rps, int num)
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001992{
1993}
Masami Hiramatsu99081ab2009-04-06 19:00:59 -07001994EXPORT_SYMBOL_GPL(unregister_kretprobes);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07001995
Masami Hiramatsu820aede2014-04-17 17:18:21 +09001996static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
Srinivasa Ds346fd592007-02-20 13:57:54 -08001997{
1998 return 0;
1999}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09002000NOKPROBE_SYMBOL(pre_handler_kretprobe);
Masami Hiramatsu4a296e02008-04-28 02:14:29 -07002001
Ananth N Mavinakayanahalli9edddaa2008-03-04 14:28:37 -08002002#endif /* CONFIG_KRETPROBES */
Hien Nguyenb94cce92005-06-23 00:09:19 -07002003
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002004/* Set the kprobe gone and remove its instruction buffer. */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002005static void kill_kprobe(struct kprobe *p)
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002006{
2007 struct kprobe *kp;
Masami Hiramatsude5bd882009-04-06 19:01:02 -07002008
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002009 p->flags |= KPROBE_FLAG_GONE;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002010 if (kprobe_aggrprobe(p)) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002011 /*
2012 * If this is an aggr_kprobe, we have to list all the
2013 * chained probes and mark them GONE.
2014 */
2015 list_for_each_entry_rcu(kp, &p->list, list)
2016 kp->flags |= KPROBE_FLAG_GONE;
2017 p->post_handler = NULL;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002018 kill_optimized_kprobe(p);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002019 }
2020 /*
2021 * Here, we can remove insn_slot safely, because no thread calls
2022 * the original probed function (which will be freed soon) any more.
2023 */
2024 arch_remove_kprobe(p);
2025}
2026
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002027/* Disable one kprobe */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002028int disable_kprobe(struct kprobe *kp)
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002029{
2030 int ret = 0;
Jessica Yu297f9232018-01-10 00:51:24 +01002031 struct kprobe *p;
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002032
2033 mutex_lock(&kprobe_mutex);
2034
Masami Hiramatsu6f0f1dd2010-12-03 18:53:57 +09002035 /* Disable this kprobe */
Jessica Yu297f9232018-01-10 00:51:24 +01002036 p = __disable_kprobe(kp);
2037 if (IS_ERR(p))
2038 ret = PTR_ERR(p);
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002039
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002040 mutex_unlock(&kprobe_mutex);
2041 return ret;
2042}
2043EXPORT_SYMBOL_GPL(disable_kprobe);
2044
2045/* Enable one kprobe */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002046int enable_kprobe(struct kprobe *kp)
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002047{
2048 int ret = 0;
2049 struct kprobe *p;
2050
2051 mutex_lock(&kprobe_mutex);
2052
2053 /* Check whether specified probe is valid. */
2054 p = __get_valid_kprobe(kp);
2055 if (unlikely(p == NULL)) {
2056 ret = -EINVAL;
2057 goto out;
2058 }
2059
2060 if (kprobe_gone(kp)) {
2061 /* This kprobe has gone, we couldn't enable it. */
2062 ret = -EINVAL;
2063 goto out;
2064 }
2065
2066 if (p != kp)
2067 kp->flags &= ~KPROBE_FLAG_DISABLED;
2068
2069 if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2070 p->flags &= ~KPROBE_FLAG_DISABLED;
Jessica Yu12310e342018-01-10 00:51:23 +01002071 ret = arm_kprobe(p);
2072 if (ret)
2073 p->flags |= KPROBE_FLAG_DISABLED;
Masami Hiramatsuc0614822010-04-27 18:33:12 -04002074 }
2075out:
2076 mutex_unlock(&kprobe_mutex);
2077 return ret;
2078}
2079EXPORT_SYMBOL_GPL(enable_kprobe);
2080
Masami Hiramatsu44585152018-04-28 21:36:33 +09002081/* Caller must NOT call this in usual path. This is only for critical case */
Masami Hiramatsu820aede2014-04-17 17:18:21 +09002082void dump_kprobe(struct kprobe *kp)
Frederic Weisbecker24851d22009-08-26 23:38:30 +02002083{
Masami Hiramatsu44585152018-04-28 21:36:33 +09002084 pr_err("Dumping kprobe:\n");
2085 pr_err("Name: %s\nOffset: %x\nAddress: %pS\n",
2086 kp->symbol_name, kp->offset, kp->addr);
Frederic Weisbecker24851d22009-08-26 23:38:30 +02002087}
Masami Hiramatsu820aede2014-04-17 17:18:21 +09002088NOKPROBE_SYMBOL(dump_kprobe);
Frederic Weisbecker24851d22009-08-26 23:38:30 +02002089
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002090/*
2091 * Lookup and populate the kprobe_blacklist.
2092 *
2093 * Unlike the kretprobe blacklist, we'll need to determine
2094 * the range of addresses that belong to the said functions,
2095 * since a kprobe need not necessarily be at the beginning
2096 * of a function.
2097 */
2098static int __init populate_kprobe_blacklist(unsigned long *start,
2099 unsigned long *end)
2100{
2101 unsigned long *iter;
2102 struct kprobe_blacklist_entry *ent;
Masami Hiramatsud81b4252014-07-17 11:44:11 +00002103 unsigned long entry, offset = 0, size = 0;
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002104
2105 for (iter = start; iter < end; iter++) {
Masami Hiramatsud81b4252014-07-17 11:44:11 +00002106 entry = arch_deref_entry_point((void *)*iter);
2107
2108 if (!kernel_text_address(entry) ||
Masami Hiramatsu44585152018-04-28 21:36:33 +09002109 !kallsyms_lookup_size_offset(entry, &size, &offset))
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002110 continue;
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002111
2112 ent = kmalloc(sizeof(*ent), GFP_KERNEL);
2113 if (!ent)
2114 return -ENOMEM;
Masami Hiramatsud81b4252014-07-17 11:44:11 +00002115 ent->start_addr = entry;
2116 ent->end_addr = entry + size;
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002117 INIT_LIST_HEAD(&ent->list);
2118 list_add_tail(&ent->list, &kprobe_blacklist);
2119 }
2120 return 0;
2121}
2122
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002123/* Module notifier call back, checking kprobes on the module */
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002124static int kprobes_module_callback(struct notifier_block *nb,
2125 unsigned long val, void *data)
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002126{
2127 struct module *mod = data;
2128 struct hlist_head *head;
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002129 struct kprobe *p;
2130 unsigned int i;
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002131 int checkcore = (val == MODULE_STATE_GOING);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002132
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002133 if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002134 return NOTIFY_DONE;
2135
2136 /*
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002137 * When MODULE_STATE_GOING was notified, both of module .text and
2138 * .init.text sections would be freed. When MODULE_STATE_LIVE was
2139 * notified, only .init.text section would be freed. We need to
2140 * disable kprobes which have been inserted in the sections.
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002141 */
2142 mutex_lock(&kprobe_mutex);
2143 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2144 head = &kprobe_table[i];
Sasha Levinb67bfe02013-02-27 17:06:00 -08002145 hlist_for_each_entry_rcu(p, head, hlist)
Masami Hiramatsuf24659d2009-01-06 14:41:55 -08002146 if (within_module_init((unsigned long)p->addr, mod) ||
2147 (checkcore &&
2148 within_module_core((unsigned long)p->addr, mod))) {
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002149 /*
2150 * The vaddr this probe is installed will soon
2151 * be vfreed buy not synced to disk. Hence,
2152 * disarming the breakpoint isn't needed.
Steven Rostedt (VMware)545a0282017-05-16 14:58:35 -04002153 *
2154 * Note, this will also move any optimized probes
2155 * that are pending to be removed from their
2156 * corresponding lists to the freeing_list and
2157 * will not be touched by the delayed
2158 * kprobe_optimizer work handler.
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002159 */
2160 kill_kprobe(p);
2161 }
2162 }
2163 mutex_unlock(&kprobe_mutex);
2164 return NOTIFY_DONE;
2165}
2166
2167static struct notifier_block kprobe_module_nb = {
2168 .notifier_call = kprobes_module_callback,
2169 .priority = 0
2170};
2171
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002172/* Markers of _kprobe_blacklist section */
2173extern unsigned long __start_kprobe_blacklist[];
2174extern unsigned long __stop_kprobe_blacklist[];
2175
Linus Torvalds1da177e2005-04-16 15:20:36 -07002176static int __init init_kprobes(void)
2177{
2178 int i, err = 0;
2179
2180 /* FIXME allocate the probe table, currently defined statically */
2181 /* initialize all list heads */
Hien Nguyenb94cce92005-06-23 00:09:19 -07002182 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002183 INIT_HLIST_HEAD(&kprobe_table[i]);
Hien Nguyenb94cce92005-06-23 00:09:19 -07002184 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
Thomas Gleixnerec4846082009-07-25 16:09:17 +02002185 raw_spin_lock_init(&(kretprobe_table_locks[i].lock));
Hien Nguyenb94cce92005-06-23 00:09:19 -07002186 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002187
Masami Hiramatsu376e2422014-04-17 17:17:05 +09002188 err = populate_kprobe_blacklist(__start_kprobe_blacklist,
2189 __stop_kprobe_blacklist);
2190 if (err) {
2191 pr_err("kprobes: failed to populate blacklist: %d\n", err);
2192 pr_err("Please take care of using kprobes.\n");
Srinivasa Ds3d8d9962008-04-28 02:14:26 -07002193 }
2194
Masami Hiramatsuf438d912007-10-16 01:27:49 -07002195 if (kretprobe_blacklist_size) {
2196 /* lookup the function address from its name */
2197 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
Naveen N. Rao49e0b462017-04-19 18:21:00 +05302198 kretprobe_blacklist[i].addr =
Naveen N. Rao290e3072017-04-19 18:21:01 +05302199 kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
Masami Hiramatsuf438d912007-10-16 01:27:49 -07002200 if (!kretprobe_blacklist[i].addr)
2201 printk("kretprobe: lookup failed: %s\n",
2202 kretprobe_blacklist[i].name);
2203 }
2204 }
2205
Masami Hiramatsub2be84d2010-02-25 08:34:15 -05002206#if defined(CONFIG_OPTPROBES)
2207#if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002208 /* Init kprobe_optinsn_slots */
2209 kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
2210#endif
Masami Hiramatsub2be84d2010-02-25 08:34:15 -05002211 /* By default, kprobes can be optimized */
2212 kprobes_allow_optimization = true;
2213#endif
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002214
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002215 /* By default, kprobes are armed */
2216 kprobes_all_disarmed = false;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002217
Rusty Lynch67729262005-07-05 18:54:50 -07002218 err = arch_init_kprobes();
Rusty Lynch802eae72005-06-27 15:17:08 -07002219 if (!err)
2220 err = register_die_notifier(&kprobe_exceptions_nb);
Masami Hiramatsue8386a02009-01-06 14:41:52 -08002221 if (!err)
2222 err = register_module_notifier(&kprobe_module_nb);
2223
Srinivasa D Sef53d9c2008-07-25 01:46:04 -07002224 kprobes_initialized = (err == 0);
Rusty Lynch802eae72005-06-27 15:17:08 -07002225
Ananth N Mavinakayanahalli8c1c9352008-01-30 13:32:53 +01002226 if (!err)
2227 init_test_probes();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002228 return err;
2229}
2230
Srinivasa Ds346fd592007-02-20 13:57:54 -08002231#ifdef CONFIG_DEBUG_FS
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002232static void report_probe(struct seq_file *pi, struct kprobe *p,
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002233 const char *sym, int offset, char *modname, struct kprobe *pp)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002234{
2235 char *kprobe_type;
Masami Hiramatsu81365a92018-04-28 21:36:02 +09002236 void *addr = p->addr;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002237
2238 if (p->pre_handler == pre_handler_kretprobe)
2239 kprobe_type = "r";
Srinivasa Ds346fd592007-02-20 13:57:54 -08002240 else
2241 kprobe_type = "k";
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002242
Masami Hiramatsu81365a92018-04-28 21:36:02 +09002243 if (!kallsyms_show_value())
2244 addr = NULL;
2245
Srinivasa Ds346fd592007-02-20 13:57:54 -08002246 if (sym)
Masami Hiramatsu81365a92018-04-28 21:36:02 +09002247 seq_printf(pi, "%px %s %s+0x%x %s ",
2248 addr, kprobe_type, sym, offset,
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002249 (modname ? modname : " "));
Masami Hiramatsu81365a92018-04-28 21:36:02 +09002250 else /* try to use %pS */
2251 seq_printf(pi, "%px %s %pS ",
2252 addr, kprobe_type, p->addr);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002253
2254 if (!pp)
2255 pp = p;
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09002256 seq_printf(pi, "%s%s%s%s\n",
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002257 (kprobe_gone(p) ? "[GONE]" : ""),
2258 ((kprobe_disabled(p) && !kprobe_gone(p)) ? "[DISABLED]" : ""),
Masami Hiramatsuae6aa162012-06-05 19:28:32 +09002259 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
2260 (kprobe_ftrace(pp) ? "[FTRACE]" : ""));
Srinivasa Ds346fd592007-02-20 13:57:54 -08002261}
2262
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002263static void *kprobe_seq_start(struct seq_file *f, loff_t *pos)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002264{
2265 return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
2266}
2267
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002268static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002269{
2270 (*pos)++;
2271 if (*pos >= KPROBE_TABLE_SIZE)
2272 return NULL;
2273 return pos;
2274}
2275
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002276static void kprobe_seq_stop(struct seq_file *f, void *v)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002277{
2278 /* Nothing to do */
2279}
2280
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002281static int show_kprobe_addr(struct seq_file *pi, void *v)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002282{
2283 struct hlist_head *head;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002284 struct kprobe *p, *kp;
2285 const char *sym = NULL;
2286 unsigned int i = *(loff_t *) v;
Alexey Dobriyanffb45122007-05-08 00:28:41 -07002287 unsigned long offset = 0;
Joe Marioab767862013-11-12 15:10:23 -08002288 char *modname, namebuf[KSYM_NAME_LEN];
Srinivasa Ds346fd592007-02-20 13:57:54 -08002289
2290 head = &kprobe_table[i];
2291 preempt_disable();
Sasha Levinb67bfe02013-02-27 17:06:00 -08002292 hlist_for_each_entry_rcu(p, head, hlist) {
Alexey Dobriyanffb45122007-05-08 00:28:41 -07002293 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
Srinivasa Ds346fd592007-02-20 13:57:54 -08002294 &offset, &modname, namebuf);
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002295 if (kprobe_aggrprobe(p)) {
Srinivasa Ds346fd592007-02-20 13:57:54 -08002296 list_for_each_entry_rcu(kp, &p->list, list)
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002297 report_probe(pi, kp, sym, offset, modname, p);
Srinivasa Ds346fd592007-02-20 13:57:54 -08002298 } else
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002299 report_probe(pi, p, sym, offset, modname, NULL);
Srinivasa Ds346fd592007-02-20 13:57:54 -08002300 }
2301 preempt_enable();
2302 return 0;
2303}
2304
James Morris88e9d342009-09-22 16:43:43 -07002305static const struct seq_operations kprobes_seq_ops = {
Srinivasa Ds346fd592007-02-20 13:57:54 -08002306 .start = kprobe_seq_start,
2307 .next = kprobe_seq_next,
2308 .stop = kprobe_seq_stop,
2309 .show = show_kprobe_addr
2310};
2311
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002312static int kprobes_open(struct inode *inode, struct file *filp)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002313{
2314 return seq_open(filp, &kprobes_seq_ops);
2315}
2316
Alexey Dobriyan828c0952009-10-01 15:43:56 -07002317static const struct file_operations debugfs_kprobes_operations = {
Srinivasa Ds346fd592007-02-20 13:57:54 -08002318 .open = kprobes_open,
2319 .read = seq_read,
2320 .llseek = seq_lseek,
2321 .release = seq_release,
2322};
2323
Masami Hiramatsu63724742014-04-17 17:18:49 +09002324/* kprobes/blacklist -- shows which functions can not be probed */
2325static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
2326{
2327 return seq_list_start(&kprobe_blacklist, *pos);
2328}
2329
2330static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos)
2331{
2332 return seq_list_next(v, &kprobe_blacklist, pos);
2333}
2334
2335static int kprobe_blacklist_seq_show(struct seq_file *m, void *v)
2336{
2337 struct kprobe_blacklist_entry *ent =
2338 list_entry(v, struct kprobe_blacklist_entry, list);
2339
Masami Hiramatsuffb9bd62018-04-28 21:35:32 +09002340 /*
2341 * If /proc/kallsyms is not showing kernel address, we won't
2342 * show them here either.
2343 */
2344 if (!kallsyms_show_value())
2345 seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL,
2346 (void *)ent->start_addr);
2347 else
2348 seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr,
2349 (void *)ent->end_addr, (void *)ent->start_addr);
Masami Hiramatsu63724742014-04-17 17:18:49 +09002350 return 0;
2351}
2352
2353static const struct seq_operations kprobe_blacklist_seq_ops = {
2354 .start = kprobe_blacklist_seq_start,
2355 .next = kprobe_blacklist_seq_next,
2356 .stop = kprobe_seq_stop, /* Reuse void function */
2357 .show = kprobe_blacklist_seq_show,
2358};
2359
2360static int kprobe_blacklist_open(struct inode *inode, struct file *filp)
2361{
2362 return seq_open(filp, &kprobe_blacklist_seq_ops);
2363}
2364
2365static const struct file_operations debugfs_kprobe_blacklist_ops = {
2366 .open = kprobe_blacklist_open,
2367 .read = seq_read,
2368 .llseek = seq_lseek,
2369 .release = seq_release,
2370};
2371
Jessica Yu12310e342018-01-10 00:51:23 +01002372static int arm_all_kprobes(void)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002373{
2374 struct hlist_head *head;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002375 struct kprobe *p;
Jessica Yu12310e342018-01-10 00:51:23 +01002376 unsigned int i, total = 0, errors = 0;
2377 int err, ret = 0;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002378
2379 mutex_lock(&kprobe_mutex);
2380
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002381 /* If kprobes are armed, just return */
2382 if (!kprobes_all_disarmed)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002383 goto already_enabled;
2384
Wang Nan977ad482015-02-13 14:40:24 -08002385 /*
2386 * optimize_kprobe() called by arm_kprobe() checks
2387 * kprobes_all_disarmed, so set kprobes_all_disarmed before
2388 * arm_kprobe.
2389 */
2390 kprobes_all_disarmed = false;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002391 /* Arming kprobes doesn't optimize kprobe itself */
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002392 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2393 head = &kprobe_table[i];
Jessica Yu12310e342018-01-10 00:51:23 +01002394 /* Arm all kprobes on a best-effort basis */
2395 hlist_for_each_entry_rcu(p, head, hlist) {
2396 if (!kprobe_disabled(p)) {
2397 err = arm_kprobe(p);
2398 if (err) {
2399 errors++;
2400 ret = err;
2401 }
2402 total++;
2403 }
2404 }
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002405 }
2406
Jessica Yu12310e342018-01-10 00:51:23 +01002407 if (errors)
2408 pr_warn("Kprobes globally enabled, but failed to arm %d out of %d probes\n",
2409 errors, total);
2410 else
2411 pr_info("Kprobes globally enabled\n");
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002412
2413already_enabled:
2414 mutex_unlock(&kprobe_mutex);
Jessica Yu12310e342018-01-10 00:51:23 +01002415 return ret;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002416}
2417
Jessica Yu297f9232018-01-10 00:51:24 +01002418static int disarm_all_kprobes(void)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002419{
2420 struct hlist_head *head;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002421 struct kprobe *p;
Jessica Yu297f9232018-01-10 00:51:24 +01002422 unsigned int i, total = 0, errors = 0;
2423 int err, ret = 0;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002424
2425 mutex_lock(&kprobe_mutex);
2426
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002427 /* If kprobes are already disarmed, just return */
Masami Hiramatsu6274de42010-12-03 18:54:09 +09002428 if (kprobes_all_disarmed) {
2429 mutex_unlock(&kprobe_mutex);
Jessica Yu297f9232018-01-10 00:51:24 +01002430 return 0;
Masami Hiramatsu6274de42010-12-03 18:54:09 +09002431 }
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002432
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002433 kprobes_all_disarmed = true;
Masami Hiramatsuafd66252010-02-25 08:34:07 -05002434
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002435 for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
2436 head = &kprobe_table[i];
Jessica Yu297f9232018-01-10 00:51:24 +01002437 /* Disarm all kprobes on a best-effort basis */
Sasha Levinb67bfe02013-02-27 17:06:00 -08002438 hlist_for_each_entry_rcu(p, head, hlist) {
Jessica Yu297f9232018-01-10 00:51:24 +01002439 if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
2440 err = disarm_kprobe(p, false);
2441 if (err) {
2442 errors++;
2443 ret = err;
2444 }
2445 total++;
2446 }
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002447 }
2448 }
Jessica Yu297f9232018-01-10 00:51:24 +01002449
2450 if (errors)
2451 pr_warn("Kprobes globally disabled, but failed to disarm %d out of %d probes\n",
2452 errors, total);
2453 else
2454 pr_info("Kprobes globally disabled\n");
2455
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002456 mutex_unlock(&kprobe_mutex);
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002457
Masami Hiramatsu6274de42010-12-03 18:54:09 +09002458 /* Wait for disarming all kprobes by optimizer */
2459 wait_for_kprobe_optimizer();
Jessica Yu297f9232018-01-10 00:51:24 +01002460
2461 return ret;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002462}
2463
2464/*
2465 * XXX: The debugfs bool file interface doesn't allow for callbacks
2466 * when the bool state is switched. We can reuse that facility when
2467 * available
2468 */
2469static ssize_t read_enabled_file_bool(struct file *file,
2470 char __user *user_buf, size_t count, loff_t *ppos)
2471{
2472 char buf[3];
2473
Masami Hiramatsue579abe2009-04-06 19:01:01 -07002474 if (!kprobes_all_disarmed)
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002475 buf[0] = '1';
2476 else
2477 buf[0] = '0';
2478 buf[1] = '\n';
2479 buf[2] = 0x00;
2480 return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
2481}
2482
2483static ssize_t write_enabled_file_bool(struct file *file,
2484 const char __user *user_buf, size_t count, loff_t *ppos)
2485{
2486 char buf[32];
Stephen Boydefeb1562012-01-12 17:17:11 -08002487 size_t buf_size;
Jessica Yu12310e342018-01-10 00:51:23 +01002488 int ret = 0;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002489
2490 buf_size = min(count, (sizeof(buf)-1));
2491 if (copy_from_user(buf, user_buf, buf_size))
2492 return -EFAULT;
2493
Mathias Krause10fb46d2013-07-03 15:05:39 -07002494 buf[buf_size] = '\0';
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002495 switch (buf[0]) {
2496 case 'y':
2497 case 'Y':
2498 case '1':
Jessica Yu12310e342018-01-10 00:51:23 +01002499 ret = arm_all_kprobes();
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002500 break;
2501 case 'n':
2502 case 'N':
2503 case '0':
Jessica Yu297f9232018-01-10 00:51:24 +01002504 ret = disarm_all_kprobes();
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002505 break;
Mathias Krause10fb46d2013-07-03 15:05:39 -07002506 default:
2507 return -EINVAL;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002508 }
2509
Jessica Yu12310e342018-01-10 00:51:23 +01002510 if (ret)
2511 return ret;
2512
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002513 return count;
2514}
2515
Alexey Dobriyan828c0952009-10-01 15:43:56 -07002516static const struct file_operations fops_kp = {
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002517 .read = read_enabled_file_bool,
2518 .write = write_enabled_file_bool,
Arnd Bergmann6038f372010-08-15 18:52:59 +02002519 .llseek = default_llseek,
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002520};
2521
Masami Hiramatsu55479f62014-04-17 17:17:54 +09002522static int __init debugfs_kprobe_init(void)
Srinivasa Ds346fd592007-02-20 13:57:54 -08002523{
2524 struct dentry *dir, *file;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002525 unsigned int value = 1;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002526
2527 dir = debugfs_create_dir("kprobes", NULL);
2528 if (!dir)
2529 return -ENOMEM;
2530
Masami Hiramatsuf2a3ab32018-04-28 21:35:01 +09002531 file = debugfs_create_file("list", 0400, dir, NULL,
Srinivasa Ds346fd592007-02-20 13:57:54 -08002532 &debugfs_kprobes_operations);
Masami Hiramatsu63724742014-04-17 17:18:49 +09002533 if (!file)
2534 goto error;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002535
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002536 file = debugfs_create_file("enabled", 0600, dir,
2537 &value, &fops_kp);
Masami Hiramatsu63724742014-04-17 17:18:49 +09002538 if (!file)
2539 goto error;
2540
Masami Hiramatsuf2a3ab32018-04-28 21:35:01 +09002541 file = debugfs_create_file("blacklist", 0400, dir, NULL,
Masami Hiramatsu63724742014-04-17 17:18:49 +09002542 &debugfs_kprobe_blacklist_ops);
2543 if (!file)
2544 goto error;
Ananth N Mavinakayanahallibf8f6e5b2007-05-08 00:34:16 -07002545
Srinivasa Ds346fd592007-02-20 13:57:54 -08002546 return 0;
Masami Hiramatsu63724742014-04-17 17:18:49 +09002547
2548error:
2549 debugfs_remove(dir);
2550 return -ENOMEM;
Srinivasa Ds346fd592007-02-20 13:57:54 -08002551}
2552
2553late_initcall(debugfs_kprobe_init);
2554#endif /* CONFIG_DEBUG_FS */
2555
2556module_init(init_kprobes);