kprobes: No need to unlock kprobe_insn_mutex
[linux-2.6.git] / kernel / kprobes.c
1 /*
2  *  Kernel Probes (KProbes)
3  *  kernel/kprobes.c
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18  *
19  * Copyright (C) IBM Corporation, 2002, 2004
20  *
21  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22  *              Probes initial implementation (includes suggestions from
23  *              Rusty Russell).
24  * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25  *              hlists and exceptions notifier as suggested by Andi Kleen.
26  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27  *              interface to access function arguments.
28  * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29  *              exceptions notifier to be first on the priority list.
30  * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31  *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32  *              <prasanna@in.ibm.com> added function-return probes.
33  */
34 #include <linux/kprobes.h>
35 #include <linux/hash.h>
36 #include <linux/init.h>
37 #include <linux/slab.h>
38 #include <linux/stddef.h>
39 #include <linux/module.h>
40 #include <linux/moduleloader.h>
41 #include <linux/kallsyms.h>
42 #include <linux/freezer.h>
43 #include <linux/seq_file.h>
44 #include <linux/debugfs.h>
45 #include <linux/kdebug.h>
46 #include <linux/memory.h>
47
48 #include <asm-generic/sections.h>
49 #include <asm/cacheflush.h>
50 #include <asm/errno.h>
51 #include <asm/uaccess.h>
52
53 #define KPROBE_HASH_BITS 6
54 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
55
56
57 /*
58  * Some oddball architectures like 64bit powerpc have function descriptors
59  * so this must be overridable.
60  */
61 #ifndef kprobe_lookup_name
62 #define kprobe_lookup_name(name, addr) \
63         addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
64 #endif
65
66 static int kprobes_initialized;
67 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
68 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
69
70 /* NOTE: change this value only with kprobe_mutex held */
71 static bool kprobes_all_disarmed;
72
73 static DEFINE_MUTEX(kprobe_mutex);      /* Protects kprobe_table */
74 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
75 static struct {
76         spinlock_t lock ____cacheline_aligned_in_smp;
77 } kretprobe_table_locks[KPROBE_TABLE_SIZE];
78
79 static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
80 {
81         return &(kretprobe_table_locks[hash].lock);
82 }
83
84 /*
85  * Normally, functions that we'd want to prohibit kprobes in, are marked
86  * __kprobes. But, there are cases where such functions already belong to
87  * a different section (__sched for preempt_schedule)
88  *
89  * For such cases, we now have a blacklist
90  */
91 static struct kprobe_blackpoint kprobe_blacklist[] = {
92         {"preempt_schedule",},
93         {NULL}    /* Terminator */
94 };
95
96 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
97 /*
98  * kprobe->ainsn.insn points to the copy of the instruction to be
99  * single-stepped. x86_64, POWER4 and above have no-exec support and
100  * stepping on the instruction on a vmalloced/kmalloced/data page
101  * is a recipe for disaster
102  */
103 #define INSNS_PER_PAGE  (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
104
105 struct kprobe_insn_page {
106         struct hlist_node hlist;
107         kprobe_opcode_t *insns;         /* Page of instruction slots */
108         char slot_used[INSNS_PER_PAGE];
109         int nused;
110         int ngarbage;
111 };
112
113 enum kprobe_slot_state {
114         SLOT_CLEAN = 0,
115         SLOT_DIRTY = 1,
116         SLOT_USED = 2,
117 };
118
119 static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_pages */
120 static struct hlist_head kprobe_insn_pages;
121 static int kprobe_garbage_slots;
122 static int collect_garbage_slots(void);
123
124 static int __kprobes check_safety(void)
125 {
126         int ret = 0;
127 #if defined(CONFIG_PREEMPT) && defined(CONFIG_FREEZER)
128         ret = freeze_processes();
129         if (ret == 0) {
130                 struct task_struct *p, *q;
131                 do_each_thread(p, q) {
132                         if (p != current && p->state == TASK_RUNNING &&
133                             p->pid != 0) {
134                                 printk("Check failed: %s is running\n",p->comm);
135                                 ret = -1;
136                                 goto loop_end;
137                         }
138                 } while_each_thread(p, q);
139         }
140 loop_end:
141         thaw_processes();
142 #else
143         synchronize_sched();
144 #endif
145         return ret;
146 }
147
148 /**
149  * __get_insn_slot() - Find a slot on an executable page for an instruction.
150  * We allocate an executable page if there's no room on existing ones.
151  */
152 static kprobe_opcode_t __kprobes *__get_insn_slot(void)
153 {
154         struct kprobe_insn_page *kip;
155         struct hlist_node *pos;
156
157  retry:
158         hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
159                 if (kip->nused < INSNS_PER_PAGE) {
160                         int i;
161                         for (i = 0; i < INSNS_PER_PAGE; i++) {
162                                 if (kip->slot_used[i] == SLOT_CLEAN) {
163                                         kip->slot_used[i] = SLOT_USED;
164                                         kip->nused++;
165                                         return kip->insns + (i * MAX_INSN_SIZE);
166                                 }
167                         }
168                         /* Surprise!  No unused slots.  Fix kip->nused. */
169                         kip->nused = INSNS_PER_PAGE;
170                 }
171         }
172
173         /* If there are any garbage slots, collect it and try again. */
174         if (kprobe_garbage_slots && collect_garbage_slots() == 0) {
175                 goto retry;
176         }
177         /* All out of space.  Need to allocate a new page. Use slot 0. */
178         kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
179         if (!kip)
180                 return NULL;
181
182         /*
183          * Use module_alloc so this page is within +/- 2GB of where the
184          * kernel image and loaded module images reside. This is required
185          * so x86_64 can correctly handle the %rip-relative fixups.
186          */
187         kip->insns = module_alloc(PAGE_SIZE);
188         if (!kip->insns) {
189                 kfree(kip);
190                 return NULL;
191         }
192         INIT_HLIST_NODE(&kip->hlist);
193         hlist_add_head(&kip->hlist, &kprobe_insn_pages);
194         memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE);
195         kip->slot_used[0] = SLOT_USED;
196         kip->nused = 1;
197         kip->ngarbage = 0;
198         return kip->insns;
199 }
200
201 kprobe_opcode_t __kprobes *get_insn_slot(void)
202 {
203         kprobe_opcode_t *ret;
204         mutex_lock(&kprobe_insn_mutex);
205         ret = __get_insn_slot();
206         mutex_unlock(&kprobe_insn_mutex);
207         return ret;
208 }
209
210 /* Return 1 if all garbages are collected, otherwise 0. */
211 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
212 {
213         kip->slot_used[idx] = SLOT_CLEAN;
214         kip->nused--;
215         if (kip->nused == 0) {
216                 /*
217                  * Page is no longer in use.  Free it unless
218                  * it's the last one.  We keep the last one
219                  * so as not to have to set it up again the
220                  * next time somebody inserts a probe.
221                  */
222                 hlist_del(&kip->hlist);
223                 if (hlist_empty(&kprobe_insn_pages)) {
224                         INIT_HLIST_NODE(&kip->hlist);
225                         hlist_add_head(&kip->hlist,
226                                        &kprobe_insn_pages);
227                 } else {
228                         module_free(NULL, kip->insns);
229                         kfree(kip);
230                 }
231                 return 1;
232         }
233         return 0;
234 }
235
236 static int __kprobes collect_garbage_slots(void)
237 {
238         struct kprobe_insn_page *kip;
239         struct hlist_node *pos, *next;
240
241         /* Ensure no-one is preepmted on the garbages */
242         if (check_safety())
243                 return -EAGAIN;
244
245         hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) {
246                 int i;
247                 if (kip->ngarbage == 0)
248                         continue;
249                 kip->ngarbage = 0;      /* we will collect all garbages */
250                 for (i = 0; i < INSNS_PER_PAGE; i++) {
251                         if (kip->slot_used[i] == SLOT_DIRTY &&
252                             collect_one_slot(kip, i))
253                                 break;
254                 }
255         }
256         kprobe_garbage_slots = 0;
257         return 0;
258 }
259
260 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
261 {
262         struct kprobe_insn_page *kip;
263         struct hlist_node *pos;
264
265         mutex_lock(&kprobe_insn_mutex);
266         hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
267                 if (kip->insns <= slot &&
268                     slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) {
269                         int i = (slot - kip->insns) / MAX_INSN_SIZE;
270                         if (dirty) {
271                                 kip->slot_used[i] = SLOT_DIRTY;
272                                 kip->ngarbage++;
273                         } else {
274                                 collect_one_slot(kip, i);
275                         }
276                         break;
277                 }
278         }
279
280         if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE)
281                 collect_garbage_slots();
282
283         mutex_unlock(&kprobe_insn_mutex);
284 }
285 #endif
286
287 /* We have preemption disabled.. so it is safe to use __ versions */
288 static inline void set_kprobe_instance(struct kprobe *kp)
289 {
290         __get_cpu_var(kprobe_instance) = kp;
291 }
292
293 static inline void reset_kprobe_instance(void)
294 {
295         __get_cpu_var(kprobe_instance) = NULL;
296 }
297
298 /*
299  * This routine is called either:
300  *      - under the kprobe_mutex - during kprobe_[un]register()
301  *                              OR
302  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
303  */
304 struct kprobe __kprobes *get_kprobe(void *addr)
305 {
306         struct hlist_head *head;
307         struct hlist_node *node;
308         struct kprobe *p;
309
310         head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
311         hlist_for_each_entry_rcu(p, node, head, hlist) {
312                 if (p->addr == addr)
313                         return p;
314         }
315         return NULL;
316 }
317
318 /* Arm a kprobe with text_mutex */
319 static void __kprobes arm_kprobe(struct kprobe *kp)
320 {
321         mutex_lock(&text_mutex);
322         arch_arm_kprobe(kp);
323         mutex_unlock(&text_mutex);
324 }
325
326 /* Disarm a kprobe with text_mutex */
327 static void __kprobes disarm_kprobe(struct kprobe *kp)
328 {
329         mutex_lock(&text_mutex);
330         arch_disarm_kprobe(kp);
331         mutex_unlock(&text_mutex);
332 }
333
334 /*
335  * Aggregate handlers for multiple kprobes support - these handlers
336  * take care of invoking the individual kprobe handlers on p->list
337  */
338 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
339 {
340         struct kprobe *kp;
341
342         list_for_each_entry_rcu(kp, &p->list, list) {
343                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
344                         set_kprobe_instance(kp);
345                         if (kp->pre_handler(kp, regs))
346                                 return 1;
347                 }
348                 reset_kprobe_instance();
349         }
350         return 0;
351 }
352
353 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
354                                         unsigned long flags)
355 {
356         struct kprobe *kp;
357
358         list_for_each_entry_rcu(kp, &p->list, list) {
359                 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
360                         set_kprobe_instance(kp);
361                         kp->post_handler(kp, regs, flags);
362                         reset_kprobe_instance();
363                 }
364         }
365 }
366
367 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
368                                         int trapnr)
369 {
370         struct kprobe *cur = __get_cpu_var(kprobe_instance);
371
372         /*
373          * if we faulted "during" the execution of a user specified
374          * probe handler, invoke just that probe's fault handler
375          */
376         if (cur && cur->fault_handler) {
377                 if (cur->fault_handler(cur, regs, trapnr))
378                         return 1;
379         }
380         return 0;
381 }
382
383 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
384 {
385         struct kprobe *cur = __get_cpu_var(kprobe_instance);
386         int ret = 0;
387
388         if (cur && cur->break_handler) {
389                 if (cur->break_handler(cur, regs))
390                         ret = 1;
391         }
392         reset_kprobe_instance();
393         return ret;
394 }
395
396 /* Walks the list and increments nmissed count for multiprobe case */
397 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
398 {
399         struct kprobe *kp;
400         if (p->pre_handler != aggr_pre_handler) {
401                 p->nmissed++;
402         } else {
403                 list_for_each_entry_rcu(kp, &p->list, list)
404                         kp->nmissed++;
405         }
406         return;
407 }
408
409 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
410                                 struct hlist_head *head)
411 {
412         struct kretprobe *rp = ri->rp;
413
414         /* remove rp inst off the rprobe_inst_table */
415         hlist_del(&ri->hlist);
416         INIT_HLIST_NODE(&ri->hlist);
417         if (likely(rp)) {
418                 spin_lock(&rp->lock);
419                 hlist_add_head(&ri->hlist, &rp->free_instances);
420                 spin_unlock(&rp->lock);
421         } else
422                 /* Unregistering */
423                 hlist_add_head(&ri->hlist, head);
424 }
425
426 void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
427                          struct hlist_head **head, unsigned long *flags)
428 {
429         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
430         spinlock_t *hlist_lock;
431
432         *head = &kretprobe_inst_table[hash];
433         hlist_lock = kretprobe_table_lock_ptr(hash);
434         spin_lock_irqsave(hlist_lock, *flags);
435 }
436
437 static void __kprobes kretprobe_table_lock(unsigned long hash,
438         unsigned long *flags)
439 {
440         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
441         spin_lock_irqsave(hlist_lock, *flags);
442 }
443
444 void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
445         unsigned long *flags)
446 {
447         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
448         spinlock_t *hlist_lock;
449
450         hlist_lock = kretprobe_table_lock_ptr(hash);
451         spin_unlock_irqrestore(hlist_lock, *flags);
452 }
453
454 void __kprobes kretprobe_table_unlock(unsigned long hash, unsigned long *flags)
455 {
456         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
457         spin_unlock_irqrestore(hlist_lock, *flags);
458 }
459
460 /*
461  * This function is called from finish_task_switch when task tk becomes dead,
462  * so that we can recycle any function-return probe instances associated
463  * with this task. These left over instances represent probed functions
464  * that have been called but will never return.
465  */
466 void __kprobes kprobe_flush_task(struct task_struct *tk)
467 {
468         struct kretprobe_instance *ri;
469         struct hlist_head *head, empty_rp;
470         struct hlist_node *node, *tmp;
471         unsigned long hash, flags = 0;
472
473         if (unlikely(!kprobes_initialized))
474                 /* Early boot.  kretprobe_table_locks not yet initialized. */
475                 return;
476
477         hash = hash_ptr(tk, KPROBE_HASH_BITS);
478         head = &kretprobe_inst_table[hash];
479         kretprobe_table_lock(hash, &flags);
480         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
481                 if (ri->task == tk)
482                         recycle_rp_inst(ri, &empty_rp);
483         }
484         kretprobe_table_unlock(hash, &flags);
485         INIT_HLIST_HEAD(&empty_rp);
486         hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
487                 hlist_del(&ri->hlist);
488                 kfree(ri);
489         }
490 }
491
492 static inline void free_rp_inst(struct kretprobe *rp)
493 {
494         struct kretprobe_instance *ri;
495         struct hlist_node *pos, *next;
496
497         hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
498                 hlist_del(&ri->hlist);
499                 kfree(ri);
500         }
501 }
502
503 static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
504 {
505         unsigned long flags, hash;
506         struct kretprobe_instance *ri;
507         struct hlist_node *pos, *next;
508         struct hlist_head *head;
509
510         /* No race here */
511         for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
512                 kretprobe_table_lock(hash, &flags);
513                 head = &kretprobe_inst_table[hash];
514                 hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
515                         if (ri->rp == rp)
516                                 ri->rp = NULL;
517                 }
518                 kretprobe_table_unlock(hash, &flags);
519         }
520         free_rp_inst(rp);
521 }
522
523 /*
524  * Keep all fields in the kprobe consistent
525  */
526 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
527 {
528         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
529         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
530 }
531
532 /*
533 * Add the new probe to ap->list. Fail if this is the
534 * second jprobe at the address - two jprobes can't coexist
535 */
536 static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
537 {
538         BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
539         if (p->break_handler) {
540                 if (ap->break_handler)
541                         return -EEXIST;
542                 list_add_tail_rcu(&p->list, &ap->list);
543                 ap->break_handler = aggr_break_handler;
544         } else
545                 list_add_rcu(&p->list, &ap->list);
546         if (p->post_handler && !ap->post_handler)
547                 ap->post_handler = aggr_post_handler;
548
549         if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
550                 ap->flags &= ~KPROBE_FLAG_DISABLED;
551                 if (!kprobes_all_disarmed)
552                         /* Arm the breakpoint again. */
553                         arm_kprobe(ap);
554         }
555         return 0;
556 }
557
558 /*
559  * Fill in the required fields of the "manager kprobe". Replace the
560  * earlier kprobe in the hlist with the manager kprobe
561  */
562 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
563 {
564         copy_kprobe(p, ap);
565         flush_insn_slot(ap);
566         ap->addr = p->addr;
567         ap->flags = p->flags;
568         ap->pre_handler = aggr_pre_handler;
569         ap->fault_handler = aggr_fault_handler;
570         /* We don't care the kprobe which has gone. */
571         if (p->post_handler && !kprobe_gone(p))
572                 ap->post_handler = aggr_post_handler;
573         if (p->break_handler && !kprobe_gone(p))
574                 ap->break_handler = aggr_break_handler;
575
576         INIT_LIST_HEAD(&ap->list);
577         list_add_rcu(&p->list, &ap->list);
578
579         hlist_replace_rcu(&p->hlist, &ap->hlist);
580 }
581
582 /*
583  * This is the second or subsequent kprobe at the address - handle
584  * the intricacies
585  */
586 static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
587                                           struct kprobe *p)
588 {
589         int ret = 0;
590         struct kprobe *ap = old_p;
591
592         if (old_p->pre_handler != aggr_pre_handler) {
593                 /* If old_p is not an aggr_probe, create new aggr_kprobe. */
594                 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
595                 if (!ap)
596                         return -ENOMEM;
597                 add_aggr_kprobe(ap, old_p);
598         }
599
600         if (kprobe_gone(ap)) {
601                 /*
602                  * Attempting to insert new probe at the same location that
603                  * had a probe in the module vaddr area which already
604                  * freed. So, the instruction slot has already been
605                  * released. We need a new slot for the new probe.
606                  */
607                 ret = arch_prepare_kprobe(ap);
608                 if (ret)
609                         /*
610                          * Even if fail to allocate new slot, don't need to
611                          * free aggr_probe. It will be used next time, or
612                          * freed by unregister_kprobe.
613                          */
614                         return ret;
615
616                 /*
617                  * Clear gone flag to prevent allocating new slot again, and
618                  * set disabled flag because it is not armed yet.
619                  */
620                 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
621                             | KPROBE_FLAG_DISABLED;
622         }
623
624         copy_kprobe(ap, p);
625         return add_new_kprobe(ap, p);
626 }
627
628 /* Try to disable aggr_kprobe, and return 1 if succeeded.*/
629 static int __kprobes try_to_disable_aggr_kprobe(struct kprobe *p)
630 {
631         struct kprobe *kp;
632
633         list_for_each_entry_rcu(kp, &p->list, list) {
634                 if (!kprobe_disabled(kp))
635                         /*
636                          * There is an active probe on the list.
637                          * We can't disable aggr_kprobe.
638                          */
639                         return 0;
640         }
641         p->flags |= KPROBE_FLAG_DISABLED;
642         return 1;
643 }
644
645 static int __kprobes in_kprobes_functions(unsigned long addr)
646 {
647         struct kprobe_blackpoint *kb;
648
649         if (addr >= (unsigned long)__kprobes_text_start &&
650             addr < (unsigned long)__kprobes_text_end)
651                 return -EINVAL;
652         /*
653          * If there exists a kprobe_blacklist, verify and
654          * fail any probe registration in the prohibited area
655          */
656         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
657                 if (kb->start_addr) {
658                         if (addr >= kb->start_addr &&
659                             addr < (kb->start_addr + kb->range))
660                                 return -EINVAL;
661                 }
662         }
663         return 0;
664 }
665
666 /*
667  * If we have a symbol_name argument, look it up and add the offset field
668  * to it. This way, we can specify a relative address to a symbol.
669  */
670 static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
671 {
672         kprobe_opcode_t *addr = p->addr;
673         if (p->symbol_name) {
674                 if (addr)
675                         return NULL;
676                 kprobe_lookup_name(p->symbol_name, addr);
677         }
678
679         if (!addr)
680                 return NULL;
681         return (kprobe_opcode_t *)(((char *)addr) + p->offset);
682 }
683
684 int __kprobes register_kprobe(struct kprobe *p)
685 {
686         int ret = 0;
687         struct kprobe *old_p;
688         struct module *probed_mod;
689         kprobe_opcode_t *addr;
690
691         addr = kprobe_addr(p);
692         if (!addr)
693                 return -EINVAL;
694         p->addr = addr;
695
696         preempt_disable();
697         if (!__kernel_text_address((unsigned long) p->addr) ||
698             in_kprobes_functions((unsigned long) p->addr)) {
699                 preempt_enable();
700                 return -EINVAL;
701         }
702
703         /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
704         p->flags &= KPROBE_FLAG_DISABLED;
705
706         /*
707          * Check if are we probing a module.
708          */
709         probed_mod = __module_text_address((unsigned long) p->addr);
710         if (probed_mod) {
711                 /*
712                  * We must hold a refcount of the probed module while updating
713                  * its code to prohibit unexpected unloading.
714                  */
715                 if (unlikely(!try_module_get(probed_mod))) {
716                         preempt_enable();
717                         return -EINVAL;
718                 }
719                 /*
720                  * If the module freed .init.text, we couldn't insert
721                  * kprobes in there.
722                  */
723                 if (within_module_init((unsigned long)p->addr, probed_mod) &&
724                     probed_mod->state != MODULE_STATE_COMING) {
725                         module_put(probed_mod);
726                         preempt_enable();
727                         return -EINVAL;
728                 }
729         }
730         preempt_enable();
731
732         p->nmissed = 0;
733         INIT_LIST_HEAD(&p->list);
734         mutex_lock(&kprobe_mutex);
735         old_p = get_kprobe(p->addr);
736         if (old_p) {
737                 ret = register_aggr_kprobe(old_p, p);
738                 goto out;
739         }
740
741         mutex_lock(&text_mutex);
742         ret = arch_prepare_kprobe(p);
743         if (ret)
744                 goto out_unlock_text;
745
746         INIT_HLIST_NODE(&p->hlist);
747         hlist_add_head_rcu(&p->hlist,
748                        &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
749
750         if (!kprobes_all_disarmed && !kprobe_disabled(p))
751                 arch_arm_kprobe(p);
752
753 out_unlock_text:
754         mutex_unlock(&text_mutex);
755 out:
756         mutex_unlock(&kprobe_mutex);
757
758         if (probed_mod)
759                 module_put(probed_mod);
760
761         return ret;
762 }
763 EXPORT_SYMBOL_GPL(register_kprobe);
764
765 /* Check passed kprobe is valid and return kprobe in kprobe_table. */
766 static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
767 {
768         struct kprobe *old_p, *list_p;
769
770         old_p = get_kprobe(p->addr);
771         if (unlikely(!old_p))
772                 return NULL;
773
774         if (p != old_p) {
775                 list_for_each_entry_rcu(list_p, &old_p->list, list)
776                         if (list_p == p)
777                         /* kprobe p is a valid probe */
778                                 goto valid;
779                 return NULL;
780         }
781 valid:
782         return old_p;
783 }
784
785 /*
786  * Unregister a kprobe without a scheduler synchronization.
787  */
788 static int __kprobes __unregister_kprobe_top(struct kprobe *p)
789 {
790         struct kprobe *old_p, *list_p;
791
792         old_p = __get_valid_kprobe(p);
793         if (old_p == NULL)
794                 return -EINVAL;
795
796         if (old_p == p ||
797             (old_p->pre_handler == aggr_pre_handler &&
798              list_is_singular(&old_p->list))) {
799                 /*
800                  * Only probe on the hash list. Disarm only if kprobes are
801                  * enabled and not gone - otherwise, the breakpoint would
802                  * already have been removed. We save on flushing icache.
803                  */
804                 if (!kprobes_all_disarmed && !kprobe_disabled(old_p))
805                         disarm_kprobe(p);
806                 hlist_del_rcu(&old_p->hlist);
807         } else {
808                 if (p->break_handler && !kprobe_gone(p))
809                         old_p->break_handler = NULL;
810                 if (p->post_handler && !kprobe_gone(p)) {
811                         list_for_each_entry_rcu(list_p, &old_p->list, list) {
812                                 if ((list_p != p) && (list_p->post_handler))
813                                         goto noclean;
814                         }
815                         old_p->post_handler = NULL;
816                 }
817 noclean:
818                 list_del_rcu(&p->list);
819                 if (!kprobe_disabled(old_p)) {
820                         try_to_disable_aggr_kprobe(old_p);
821                         if (!kprobes_all_disarmed && kprobe_disabled(old_p))
822                                 disarm_kprobe(old_p);
823                 }
824         }
825         return 0;
826 }
827
828 static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
829 {
830         struct kprobe *old_p;
831
832         if (list_empty(&p->list))
833                 arch_remove_kprobe(p);
834         else if (list_is_singular(&p->list)) {
835                 /* "p" is the last child of an aggr_kprobe */
836                 old_p = list_entry(p->list.next, struct kprobe, list);
837                 list_del(&p->list);
838                 arch_remove_kprobe(old_p);
839                 kfree(old_p);
840         }
841 }
842
843 int __kprobes register_kprobes(struct kprobe **kps, int num)
844 {
845         int i, ret = 0;
846
847         if (num <= 0)
848                 return -EINVAL;
849         for (i = 0; i < num; i++) {
850                 ret = register_kprobe(kps[i]);
851                 if (ret < 0) {
852                         if (i > 0)
853                                 unregister_kprobes(kps, i);
854                         break;
855                 }
856         }
857         return ret;
858 }
859 EXPORT_SYMBOL_GPL(register_kprobes);
860
861 void __kprobes unregister_kprobe(struct kprobe *p)
862 {
863         unregister_kprobes(&p, 1);
864 }
865 EXPORT_SYMBOL_GPL(unregister_kprobe);
866
867 void __kprobes unregister_kprobes(struct kprobe **kps, int num)
868 {
869         int i;
870
871         if (num <= 0)
872                 return;
873         mutex_lock(&kprobe_mutex);
874         for (i = 0; i < num; i++)
875                 if (__unregister_kprobe_top(kps[i]) < 0)
876                         kps[i]->addr = NULL;
877         mutex_unlock(&kprobe_mutex);
878
879         synchronize_sched();
880         for (i = 0; i < num; i++)
881                 if (kps[i]->addr)
882                         __unregister_kprobe_bottom(kps[i]);
883 }
884 EXPORT_SYMBOL_GPL(unregister_kprobes);
885
886 static struct notifier_block kprobe_exceptions_nb = {
887         .notifier_call = kprobe_exceptions_notify,
888         .priority = 0x7fffffff /* we need to be notified first */
889 };
890
891 unsigned long __weak arch_deref_entry_point(void *entry)
892 {
893         return (unsigned long)entry;
894 }
895
896 int __kprobes register_jprobes(struct jprobe **jps, int num)
897 {
898         struct jprobe *jp;
899         int ret = 0, i;
900
901         if (num <= 0)
902                 return -EINVAL;
903         for (i = 0; i < num; i++) {
904                 unsigned long addr;
905                 jp = jps[i];
906                 addr = arch_deref_entry_point(jp->entry);
907
908                 if (!kernel_text_address(addr))
909                         ret = -EINVAL;
910                 else {
911                         /* Todo: Verify probepoint is a function entry point */
912                         jp->kp.pre_handler = setjmp_pre_handler;
913                         jp->kp.break_handler = longjmp_break_handler;
914                         ret = register_kprobe(&jp->kp);
915                 }
916                 if (ret < 0) {
917                         if (i > 0)
918                                 unregister_jprobes(jps, i);
919                         break;
920                 }
921         }
922         return ret;
923 }
924 EXPORT_SYMBOL_GPL(register_jprobes);
925
926 int __kprobes register_jprobe(struct jprobe *jp)
927 {
928         return register_jprobes(&jp, 1);
929 }
930 EXPORT_SYMBOL_GPL(register_jprobe);
931
932 void __kprobes unregister_jprobe(struct jprobe *jp)
933 {
934         unregister_jprobes(&jp, 1);
935 }
936 EXPORT_SYMBOL_GPL(unregister_jprobe);
937
938 void __kprobes unregister_jprobes(struct jprobe **jps, int num)
939 {
940         int i;
941
942         if (num <= 0)
943                 return;
944         mutex_lock(&kprobe_mutex);
945         for (i = 0; i < num; i++)
946                 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
947                         jps[i]->kp.addr = NULL;
948         mutex_unlock(&kprobe_mutex);
949
950         synchronize_sched();
951         for (i = 0; i < num; i++) {
952                 if (jps[i]->kp.addr)
953                         __unregister_kprobe_bottom(&jps[i]->kp);
954         }
955 }
956 EXPORT_SYMBOL_GPL(unregister_jprobes);
957
958 #ifdef CONFIG_KRETPROBES
959 /*
960  * This kprobe pre_handler is registered with every kretprobe. When probe
961  * hits it will set up the return probe.
962  */
963 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
964                                            struct pt_regs *regs)
965 {
966         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
967         unsigned long hash, flags = 0;
968         struct kretprobe_instance *ri;
969
970         /*TODO: consider to only swap the RA after the last pre_handler fired */
971         hash = hash_ptr(current, KPROBE_HASH_BITS);
972         spin_lock_irqsave(&rp->lock, flags);
973         if (!hlist_empty(&rp->free_instances)) {
974                 ri = hlist_entry(rp->free_instances.first,
975                                 struct kretprobe_instance, hlist);
976                 hlist_del(&ri->hlist);
977                 spin_unlock_irqrestore(&rp->lock, flags);
978
979                 ri->rp = rp;
980                 ri->task = current;
981
982                 if (rp->entry_handler && rp->entry_handler(ri, regs))
983                         return 0;
984
985                 arch_prepare_kretprobe(ri, regs);
986
987                 /* XXX(hch): why is there no hlist_move_head? */
988                 INIT_HLIST_NODE(&ri->hlist);
989                 kretprobe_table_lock(hash, &flags);
990                 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
991                 kretprobe_table_unlock(hash, &flags);
992         } else {
993                 rp->nmissed++;
994                 spin_unlock_irqrestore(&rp->lock, flags);
995         }
996         return 0;
997 }
998
999 int __kprobes register_kretprobe(struct kretprobe *rp)
1000 {
1001         int ret = 0;
1002         struct kretprobe_instance *inst;
1003         int i;
1004         void *addr;
1005
1006         if (kretprobe_blacklist_size) {
1007                 addr = kprobe_addr(&rp->kp);
1008                 if (!addr)
1009                         return -EINVAL;
1010
1011                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1012                         if (kretprobe_blacklist[i].addr == addr)
1013                                 return -EINVAL;
1014                 }
1015         }
1016
1017         rp->kp.pre_handler = pre_handler_kretprobe;
1018         rp->kp.post_handler = NULL;
1019         rp->kp.fault_handler = NULL;
1020         rp->kp.break_handler = NULL;
1021
1022         /* Pre-allocate memory for max kretprobe instances */
1023         if (rp->maxactive <= 0) {
1024 #ifdef CONFIG_PREEMPT
1025                 rp->maxactive = max(10, 2 * NR_CPUS);
1026 #else
1027                 rp->maxactive = NR_CPUS;
1028 #endif
1029         }
1030         spin_lock_init(&rp->lock);
1031         INIT_HLIST_HEAD(&rp->free_instances);
1032         for (i = 0; i < rp->maxactive; i++) {
1033                 inst = kmalloc(sizeof(struct kretprobe_instance) +
1034                                rp->data_size, GFP_KERNEL);
1035                 if (inst == NULL) {
1036                         free_rp_inst(rp);
1037                         return -ENOMEM;
1038                 }
1039                 INIT_HLIST_NODE(&inst->hlist);
1040                 hlist_add_head(&inst->hlist, &rp->free_instances);
1041         }
1042
1043         rp->nmissed = 0;
1044         /* Establish function entry probe point */
1045         ret = register_kprobe(&rp->kp);
1046         if (ret != 0)
1047                 free_rp_inst(rp);
1048         return ret;
1049 }
1050 EXPORT_SYMBOL_GPL(register_kretprobe);
1051
1052 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1053 {
1054         int ret = 0, i;
1055
1056         if (num <= 0)
1057                 return -EINVAL;
1058         for (i = 0; i < num; i++) {
1059                 ret = register_kretprobe(rps[i]);
1060                 if (ret < 0) {
1061                         if (i > 0)
1062                                 unregister_kretprobes(rps, i);
1063                         break;
1064                 }
1065         }
1066         return ret;
1067 }
1068 EXPORT_SYMBOL_GPL(register_kretprobes);
1069
1070 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1071 {
1072         unregister_kretprobes(&rp, 1);
1073 }
1074 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1075
1076 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1077 {
1078         int i;
1079
1080         if (num <= 0)
1081                 return;
1082         mutex_lock(&kprobe_mutex);
1083         for (i = 0; i < num; i++)
1084                 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1085                         rps[i]->kp.addr = NULL;
1086         mutex_unlock(&kprobe_mutex);
1087
1088         synchronize_sched();
1089         for (i = 0; i < num; i++) {
1090                 if (rps[i]->kp.addr) {
1091                         __unregister_kprobe_bottom(&rps[i]->kp);
1092                         cleanup_rp_inst(rps[i]);
1093                 }
1094         }
1095 }
1096 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1097
1098 #else /* CONFIG_KRETPROBES */
1099 int __kprobes register_kretprobe(struct kretprobe *rp)
1100 {
1101         return -ENOSYS;
1102 }
1103 EXPORT_SYMBOL_GPL(register_kretprobe);
1104
1105 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1106 {
1107         return -ENOSYS;
1108 }
1109 EXPORT_SYMBOL_GPL(register_kretprobes);
1110
1111 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1112 {
1113 }
1114 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1115
1116 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1117 {
1118 }
1119 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1120
1121 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1122                                            struct pt_regs *regs)
1123 {
1124         return 0;
1125 }
1126
1127 #endif /* CONFIG_KRETPROBES */
1128
1129 /* Set the kprobe gone and remove its instruction buffer. */
1130 static void __kprobes kill_kprobe(struct kprobe *p)
1131 {
1132         struct kprobe *kp;
1133
1134         p->flags |= KPROBE_FLAG_GONE;
1135         if (p->pre_handler == aggr_pre_handler) {
1136                 /*
1137                  * If this is an aggr_kprobe, we have to list all the
1138                  * chained probes and mark them GONE.
1139                  */
1140                 list_for_each_entry_rcu(kp, &p->list, list)
1141                         kp->flags |= KPROBE_FLAG_GONE;
1142                 p->post_handler = NULL;
1143                 p->break_handler = NULL;
1144         }
1145         /*
1146          * Here, we can remove insn_slot safely, because no thread calls
1147          * the original probed function (which will be freed soon) any more.
1148          */
1149         arch_remove_kprobe(p);
1150 }
1151
1152 /* Module notifier call back, checking kprobes on the module */
1153 static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1154                                              unsigned long val, void *data)
1155 {
1156         struct module *mod = data;
1157         struct hlist_head *head;
1158         struct hlist_node *node;
1159         struct kprobe *p;
1160         unsigned int i;
1161         int checkcore = (val == MODULE_STATE_GOING);
1162
1163         if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1164                 return NOTIFY_DONE;
1165
1166         /*
1167          * When MODULE_STATE_GOING was notified, both of module .text and
1168          * .init.text sections would be freed. When MODULE_STATE_LIVE was
1169          * notified, only .init.text section would be freed. We need to
1170          * disable kprobes which have been inserted in the sections.
1171          */
1172         mutex_lock(&kprobe_mutex);
1173         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1174                 head = &kprobe_table[i];
1175                 hlist_for_each_entry_rcu(p, node, head, hlist)
1176                         if (within_module_init((unsigned long)p->addr, mod) ||
1177                             (checkcore &&
1178                              within_module_core((unsigned long)p->addr, mod))) {
1179                                 /*
1180                                  * The vaddr this probe is installed will soon
1181                                  * be vfreed buy not synced to disk. Hence,
1182                                  * disarming the breakpoint isn't needed.
1183                                  */
1184                                 kill_kprobe(p);
1185                         }
1186         }
1187         mutex_unlock(&kprobe_mutex);
1188         return NOTIFY_DONE;
1189 }
1190
1191 static struct notifier_block kprobe_module_nb = {
1192         .notifier_call = kprobes_module_callback,
1193         .priority = 0
1194 };
1195
1196 static int __init init_kprobes(void)
1197 {
1198         int i, err = 0;
1199         unsigned long offset = 0, size = 0;
1200         char *modname, namebuf[128];
1201         const char *symbol_name;
1202         void *addr;
1203         struct kprobe_blackpoint *kb;
1204
1205         /* FIXME allocate the probe table, currently defined statically */
1206         /* initialize all list heads */
1207         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1208                 INIT_HLIST_HEAD(&kprobe_table[i]);
1209                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1210                 spin_lock_init(&(kretprobe_table_locks[i].lock));
1211         }
1212
1213         /*
1214          * Lookup and populate the kprobe_blacklist.
1215          *
1216          * Unlike the kretprobe blacklist, we'll need to determine
1217          * the range of addresses that belong to the said functions,
1218          * since a kprobe need not necessarily be at the beginning
1219          * of a function.
1220          */
1221         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1222                 kprobe_lookup_name(kb->name, addr);
1223                 if (!addr)
1224                         continue;
1225
1226                 kb->start_addr = (unsigned long)addr;
1227                 symbol_name = kallsyms_lookup(kb->start_addr,
1228                                 &size, &offset, &modname, namebuf);
1229                 if (!symbol_name)
1230                         kb->range = 0;
1231                 else
1232                         kb->range = size;
1233         }
1234
1235         if (kretprobe_blacklist_size) {
1236                 /* lookup the function address from its name */
1237                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1238                         kprobe_lookup_name(kretprobe_blacklist[i].name,
1239                                            kretprobe_blacklist[i].addr);
1240                         if (!kretprobe_blacklist[i].addr)
1241                                 printk("kretprobe: lookup failed: %s\n",
1242                                        kretprobe_blacklist[i].name);
1243                 }
1244         }
1245
1246         /* By default, kprobes are armed */
1247         kprobes_all_disarmed = false;
1248
1249         err = arch_init_kprobes();
1250         if (!err)
1251                 err = register_die_notifier(&kprobe_exceptions_nb);
1252         if (!err)
1253                 err = register_module_notifier(&kprobe_module_nb);
1254
1255         kprobes_initialized = (err == 0);
1256
1257         if (!err)
1258                 init_test_probes();
1259         return err;
1260 }
1261
1262 #ifdef CONFIG_DEBUG_FS
1263 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
1264                 const char *sym, int offset,char *modname)
1265 {
1266         char *kprobe_type;
1267
1268         if (p->pre_handler == pre_handler_kretprobe)
1269                 kprobe_type = "r";
1270         else if (p->pre_handler == setjmp_pre_handler)
1271                 kprobe_type = "j";
1272         else
1273                 kprobe_type = "k";
1274         if (sym)
1275                 seq_printf(pi, "%p  %s  %s+0x%x  %s %s%s\n",
1276                         p->addr, kprobe_type, sym, offset,
1277                         (modname ? modname : " "),
1278                         (kprobe_gone(p) ? "[GONE]" : ""),
1279                         ((kprobe_disabled(p) && !kprobe_gone(p)) ?
1280                          "[DISABLED]" : ""));
1281         else
1282                 seq_printf(pi, "%p  %s  %p %s%s\n",
1283                         p->addr, kprobe_type, p->addr,
1284                         (kprobe_gone(p) ? "[GONE]" : ""),
1285                         ((kprobe_disabled(p) && !kprobe_gone(p)) ?
1286                          "[DISABLED]" : ""));
1287 }
1288
1289 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
1290 {
1291         return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
1292 }
1293
1294 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
1295 {
1296         (*pos)++;
1297         if (*pos >= KPROBE_TABLE_SIZE)
1298                 return NULL;
1299         return pos;
1300 }
1301
1302 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
1303 {
1304         /* Nothing to do */
1305 }
1306
1307 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
1308 {
1309         struct hlist_head *head;
1310         struct hlist_node *node;
1311         struct kprobe *p, *kp;
1312         const char *sym = NULL;
1313         unsigned int i = *(loff_t *) v;
1314         unsigned long offset = 0;
1315         char *modname, namebuf[128];
1316
1317         head = &kprobe_table[i];
1318         preempt_disable();
1319         hlist_for_each_entry_rcu(p, node, head, hlist) {
1320                 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
1321                                         &offset, &modname, namebuf);
1322                 if (p->pre_handler == aggr_pre_handler) {
1323                         list_for_each_entry_rcu(kp, &p->list, list)
1324                                 report_probe(pi, kp, sym, offset, modname);
1325                 } else
1326                         report_probe(pi, p, sym, offset, modname);
1327         }
1328         preempt_enable();
1329         return 0;
1330 }
1331
1332 static struct seq_operations kprobes_seq_ops = {
1333         .start = kprobe_seq_start,
1334         .next  = kprobe_seq_next,
1335         .stop  = kprobe_seq_stop,
1336         .show  = show_kprobe_addr
1337 };
1338
1339 static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
1340 {
1341         return seq_open(filp, &kprobes_seq_ops);
1342 }
1343
1344 static struct file_operations debugfs_kprobes_operations = {
1345         .open           = kprobes_open,
1346         .read           = seq_read,
1347         .llseek         = seq_lseek,
1348         .release        = seq_release,
1349 };
1350
1351 /* Disable one kprobe */
1352 int __kprobes disable_kprobe(struct kprobe *kp)
1353 {
1354         int ret = 0;
1355         struct kprobe *p;
1356
1357         mutex_lock(&kprobe_mutex);
1358
1359         /* Check whether specified probe is valid. */
1360         p = __get_valid_kprobe(kp);
1361         if (unlikely(p == NULL)) {
1362                 ret = -EINVAL;
1363                 goto out;
1364         }
1365
1366         /* If the probe is already disabled (or gone), just return */
1367         if (kprobe_disabled(kp))
1368                 goto out;
1369
1370         kp->flags |= KPROBE_FLAG_DISABLED;
1371         if (p != kp)
1372                 /* When kp != p, p is always enabled. */
1373                 try_to_disable_aggr_kprobe(p);
1374
1375         if (!kprobes_all_disarmed && kprobe_disabled(p))
1376                 disarm_kprobe(p);
1377 out:
1378         mutex_unlock(&kprobe_mutex);
1379         return ret;
1380 }
1381 EXPORT_SYMBOL_GPL(disable_kprobe);
1382
1383 /* Enable one kprobe */
1384 int __kprobes enable_kprobe(struct kprobe *kp)
1385 {
1386         int ret = 0;
1387         struct kprobe *p;
1388
1389         mutex_lock(&kprobe_mutex);
1390
1391         /* Check whether specified probe is valid. */
1392         p = __get_valid_kprobe(kp);
1393         if (unlikely(p == NULL)) {
1394                 ret = -EINVAL;
1395                 goto out;
1396         }
1397
1398         if (kprobe_gone(kp)) {
1399                 /* This kprobe has gone, we couldn't enable it. */
1400                 ret = -EINVAL;
1401                 goto out;
1402         }
1403
1404         if (!kprobes_all_disarmed && kprobe_disabled(p))
1405                 arm_kprobe(p);
1406
1407         p->flags &= ~KPROBE_FLAG_DISABLED;
1408         if (p != kp)
1409                 kp->flags &= ~KPROBE_FLAG_DISABLED;
1410 out:
1411         mutex_unlock(&kprobe_mutex);
1412         return ret;
1413 }
1414 EXPORT_SYMBOL_GPL(enable_kprobe);
1415
1416 static void __kprobes arm_all_kprobes(void)
1417 {
1418         struct hlist_head *head;
1419         struct hlist_node *node;
1420         struct kprobe *p;
1421         unsigned int i;
1422
1423         mutex_lock(&kprobe_mutex);
1424
1425         /* If kprobes are armed, just return */
1426         if (!kprobes_all_disarmed)
1427                 goto already_enabled;
1428
1429         mutex_lock(&text_mutex);
1430         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1431                 head = &kprobe_table[i];
1432                 hlist_for_each_entry_rcu(p, node, head, hlist)
1433                         if (!kprobe_disabled(p))
1434                                 arch_arm_kprobe(p);
1435         }
1436         mutex_unlock(&text_mutex);
1437
1438         kprobes_all_disarmed = false;
1439         printk(KERN_INFO "Kprobes globally enabled\n");
1440
1441 already_enabled:
1442         mutex_unlock(&kprobe_mutex);
1443         return;
1444 }
1445
1446 static void __kprobes disarm_all_kprobes(void)
1447 {
1448         struct hlist_head *head;
1449         struct hlist_node *node;
1450         struct kprobe *p;
1451         unsigned int i;
1452
1453         mutex_lock(&kprobe_mutex);
1454
1455         /* If kprobes are already disarmed, just return */
1456         if (kprobes_all_disarmed)
1457                 goto already_disabled;
1458
1459         kprobes_all_disarmed = true;
1460         printk(KERN_INFO "Kprobes globally disabled\n");
1461         mutex_lock(&text_mutex);
1462         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1463                 head = &kprobe_table[i];
1464                 hlist_for_each_entry_rcu(p, node, head, hlist) {
1465                         if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
1466                                 arch_disarm_kprobe(p);
1467                 }
1468         }
1469
1470         mutex_unlock(&text_mutex);
1471         mutex_unlock(&kprobe_mutex);
1472         /* Allow all currently running kprobes to complete */
1473         synchronize_sched();
1474         return;
1475
1476 already_disabled:
1477         mutex_unlock(&kprobe_mutex);
1478         return;
1479 }
1480
1481 /*
1482  * XXX: The debugfs bool file interface doesn't allow for callbacks
1483  * when the bool state is switched. We can reuse that facility when
1484  * available
1485  */
1486 static ssize_t read_enabled_file_bool(struct file *file,
1487                char __user *user_buf, size_t count, loff_t *ppos)
1488 {
1489         char buf[3];
1490
1491         if (!kprobes_all_disarmed)
1492                 buf[0] = '1';
1493         else
1494                 buf[0] = '0';
1495         buf[1] = '\n';
1496         buf[2] = 0x00;
1497         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1498 }
1499
1500 static ssize_t write_enabled_file_bool(struct file *file,
1501                const char __user *user_buf, size_t count, loff_t *ppos)
1502 {
1503         char buf[32];
1504         int buf_size;
1505
1506         buf_size = min(count, (sizeof(buf)-1));
1507         if (copy_from_user(buf, user_buf, buf_size))
1508                 return -EFAULT;
1509
1510         switch (buf[0]) {
1511         case 'y':
1512         case 'Y':
1513         case '1':
1514                 arm_all_kprobes();
1515                 break;
1516         case 'n':
1517         case 'N':
1518         case '0':
1519                 disarm_all_kprobes();
1520                 break;
1521         }
1522
1523         return count;
1524 }
1525
1526 static struct file_operations fops_kp = {
1527         .read =         read_enabled_file_bool,
1528         .write =        write_enabled_file_bool,
1529 };
1530
1531 static int __kprobes debugfs_kprobe_init(void)
1532 {
1533         struct dentry *dir, *file;
1534         unsigned int value = 1;
1535
1536         dir = debugfs_create_dir("kprobes", NULL);
1537         if (!dir)
1538                 return -ENOMEM;
1539
1540         file = debugfs_create_file("list", 0444, dir, NULL,
1541                                 &debugfs_kprobes_operations);
1542         if (!file) {
1543                 debugfs_remove(dir);
1544                 return -ENOMEM;
1545         }
1546
1547         file = debugfs_create_file("enabled", 0600, dir,
1548                                         &value, &fops_kp);
1549         if (!file) {
1550                 debugfs_remove(dir);
1551                 return -ENOMEM;
1552         }
1553
1554         return 0;
1555 }
1556
1557 late_initcall(debugfs_kprobe_init);
1558 #endif /* CONFIG_DEBUG_FS */
1559
1560 module_init(init_kprobes);
1561
1562 /* defined in arch/.../kernel/kprobes.c */
1563 EXPORT_SYMBOL_GPL(jprobe_return);