Merge branch 'llseek' of git://git.kernel.org/pub/scm/linux/kernel/git/arnd/bkl
[linux-2.6.git] / kernel / kprobes.c
1 /*
2  *  Kernel Probes (KProbes)
3  *  kernel/kprobes.c
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18  *
19  * Copyright (C) IBM Corporation, 2002, 2004
20  *
21  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22  *              Probes initial implementation (includes suggestions from
23  *              Rusty Russell).
24  * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25  *              hlists and exceptions notifier as suggested by Andi Kleen.
26  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27  *              interface to access function arguments.
28  * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29  *              exceptions notifier to be first on the priority list.
30  * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31  *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32  *              <prasanna@in.ibm.com> added function-return probes.
33  */
34 #include <linux/kprobes.h>
35 #include <linux/hash.h>
36 #include <linux/init.h>
37 #include <linux/slab.h>
38 #include <linux/stddef.h>
39 #include <linux/module.h>
40 #include <linux/moduleloader.h>
41 #include <linux/kallsyms.h>
42 #include <linux/freezer.h>
43 #include <linux/seq_file.h>
44 #include <linux/debugfs.h>
45 #include <linux/sysctl.h>
46 #include <linux/kdebug.h>
47 #include <linux/memory.h>
48 #include <linux/ftrace.h>
49 #include <linux/cpu.h>
50 #include <linux/jump_label.h>
51
52 #include <asm-generic/sections.h>
53 #include <asm/cacheflush.h>
54 #include <asm/errno.h>
55 #include <asm/uaccess.h>
56
57 #define KPROBE_HASH_BITS 6
58 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
59
60
61 /*
62  * Some oddball architectures like 64bit powerpc have function descriptors
63  * so this must be overridable.
64  */
65 #ifndef kprobe_lookup_name
66 #define kprobe_lookup_name(name, addr) \
67         addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
68 #endif
69
70 static int kprobes_initialized;
71 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
72 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
73
74 /* NOTE: change this value only with kprobe_mutex held */
75 static bool kprobes_all_disarmed;
76
77 static DEFINE_MUTEX(kprobe_mutex);      /* Protects kprobe_table */
78 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
79 static struct {
80         spinlock_t lock ____cacheline_aligned_in_smp;
81 } kretprobe_table_locks[KPROBE_TABLE_SIZE];
82
83 static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
84 {
85         return &(kretprobe_table_locks[hash].lock);
86 }
87
88 /*
89  * Normally, functions that we'd want to prohibit kprobes in, are marked
90  * __kprobes. But, there are cases where such functions already belong to
91  * a different section (__sched for preempt_schedule)
92  *
93  * For such cases, we now have a blacklist
94  */
95 static struct kprobe_blackpoint kprobe_blacklist[] = {
96         {"preempt_schedule",},
97         {"native_get_debugreg",},
98         {"irq_entries_start",},
99         {"common_interrupt",},
100         {"mcount",},    /* mcount can be called from everywhere */
101         {NULL}    /* Terminator */
102 };
103
104 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
105 /*
106  * kprobe->ainsn.insn points to the copy of the instruction to be
107  * single-stepped. x86_64, POWER4 and above have no-exec support and
108  * stepping on the instruction on a vmalloced/kmalloced/data page
109  * is a recipe for disaster
110  */
111 struct kprobe_insn_page {
112         struct list_head list;
113         kprobe_opcode_t *insns;         /* Page of instruction slots */
114         int nused;
115         int ngarbage;
116         char slot_used[];
117 };
118
119 #define KPROBE_INSN_PAGE_SIZE(slots)                    \
120         (offsetof(struct kprobe_insn_page, slot_used) + \
121          (sizeof(char) * (slots)))
122
123 struct kprobe_insn_cache {
124         struct list_head pages; /* list of kprobe_insn_page */
125         size_t insn_size;       /* size of instruction slot */
126         int nr_garbage;
127 };
128
129 static int slots_per_page(struct kprobe_insn_cache *c)
130 {
131         return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
132 }
133
134 enum kprobe_slot_state {
135         SLOT_CLEAN = 0,
136         SLOT_DIRTY = 1,
137         SLOT_USED = 2,
138 };
139
140 static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */
141 static struct kprobe_insn_cache kprobe_insn_slots = {
142         .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
143         .insn_size = MAX_INSN_SIZE,
144         .nr_garbage = 0,
145 };
146 static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);
147
148 /**
149  * __get_insn_slot() - Find a slot on an executable page for an instruction.
150  * We allocate an executable page if there's no room on existing ones.
151  */
152 static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c)
153 {
154         struct kprobe_insn_page *kip;
155
156  retry:
157         list_for_each_entry(kip, &c->pages, list) {
158                 if (kip->nused < slots_per_page(c)) {
159                         int i;
160                         for (i = 0; i < slots_per_page(c); i++) {
161                                 if (kip->slot_used[i] == SLOT_CLEAN) {
162                                         kip->slot_used[i] = SLOT_USED;
163                                         kip->nused++;
164                                         return kip->insns + (i * c->insn_size);
165                                 }
166                         }
167                         /* kip->nused is broken. Fix it. */
168                         kip->nused = slots_per_page(c);
169                         WARN_ON(1);
170                 }
171         }
172
173         /* If there are any garbage slots, collect it and try again. */
174         if (c->nr_garbage && collect_garbage_slots(c) == 0)
175                 goto retry;
176
177         /* All out of space.  Need to allocate a new page. */
178         kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
179         if (!kip)
180                 return NULL;
181
182         /*
183          * Use module_alloc so this page is within +/- 2GB of where the
184          * kernel image and loaded module images reside. This is required
185          * so x86_64 can correctly handle the %rip-relative fixups.
186          */
187         kip->insns = module_alloc(PAGE_SIZE);
188         if (!kip->insns) {
189                 kfree(kip);
190                 return NULL;
191         }
192         INIT_LIST_HEAD(&kip->list);
193         memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
194         kip->slot_used[0] = SLOT_USED;
195         kip->nused = 1;
196         kip->ngarbage = 0;
197         list_add(&kip->list, &c->pages);
198         return kip->insns;
199 }
200
201
202 kprobe_opcode_t __kprobes *get_insn_slot(void)
203 {
204         kprobe_opcode_t *ret = NULL;
205
206         mutex_lock(&kprobe_insn_mutex);
207         ret = __get_insn_slot(&kprobe_insn_slots);
208         mutex_unlock(&kprobe_insn_mutex);
209
210         return ret;
211 }
212
213 /* Return 1 if all garbages are collected, otherwise 0. */
214 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
215 {
216         kip->slot_used[idx] = SLOT_CLEAN;
217         kip->nused--;
218         if (kip->nused == 0) {
219                 /*
220                  * Page is no longer in use.  Free it unless
221                  * it's the last one.  We keep the last one
222                  * so as not to have to set it up again the
223                  * next time somebody inserts a probe.
224                  */
225                 if (!list_is_singular(&kip->list)) {
226                         list_del(&kip->list);
227                         module_free(NULL, kip->insns);
228                         kfree(kip);
229                 }
230                 return 1;
231         }
232         return 0;
233 }
234
235 static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)
236 {
237         struct kprobe_insn_page *kip, *next;
238
239         /* Ensure no-one is interrupted on the garbages */
240         synchronize_sched();
241
242         list_for_each_entry_safe(kip, next, &c->pages, list) {
243                 int i;
244                 if (kip->ngarbage == 0)
245                         continue;
246                 kip->ngarbage = 0;      /* we will collect all garbages */
247                 for (i = 0; i < slots_per_page(c); i++) {
248                         if (kip->slot_used[i] == SLOT_DIRTY &&
249                             collect_one_slot(kip, i))
250                                 break;
251                 }
252         }
253         c->nr_garbage = 0;
254         return 0;
255 }
256
257 static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,
258                                        kprobe_opcode_t *slot, int dirty)
259 {
260         struct kprobe_insn_page *kip;
261
262         list_for_each_entry(kip, &c->pages, list) {
263                 long idx = ((long)slot - (long)kip->insns) /
264                                 (c->insn_size * sizeof(kprobe_opcode_t));
265                 if (idx >= 0 && idx < slots_per_page(c)) {
266                         WARN_ON(kip->slot_used[idx] != SLOT_USED);
267                         if (dirty) {
268                                 kip->slot_used[idx] = SLOT_DIRTY;
269                                 kip->ngarbage++;
270                                 if (++c->nr_garbage > slots_per_page(c))
271                                         collect_garbage_slots(c);
272                         } else
273                                 collect_one_slot(kip, idx);
274                         return;
275                 }
276         }
277         /* Could not free this slot. */
278         WARN_ON(1);
279 }
280
281 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
282 {
283         mutex_lock(&kprobe_insn_mutex);
284         __free_insn_slot(&kprobe_insn_slots, slot, dirty);
285         mutex_unlock(&kprobe_insn_mutex);
286 }
287 #ifdef CONFIG_OPTPROBES
288 /* For optimized_kprobe buffer */
289 static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */
290 static struct kprobe_insn_cache kprobe_optinsn_slots = {
291         .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
292         /* .insn_size is initialized later */
293         .nr_garbage = 0,
294 };
295 /* Get a slot for optimized_kprobe buffer */
296 kprobe_opcode_t __kprobes *get_optinsn_slot(void)
297 {
298         kprobe_opcode_t *ret = NULL;
299
300         mutex_lock(&kprobe_optinsn_mutex);
301         ret = __get_insn_slot(&kprobe_optinsn_slots);
302         mutex_unlock(&kprobe_optinsn_mutex);
303
304         return ret;
305 }
306
307 void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty)
308 {
309         mutex_lock(&kprobe_optinsn_mutex);
310         __free_insn_slot(&kprobe_optinsn_slots, slot, dirty);
311         mutex_unlock(&kprobe_optinsn_mutex);
312 }
313 #endif
314 #endif
315
316 /* We have preemption disabled.. so it is safe to use __ versions */
317 static inline void set_kprobe_instance(struct kprobe *kp)
318 {
319         __get_cpu_var(kprobe_instance) = kp;
320 }
321
322 static inline void reset_kprobe_instance(void)
323 {
324         __get_cpu_var(kprobe_instance) = NULL;
325 }
326
327 /*
328  * This routine is called either:
329  *      - under the kprobe_mutex - during kprobe_[un]register()
330  *                              OR
331  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
332  */
333 struct kprobe __kprobes *get_kprobe(void *addr)
334 {
335         struct hlist_head *head;
336         struct hlist_node *node;
337         struct kprobe *p;
338
339         head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
340         hlist_for_each_entry_rcu(p, node, head, hlist) {
341                 if (p->addr == addr)
342                         return p;
343         }
344
345         return NULL;
346 }
347
348 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
349
350 /* Return true if the kprobe is an aggregator */
351 static inline int kprobe_aggrprobe(struct kprobe *p)
352 {
353         return p->pre_handler == aggr_pre_handler;
354 }
355
356 /*
357  * Keep all fields in the kprobe consistent
358  */
359 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
360 {
361         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
362         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
363 }
364
365 #ifdef CONFIG_OPTPROBES
366 /* NOTE: change this value only with kprobe_mutex held */
367 static bool kprobes_allow_optimization;
368
369 /*
370  * Call all pre_handler on the list, but ignores its return value.
371  * This must be called from arch-dep optimized caller.
372  */
373 void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
374 {
375         struct kprobe *kp;
376
377         list_for_each_entry_rcu(kp, &p->list, list) {
378                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
379                         set_kprobe_instance(kp);
380                         kp->pre_handler(kp, regs);
381                 }
382                 reset_kprobe_instance();
383         }
384 }
385
386 /* Return true(!0) if the kprobe is ready for optimization. */
387 static inline int kprobe_optready(struct kprobe *p)
388 {
389         struct optimized_kprobe *op;
390
391         if (kprobe_aggrprobe(p)) {
392                 op = container_of(p, struct optimized_kprobe, kp);
393                 return arch_prepared_optinsn(&op->optinsn);
394         }
395
396         return 0;
397 }
398
399 /*
400  * Return an optimized kprobe whose optimizing code replaces
401  * instructions including addr (exclude breakpoint).
402  */
403 static struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)
404 {
405         int i;
406         struct kprobe *p = NULL;
407         struct optimized_kprobe *op;
408
409         /* Don't check i == 0, since that is a breakpoint case. */
410         for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
411                 p = get_kprobe((void *)(addr - i));
412
413         if (p && kprobe_optready(p)) {
414                 op = container_of(p, struct optimized_kprobe, kp);
415                 if (arch_within_optimized_kprobe(op, addr))
416                         return p;
417         }
418
419         return NULL;
420 }
421
422 /* Optimization staging list, protected by kprobe_mutex */
423 static LIST_HEAD(optimizing_list);
424
425 static void kprobe_optimizer(struct work_struct *work);
426 static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
427 #define OPTIMIZE_DELAY 5
428
429 /* Kprobe jump optimizer */
430 static __kprobes void kprobe_optimizer(struct work_struct *work)
431 {
432         struct optimized_kprobe *op, *tmp;
433
434         /* Lock modules while optimizing kprobes */
435         mutex_lock(&module_mutex);
436         mutex_lock(&kprobe_mutex);
437         if (kprobes_all_disarmed || !kprobes_allow_optimization)
438                 goto end;
439
440         /*
441          * Wait for quiesence period to ensure all running interrupts
442          * are done. Because optprobe may modify multiple instructions
443          * there is a chance that Nth instruction is interrupted. In that
444          * case, running interrupt can return to 2nd-Nth byte of jump
445          * instruction. This wait is for avoiding it.
446          */
447         synchronize_sched();
448
449         /*
450          * The optimization/unoptimization refers online_cpus via
451          * stop_machine() and cpu-hotplug modifies online_cpus.
452          * And same time, text_mutex will be held in cpu-hotplug and here.
453          * This combination can cause a deadlock (cpu-hotplug try to lock
454          * text_mutex but stop_machine can not be done because online_cpus
455          * has been changed)
456          * To avoid this deadlock, we need to call get_online_cpus()
457          * for preventing cpu-hotplug outside of text_mutex locking.
458          */
459         get_online_cpus();
460         mutex_lock(&text_mutex);
461         list_for_each_entry_safe(op, tmp, &optimizing_list, list) {
462                 WARN_ON(kprobe_disabled(&op->kp));
463                 if (arch_optimize_kprobe(op) < 0)
464                         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
465                 list_del_init(&op->list);
466         }
467         mutex_unlock(&text_mutex);
468         put_online_cpus();
469 end:
470         mutex_unlock(&kprobe_mutex);
471         mutex_unlock(&module_mutex);
472 }
473
474 /* Optimize kprobe if p is ready to be optimized */
475 static __kprobes void optimize_kprobe(struct kprobe *p)
476 {
477         struct optimized_kprobe *op;
478
479         /* Check if the kprobe is disabled or not ready for optimization. */
480         if (!kprobe_optready(p) || !kprobes_allow_optimization ||
481             (kprobe_disabled(p) || kprobes_all_disarmed))
482                 return;
483
484         /* Both of break_handler and post_handler are not supported. */
485         if (p->break_handler || p->post_handler)
486                 return;
487
488         op = container_of(p, struct optimized_kprobe, kp);
489
490         /* Check there is no other kprobes at the optimized instructions */
491         if (arch_check_optimized_kprobe(op) < 0)
492                 return;
493
494         /* Check if it is already optimized. */
495         if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
496                 return;
497
498         op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
499         list_add(&op->list, &optimizing_list);
500         if (!delayed_work_pending(&optimizing_work))
501                 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
502 }
503
504 /* Unoptimize a kprobe if p is optimized */
505 static __kprobes void unoptimize_kprobe(struct kprobe *p)
506 {
507         struct optimized_kprobe *op;
508
509         if ((p->flags & KPROBE_FLAG_OPTIMIZED) && kprobe_aggrprobe(p)) {
510                 op = container_of(p, struct optimized_kprobe, kp);
511                 if (!list_empty(&op->list))
512                         /* Dequeue from the optimization queue */
513                         list_del_init(&op->list);
514                 else
515                         /* Replace jump with break */
516                         arch_unoptimize_kprobe(op);
517                 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
518         }
519 }
520
521 /* Remove optimized instructions */
522 static void __kprobes kill_optimized_kprobe(struct kprobe *p)
523 {
524         struct optimized_kprobe *op;
525
526         op = container_of(p, struct optimized_kprobe, kp);
527         if (!list_empty(&op->list)) {
528                 /* Dequeue from the optimization queue */
529                 list_del_init(&op->list);
530                 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
531         }
532         /* Don't unoptimize, because the target code will be freed. */
533         arch_remove_optimized_kprobe(op);
534 }
535
536 /* Try to prepare optimized instructions */
537 static __kprobes void prepare_optimized_kprobe(struct kprobe *p)
538 {
539         struct optimized_kprobe *op;
540
541         op = container_of(p, struct optimized_kprobe, kp);
542         arch_prepare_optimized_kprobe(op);
543 }
544
545 /* Free optimized instructions and optimized_kprobe */
546 static __kprobes void free_aggr_kprobe(struct kprobe *p)
547 {
548         struct optimized_kprobe *op;
549
550         op = container_of(p, struct optimized_kprobe, kp);
551         arch_remove_optimized_kprobe(op);
552         kfree(op);
553 }
554
555 /* Allocate new optimized_kprobe and try to prepare optimized instructions */
556 static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
557 {
558         struct optimized_kprobe *op;
559
560         op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
561         if (!op)
562                 return NULL;
563
564         INIT_LIST_HEAD(&op->list);
565         op->kp.addr = p->addr;
566         arch_prepare_optimized_kprobe(op);
567
568         return &op->kp;
569 }
570
571 static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
572
573 /*
574  * Prepare an optimized_kprobe and optimize it
575  * NOTE: p must be a normal registered kprobe
576  */
577 static __kprobes void try_to_optimize_kprobe(struct kprobe *p)
578 {
579         struct kprobe *ap;
580         struct optimized_kprobe *op;
581
582         ap = alloc_aggr_kprobe(p);
583         if (!ap)
584                 return;
585
586         op = container_of(ap, struct optimized_kprobe, kp);
587         if (!arch_prepared_optinsn(&op->optinsn)) {
588                 /* If failed to setup optimizing, fallback to kprobe */
589                 free_aggr_kprobe(ap);
590                 return;
591         }
592
593         init_aggr_kprobe(ap, p);
594         optimize_kprobe(ap);
595 }
596
597 #ifdef CONFIG_SYSCTL
598 static void __kprobes optimize_all_kprobes(void)
599 {
600         struct hlist_head *head;
601         struct hlist_node *node;
602         struct kprobe *p;
603         unsigned int i;
604
605         /* If optimization is already allowed, just return */
606         if (kprobes_allow_optimization)
607                 return;
608
609         kprobes_allow_optimization = true;
610         mutex_lock(&text_mutex);
611         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
612                 head = &kprobe_table[i];
613                 hlist_for_each_entry_rcu(p, node, head, hlist)
614                         if (!kprobe_disabled(p))
615                                 optimize_kprobe(p);
616         }
617         mutex_unlock(&text_mutex);
618         printk(KERN_INFO "Kprobes globally optimized\n");
619 }
620
621 static void __kprobes unoptimize_all_kprobes(void)
622 {
623         struct hlist_head *head;
624         struct hlist_node *node;
625         struct kprobe *p;
626         unsigned int i;
627
628         /* If optimization is already prohibited, just return */
629         if (!kprobes_allow_optimization)
630                 return;
631
632         kprobes_allow_optimization = false;
633         printk(KERN_INFO "Kprobes globally unoptimized\n");
634         get_online_cpus();      /* For avoiding text_mutex deadlock */
635         mutex_lock(&text_mutex);
636         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
637                 head = &kprobe_table[i];
638                 hlist_for_each_entry_rcu(p, node, head, hlist) {
639                         if (!kprobe_disabled(p))
640                                 unoptimize_kprobe(p);
641                 }
642         }
643
644         mutex_unlock(&text_mutex);
645         put_online_cpus();
646         /* Allow all currently running kprobes to complete */
647         synchronize_sched();
648 }
649
650 int sysctl_kprobes_optimization;
651 int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
652                                       void __user *buffer, size_t *length,
653                                       loff_t *ppos)
654 {
655         int ret;
656
657         mutex_lock(&kprobe_mutex);
658         sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
659         ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
660
661         if (sysctl_kprobes_optimization)
662                 optimize_all_kprobes();
663         else
664                 unoptimize_all_kprobes();
665         mutex_unlock(&kprobe_mutex);
666
667         return ret;
668 }
669 #endif /* CONFIG_SYSCTL */
670
671 static void __kprobes __arm_kprobe(struct kprobe *p)
672 {
673         struct kprobe *old_p;
674
675         /* Check collision with other optimized kprobes */
676         old_p = get_optimized_kprobe((unsigned long)p->addr);
677         if (unlikely(old_p))
678                 unoptimize_kprobe(old_p); /* Fallback to unoptimized kprobe */
679
680         arch_arm_kprobe(p);
681         optimize_kprobe(p);     /* Try to optimize (add kprobe to a list) */
682 }
683
684 static void __kprobes __disarm_kprobe(struct kprobe *p)
685 {
686         struct kprobe *old_p;
687
688         unoptimize_kprobe(p);   /* Try to unoptimize */
689         arch_disarm_kprobe(p);
690
691         /* If another kprobe was blocked, optimize it. */
692         old_p = get_optimized_kprobe((unsigned long)p->addr);
693         if (unlikely(old_p))
694                 optimize_kprobe(old_p);
695 }
696
697 #else /* !CONFIG_OPTPROBES */
698
699 #define optimize_kprobe(p)                      do {} while (0)
700 #define unoptimize_kprobe(p)                    do {} while (0)
701 #define kill_optimized_kprobe(p)                do {} while (0)
702 #define prepare_optimized_kprobe(p)             do {} while (0)
703 #define try_to_optimize_kprobe(p)               do {} while (0)
704 #define __arm_kprobe(p)                         arch_arm_kprobe(p)
705 #define __disarm_kprobe(p)                      arch_disarm_kprobe(p)
706
707 static __kprobes void free_aggr_kprobe(struct kprobe *p)
708 {
709         kfree(p);
710 }
711
712 static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
713 {
714         return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
715 }
716 #endif /* CONFIG_OPTPROBES */
717
718 /* Arm a kprobe with text_mutex */
719 static void __kprobes arm_kprobe(struct kprobe *kp)
720 {
721         /*
722          * Here, since __arm_kprobe() doesn't use stop_machine(),
723          * this doesn't cause deadlock on text_mutex. So, we don't
724          * need get_online_cpus().
725          */
726         mutex_lock(&text_mutex);
727         __arm_kprobe(kp);
728         mutex_unlock(&text_mutex);
729 }
730
731 /* Disarm a kprobe with text_mutex */
732 static void __kprobes disarm_kprobe(struct kprobe *kp)
733 {
734         get_online_cpus();      /* For avoiding text_mutex deadlock */
735         mutex_lock(&text_mutex);
736         __disarm_kprobe(kp);
737         mutex_unlock(&text_mutex);
738         put_online_cpus();
739 }
740
741 /*
742  * Aggregate handlers for multiple kprobes support - these handlers
743  * take care of invoking the individual kprobe handlers on p->list
744  */
745 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
746 {
747         struct kprobe *kp;
748
749         list_for_each_entry_rcu(kp, &p->list, list) {
750                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
751                         set_kprobe_instance(kp);
752                         if (kp->pre_handler(kp, regs))
753                                 return 1;
754                 }
755                 reset_kprobe_instance();
756         }
757         return 0;
758 }
759
760 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
761                                         unsigned long flags)
762 {
763         struct kprobe *kp;
764
765         list_for_each_entry_rcu(kp, &p->list, list) {
766                 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
767                         set_kprobe_instance(kp);
768                         kp->post_handler(kp, regs, flags);
769                         reset_kprobe_instance();
770                 }
771         }
772 }
773
774 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
775                                         int trapnr)
776 {
777         struct kprobe *cur = __get_cpu_var(kprobe_instance);
778
779         /*
780          * if we faulted "during" the execution of a user specified
781          * probe handler, invoke just that probe's fault handler
782          */
783         if (cur && cur->fault_handler) {
784                 if (cur->fault_handler(cur, regs, trapnr))
785                         return 1;
786         }
787         return 0;
788 }
789
790 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
791 {
792         struct kprobe *cur = __get_cpu_var(kprobe_instance);
793         int ret = 0;
794
795         if (cur && cur->break_handler) {
796                 if (cur->break_handler(cur, regs))
797                         ret = 1;
798         }
799         reset_kprobe_instance();
800         return ret;
801 }
802
803 /* Walks the list and increments nmissed count for multiprobe case */
804 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
805 {
806         struct kprobe *kp;
807         if (!kprobe_aggrprobe(p)) {
808                 p->nmissed++;
809         } else {
810                 list_for_each_entry_rcu(kp, &p->list, list)
811                         kp->nmissed++;
812         }
813         return;
814 }
815
816 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
817                                 struct hlist_head *head)
818 {
819         struct kretprobe *rp = ri->rp;
820
821         /* remove rp inst off the rprobe_inst_table */
822         hlist_del(&ri->hlist);
823         INIT_HLIST_NODE(&ri->hlist);
824         if (likely(rp)) {
825                 spin_lock(&rp->lock);
826                 hlist_add_head(&ri->hlist, &rp->free_instances);
827                 spin_unlock(&rp->lock);
828         } else
829                 /* Unregistering */
830                 hlist_add_head(&ri->hlist, head);
831 }
832
833 void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
834                          struct hlist_head **head, unsigned long *flags)
835 __acquires(hlist_lock)
836 {
837         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
838         spinlock_t *hlist_lock;
839
840         *head = &kretprobe_inst_table[hash];
841         hlist_lock = kretprobe_table_lock_ptr(hash);
842         spin_lock_irqsave(hlist_lock, *flags);
843 }
844
845 static void __kprobes kretprobe_table_lock(unsigned long hash,
846         unsigned long *flags)
847 __acquires(hlist_lock)
848 {
849         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
850         spin_lock_irqsave(hlist_lock, *flags);
851 }
852
853 void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
854         unsigned long *flags)
855 __releases(hlist_lock)
856 {
857         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
858         spinlock_t *hlist_lock;
859
860         hlist_lock = kretprobe_table_lock_ptr(hash);
861         spin_unlock_irqrestore(hlist_lock, *flags);
862 }
863
864 static void __kprobes kretprobe_table_unlock(unsigned long hash,
865        unsigned long *flags)
866 __releases(hlist_lock)
867 {
868         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
869         spin_unlock_irqrestore(hlist_lock, *flags);
870 }
871
872 /*
873  * This function is called from finish_task_switch when task tk becomes dead,
874  * so that we can recycle any function-return probe instances associated
875  * with this task. These left over instances represent probed functions
876  * that have been called but will never return.
877  */
878 void __kprobes kprobe_flush_task(struct task_struct *tk)
879 {
880         struct kretprobe_instance *ri;
881         struct hlist_head *head, empty_rp;
882         struct hlist_node *node, *tmp;
883         unsigned long hash, flags = 0;
884
885         if (unlikely(!kprobes_initialized))
886                 /* Early boot.  kretprobe_table_locks not yet initialized. */
887                 return;
888
889         hash = hash_ptr(tk, KPROBE_HASH_BITS);
890         head = &kretprobe_inst_table[hash];
891         kretprobe_table_lock(hash, &flags);
892         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
893                 if (ri->task == tk)
894                         recycle_rp_inst(ri, &empty_rp);
895         }
896         kretprobe_table_unlock(hash, &flags);
897         INIT_HLIST_HEAD(&empty_rp);
898         hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
899                 hlist_del(&ri->hlist);
900                 kfree(ri);
901         }
902 }
903
904 static inline void free_rp_inst(struct kretprobe *rp)
905 {
906         struct kretprobe_instance *ri;
907         struct hlist_node *pos, *next;
908
909         hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
910                 hlist_del(&ri->hlist);
911                 kfree(ri);
912         }
913 }
914
915 static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
916 {
917         unsigned long flags, hash;
918         struct kretprobe_instance *ri;
919         struct hlist_node *pos, *next;
920         struct hlist_head *head;
921
922         /* No race here */
923         for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
924                 kretprobe_table_lock(hash, &flags);
925                 head = &kretprobe_inst_table[hash];
926                 hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
927                         if (ri->rp == rp)
928                                 ri->rp = NULL;
929                 }
930                 kretprobe_table_unlock(hash, &flags);
931         }
932         free_rp_inst(rp);
933 }
934
935 /*
936 * Add the new probe to ap->list. Fail if this is the
937 * second jprobe at the address - two jprobes can't coexist
938 */
939 static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
940 {
941         BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
942
943         if (p->break_handler || p->post_handler)
944                 unoptimize_kprobe(ap);  /* Fall back to normal kprobe */
945
946         if (p->break_handler) {
947                 if (ap->break_handler)
948                         return -EEXIST;
949                 list_add_tail_rcu(&p->list, &ap->list);
950                 ap->break_handler = aggr_break_handler;
951         } else
952                 list_add_rcu(&p->list, &ap->list);
953         if (p->post_handler && !ap->post_handler)
954                 ap->post_handler = aggr_post_handler;
955
956         if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
957                 ap->flags &= ~KPROBE_FLAG_DISABLED;
958                 if (!kprobes_all_disarmed)
959                         /* Arm the breakpoint again. */
960                         __arm_kprobe(ap);
961         }
962         return 0;
963 }
964
965 /*
966  * Fill in the required fields of the "manager kprobe". Replace the
967  * earlier kprobe in the hlist with the manager kprobe
968  */
969 static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
970 {
971         /* Copy p's insn slot to ap */
972         copy_kprobe(p, ap);
973         flush_insn_slot(ap);
974         ap->addr = p->addr;
975         ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
976         ap->pre_handler = aggr_pre_handler;
977         ap->fault_handler = aggr_fault_handler;
978         /* We don't care the kprobe which has gone. */
979         if (p->post_handler && !kprobe_gone(p))
980                 ap->post_handler = aggr_post_handler;
981         if (p->break_handler && !kprobe_gone(p))
982                 ap->break_handler = aggr_break_handler;
983
984         INIT_LIST_HEAD(&ap->list);
985         INIT_HLIST_NODE(&ap->hlist);
986
987         list_add_rcu(&p->list, &ap->list);
988         hlist_replace_rcu(&p->hlist, &ap->hlist);
989 }
990
991 /*
992  * This is the second or subsequent kprobe at the address - handle
993  * the intricacies
994  */
995 static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
996                                           struct kprobe *p)
997 {
998         int ret = 0;
999         struct kprobe *ap = old_p;
1000
1001         if (!kprobe_aggrprobe(old_p)) {
1002                 /* If old_p is not an aggr_kprobe, create new aggr_kprobe. */
1003                 ap = alloc_aggr_kprobe(old_p);
1004                 if (!ap)
1005                         return -ENOMEM;
1006                 init_aggr_kprobe(ap, old_p);
1007         }
1008
1009         if (kprobe_gone(ap)) {
1010                 /*
1011                  * Attempting to insert new probe at the same location that
1012                  * had a probe in the module vaddr area which already
1013                  * freed. So, the instruction slot has already been
1014                  * released. We need a new slot for the new probe.
1015                  */
1016                 ret = arch_prepare_kprobe(ap);
1017                 if (ret)
1018                         /*
1019                          * Even if fail to allocate new slot, don't need to
1020                          * free aggr_probe. It will be used next time, or
1021                          * freed by unregister_kprobe.
1022                          */
1023                         return ret;
1024
1025                 /* Prepare optimized instructions if possible. */
1026                 prepare_optimized_kprobe(ap);
1027
1028                 /*
1029                  * Clear gone flag to prevent allocating new slot again, and
1030                  * set disabled flag because it is not armed yet.
1031                  */
1032                 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1033                             | KPROBE_FLAG_DISABLED;
1034         }
1035
1036         /* Copy ap's insn slot to p */
1037         copy_kprobe(ap, p);
1038         return add_new_kprobe(ap, p);
1039 }
1040
1041 /* Try to disable aggr_kprobe, and return 1 if succeeded.*/
1042 static int __kprobes try_to_disable_aggr_kprobe(struct kprobe *p)
1043 {
1044         struct kprobe *kp;
1045
1046         list_for_each_entry_rcu(kp, &p->list, list) {
1047                 if (!kprobe_disabled(kp))
1048                         /*
1049                          * There is an active probe on the list.
1050                          * We can't disable aggr_kprobe.
1051                          */
1052                         return 0;
1053         }
1054         p->flags |= KPROBE_FLAG_DISABLED;
1055         return 1;
1056 }
1057
1058 static int __kprobes in_kprobes_functions(unsigned long addr)
1059 {
1060         struct kprobe_blackpoint *kb;
1061
1062         if (addr >= (unsigned long)__kprobes_text_start &&
1063             addr < (unsigned long)__kprobes_text_end)
1064                 return -EINVAL;
1065         /*
1066          * If there exists a kprobe_blacklist, verify and
1067          * fail any probe registration in the prohibited area
1068          */
1069         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1070                 if (kb->start_addr) {
1071                         if (addr >= kb->start_addr &&
1072                             addr < (kb->start_addr + kb->range))
1073                                 return -EINVAL;
1074                 }
1075         }
1076         return 0;
1077 }
1078
1079 /*
1080  * If we have a symbol_name argument, look it up and add the offset field
1081  * to it. This way, we can specify a relative address to a symbol.
1082  */
1083 static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
1084 {
1085         kprobe_opcode_t *addr = p->addr;
1086         if (p->symbol_name) {
1087                 if (addr)
1088                         return NULL;
1089                 kprobe_lookup_name(p->symbol_name, addr);
1090         }
1091
1092         if (!addr)
1093                 return NULL;
1094         return (kprobe_opcode_t *)(((char *)addr) + p->offset);
1095 }
1096
1097 /* Check passed kprobe is valid and return kprobe in kprobe_table. */
1098 static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
1099 {
1100         struct kprobe *old_p, *list_p;
1101
1102         old_p = get_kprobe(p->addr);
1103         if (unlikely(!old_p))
1104                 return NULL;
1105
1106         if (p != old_p) {
1107                 list_for_each_entry_rcu(list_p, &old_p->list, list)
1108                         if (list_p == p)
1109                         /* kprobe p is a valid probe */
1110                                 goto valid;
1111                 return NULL;
1112         }
1113 valid:
1114         return old_p;
1115 }
1116
1117 /* Return error if the kprobe is being re-registered */
1118 static inline int check_kprobe_rereg(struct kprobe *p)
1119 {
1120         int ret = 0;
1121         struct kprobe *old_p;
1122
1123         mutex_lock(&kprobe_mutex);
1124         old_p = __get_valid_kprobe(p);
1125         if (old_p)
1126                 ret = -EINVAL;
1127         mutex_unlock(&kprobe_mutex);
1128         return ret;
1129 }
1130
1131 int __kprobes register_kprobe(struct kprobe *p)
1132 {
1133         int ret = 0;
1134         struct kprobe *old_p;
1135         struct module *probed_mod;
1136         kprobe_opcode_t *addr;
1137
1138         addr = kprobe_addr(p);
1139         if (!addr)
1140                 return -EINVAL;
1141         p->addr = addr;
1142
1143         ret = check_kprobe_rereg(p);
1144         if (ret)
1145                 return ret;
1146
1147         preempt_disable();
1148         if (!kernel_text_address((unsigned long) p->addr) ||
1149             in_kprobes_functions((unsigned long) p->addr) ||
1150             ftrace_text_reserved(p->addr, p->addr) ||
1151             jump_label_text_reserved(p->addr, p->addr)) {
1152                 preempt_enable();
1153                 return -EINVAL;
1154         }
1155
1156         /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1157         p->flags &= KPROBE_FLAG_DISABLED;
1158
1159         /*
1160          * Check if are we probing a module.
1161          */
1162         probed_mod = __module_text_address((unsigned long) p->addr);
1163         if (probed_mod) {
1164                 /*
1165                  * We must hold a refcount of the probed module while updating
1166                  * its code to prohibit unexpected unloading.
1167                  */
1168                 if (unlikely(!try_module_get(probed_mod))) {
1169                         preempt_enable();
1170                         return -EINVAL;
1171                 }
1172                 /*
1173                  * If the module freed .init.text, we couldn't insert
1174                  * kprobes in there.
1175                  */
1176                 if (within_module_init((unsigned long)p->addr, probed_mod) &&
1177                     probed_mod->state != MODULE_STATE_COMING) {
1178                         module_put(probed_mod);
1179                         preempt_enable();
1180                         return -EINVAL;
1181                 }
1182         }
1183         preempt_enable();
1184
1185         p->nmissed = 0;
1186         INIT_LIST_HEAD(&p->list);
1187         mutex_lock(&kprobe_mutex);
1188
1189         get_online_cpus();      /* For avoiding text_mutex deadlock. */
1190         mutex_lock(&text_mutex);
1191
1192         old_p = get_kprobe(p->addr);
1193         if (old_p) {
1194                 /* Since this may unoptimize old_p, locking text_mutex. */
1195                 ret = register_aggr_kprobe(old_p, p);
1196                 goto out;
1197         }
1198
1199         ret = arch_prepare_kprobe(p);
1200         if (ret)
1201                 goto out;
1202
1203         INIT_HLIST_NODE(&p->hlist);
1204         hlist_add_head_rcu(&p->hlist,
1205                        &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1206
1207         if (!kprobes_all_disarmed && !kprobe_disabled(p))
1208                 __arm_kprobe(p);
1209
1210         /* Try to optimize kprobe */
1211         try_to_optimize_kprobe(p);
1212
1213 out:
1214         mutex_unlock(&text_mutex);
1215         put_online_cpus();
1216         mutex_unlock(&kprobe_mutex);
1217
1218         if (probed_mod)
1219                 module_put(probed_mod);
1220
1221         return ret;
1222 }
1223 EXPORT_SYMBOL_GPL(register_kprobe);
1224
1225 /*
1226  * Unregister a kprobe without a scheduler synchronization.
1227  */
1228 static int __kprobes __unregister_kprobe_top(struct kprobe *p)
1229 {
1230         struct kprobe *old_p, *list_p;
1231
1232         old_p = __get_valid_kprobe(p);
1233         if (old_p == NULL)
1234                 return -EINVAL;
1235
1236         if (old_p == p ||
1237             (kprobe_aggrprobe(old_p) &&
1238              list_is_singular(&old_p->list))) {
1239                 /*
1240                  * Only probe on the hash list. Disarm only if kprobes are
1241                  * enabled and not gone - otherwise, the breakpoint would
1242                  * already have been removed. We save on flushing icache.
1243                  */
1244                 if (!kprobes_all_disarmed && !kprobe_disabled(old_p))
1245                         disarm_kprobe(old_p);
1246                 hlist_del_rcu(&old_p->hlist);
1247         } else {
1248                 if (p->break_handler && !kprobe_gone(p))
1249                         old_p->break_handler = NULL;
1250                 if (p->post_handler && !kprobe_gone(p)) {
1251                         list_for_each_entry_rcu(list_p, &old_p->list, list) {
1252                                 if ((list_p != p) && (list_p->post_handler))
1253                                         goto noclean;
1254                         }
1255                         old_p->post_handler = NULL;
1256                 }
1257 noclean:
1258                 list_del_rcu(&p->list);
1259                 if (!kprobe_disabled(old_p)) {
1260                         try_to_disable_aggr_kprobe(old_p);
1261                         if (!kprobes_all_disarmed) {
1262                                 if (kprobe_disabled(old_p))
1263                                         disarm_kprobe(old_p);
1264                                 else
1265                                         /* Try to optimize this probe again */
1266                                         optimize_kprobe(old_p);
1267                         }
1268                 }
1269         }
1270         return 0;
1271 }
1272
1273 static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
1274 {
1275         struct kprobe *old_p;
1276
1277         if (list_empty(&p->list))
1278                 arch_remove_kprobe(p);
1279         else if (list_is_singular(&p->list)) {
1280                 /* "p" is the last child of an aggr_kprobe */
1281                 old_p = list_entry(p->list.next, struct kprobe, list);
1282                 list_del(&p->list);
1283                 arch_remove_kprobe(old_p);
1284                 free_aggr_kprobe(old_p);
1285         }
1286 }
1287
1288 int __kprobes register_kprobes(struct kprobe **kps, int num)
1289 {
1290         int i, ret = 0;
1291
1292         if (num <= 0)
1293                 return -EINVAL;
1294         for (i = 0; i < num; i++) {
1295                 ret = register_kprobe(kps[i]);
1296                 if (ret < 0) {
1297                         if (i > 0)
1298                                 unregister_kprobes(kps, i);
1299                         break;
1300                 }
1301         }
1302         return ret;
1303 }
1304 EXPORT_SYMBOL_GPL(register_kprobes);
1305
1306 void __kprobes unregister_kprobe(struct kprobe *p)
1307 {
1308         unregister_kprobes(&p, 1);
1309 }
1310 EXPORT_SYMBOL_GPL(unregister_kprobe);
1311
1312 void __kprobes unregister_kprobes(struct kprobe **kps, int num)
1313 {
1314         int i;
1315
1316         if (num <= 0)
1317                 return;
1318         mutex_lock(&kprobe_mutex);
1319         for (i = 0; i < num; i++)
1320                 if (__unregister_kprobe_top(kps[i]) < 0)
1321                         kps[i]->addr = NULL;
1322         mutex_unlock(&kprobe_mutex);
1323
1324         synchronize_sched();
1325         for (i = 0; i < num; i++)
1326                 if (kps[i]->addr)
1327                         __unregister_kprobe_bottom(kps[i]);
1328 }
1329 EXPORT_SYMBOL_GPL(unregister_kprobes);
1330
1331 static struct notifier_block kprobe_exceptions_nb = {
1332         .notifier_call = kprobe_exceptions_notify,
1333         .priority = 0x7fffffff /* we need to be notified first */
1334 };
1335
1336 unsigned long __weak arch_deref_entry_point(void *entry)
1337 {
1338         return (unsigned long)entry;
1339 }
1340
1341 int __kprobes register_jprobes(struct jprobe **jps, int num)
1342 {
1343         struct jprobe *jp;
1344         int ret = 0, i;
1345
1346         if (num <= 0)
1347                 return -EINVAL;
1348         for (i = 0; i < num; i++) {
1349                 unsigned long addr, offset;
1350                 jp = jps[i];
1351                 addr = arch_deref_entry_point(jp->entry);
1352
1353                 /* Verify probepoint is a function entry point */
1354                 if (kallsyms_lookup_size_offset(addr, NULL, &offset) &&
1355                     offset == 0) {
1356                         jp->kp.pre_handler = setjmp_pre_handler;
1357                         jp->kp.break_handler = longjmp_break_handler;
1358                         ret = register_kprobe(&jp->kp);
1359                 } else
1360                         ret = -EINVAL;
1361
1362                 if (ret < 0) {
1363                         if (i > 0)
1364                                 unregister_jprobes(jps, i);
1365                         break;
1366                 }
1367         }
1368         return ret;
1369 }
1370 EXPORT_SYMBOL_GPL(register_jprobes);
1371
1372 int __kprobes register_jprobe(struct jprobe *jp)
1373 {
1374         return register_jprobes(&jp, 1);
1375 }
1376 EXPORT_SYMBOL_GPL(register_jprobe);
1377
1378 void __kprobes unregister_jprobe(struct jprobe *jp)
1379 {
1380         unregister_jprobes(&jp, 1);
1381 }
1382 EXPORT_SYMBOL_GPL(unregister_jprobe);
1383
1384 void __kprobes unregister_jprobes(struct jprobe **jps, int num)
1385 {
1386         int i;
1387
1388         if (num <= 0)
1389                 return;
1390         mutex_lock(&kprobe_mutex);
1391         for (i = 0; i < num; i++)
1392                 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1393                         jps[i]->kp.addr = NULL;
1394         mutex_unlock(&kprobe_mutex);
1395
1396         synchronize_sched();
1397         for (i = 0; i < num; i++) {
1398                 if (jps[i]->kp.addr)
1399                         __unregister_kprobe_bottom(&jps[i]->kp);
1400         }
1401 }
1402 EXPORT_SYMBOL_GPL(unregister_jprobes);
1403
1404 #ifdef CONFIG_KRETPROBES
1405 /*
1406  * This kprobe pre_handler is registered with every kretprobe. When probe
1407  * hits it will set up the return probe.
1408  */
1409 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1410                                            struct pt_regs *regs)
1411 {
1412         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1413         unsigned long hash, flags = 0;
1414         struct kretprobe_instance *ri;
1415
1416         /*TODO: consider to only swap the RA after the last pre_handler fired */
1417         hash = hash_ptr(current, KPROBE_HASH_BITS);
1418         spin_lock_irqsave(&rp->lock, flags);
1419         if (!hlist_empty(&rp->free_instances)) {
1420                 ri = hlist_entry(rp->free_instances.first,
1421                                 struct kretprobe_instance, hlist);
1422                 hlist_del(&ri->hlist);
1423                 spin_unlock_irqrestore(&rp->lock, flags);
1424
1425                 ri->rp = rp;
1426                 ri->task = current;
1427
1428                 if (rp->entry_handler && rp->entry_handler(ri, regs))
1429                         return 0;
1430
1431                 arch_prepare_kretprobe(ri, regs);
1432
1433                 /* XXX(hch): why is there no hlist_move_head? */
1434                 INIT_HLIST_NODE(&ri->hlist);
1435                 kretprobe_table_lock(hash, &flags);
1436                 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1437                 kretprobe_table_unlock(hash, &flags);
1438         } else {
1439                 rp->nmissed++;
1440                 spin_unlock_irqrestore(&rp->lock, flags);
1441         }
1442         return 0;
1443 }
1444
1445 int __kprobes register_kretprobe(struct kretprobe *rp)
1446 {
1447         int ret = 0;
1448         struct kretprobe_instance *inst;
1449         int i;
1450         void *addr;
1451
1452         if (kretprobe_blacklist_size) {
1453                 addr = kprobe_addr(&rp->kp);
1454                 if (!addr)
1455                         return -EINVAL;
1456
1457                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1458                         if (kretprobe_blacklist[i].addr == addr)
1459                                 return -EINVAL;
1460                 }
1461         }
1462
1463         rp->kp.pre_handler = pre_handler_kretprobe;
1464         rp->kp.post_handler = NULL;
1465         rp->kp.fault_handler = NULL;
1466         rp->kp.break_handler = NULL;
1467
1468         /* Pre-allocate memory for max kretprobe instances */
1469         if (rp->maxactive <= 0) {
1470 #ifdef CONFIG_PREEMPT
1471                 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
1472 #else
1473                 rp->maxactive = num_possible_cpus();
1474 #endif
1475         }
1476         spin_lock_init(&rp->lock);
1477         INIT_HLIST_HEAD(&rp->free_instances);
1478         for (i = 0; i < rp->maxactive; i++) {
1479                 inst = kmalloc(sizeof(struct kretprobe_instance) +
1480                                rp->data_size, GFP_KERNEL);
1481                 if (inst == NULL) {
1482                         free_rp_inst(rp);
1483                         return -ENOMEM;
1484                 }
1485                 INIT_HLIST_NODE(&inst->hlist);
1486                 hlist_add_head(&inst->hlist, &rp->free_instances);
1487         }
1488
1489         rp->nmissed = 0;
1490         /* Establish function entry probe point */
1491         ret = register_kprobe(&rp->kp);
1492         if (ret != 0)
1493                 free_rp_inst(rp);
1494         return ret;
1495 }
1496 EXPORT_SYMBOL_GPL(register_kretprobe);
1497
1498 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1499 {
1500         int ret = 0, i;
1501
1502         if (num <= 0)
1503                 return -EINVAL;
1504         for (i = 0; i < num; i++) {
1505                 ret = register_kretprobe(rps[i]);
1506                 if (ret < 0) {
1507                         if (i > 0)
1508                                 unregister_kretprobes(rps, i);
1509                         break;
1510                 }
1511         }
1512         return ret;
1513 }
1514 EXPORT_SYMBOL_GPL(register_kretprobes);
1515
1516 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1517 {
1518         unregister_kretprobes(&rp, 1);
1519 }
1520 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1521
1522 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1523 {
1524         int i;
1525
1526         if (num <= 0)
1527                 return;
1528         mutex_lock(&kprobe_mutex);
1529         for (i = 0; i < num; i++)
1530                 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1531                         rps[i]->kp.addr = NULL;
1532         mutex_unlock(&kprobe_mutex);
1533
1534         synchronize_sched();
1535         for (i = 0; i < num; i++) {
1536                 if (rps[i]->kp.addr) {
1537                         __unregister_kprobe_bottom(&rps[i]->kp);
1538                         cleanup_rp_inst(rps[i]);
1539                 }
1540         }
1541 }
1542 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1543
1544 #else /* CONFIG_KRETPROBES */
1545 int __kprobes register_kretprobe(struct kretprobe *rp)
1546 {
1547         return -ENOSYS;
1548 }
1549 EXPORT_SYMBOL_GPL(register_kretprobe);
1550
1551 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1552 {
1553         return -ENOSYS;
1554 }
1555 EXPORT_SYMBOL_GPL(register_kretprobes);
1556
1557 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1558 {
1559 }
1560 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1561
1562 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1563 {
1564 }
1565 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1566
1567 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1568                                            struct pt_regs *regs)
1569 {
1570         return 0;
1571 }
1572
1573 #endif /* CONFIG_KRETPROBES */
1574
1575 /* Set the kprobe gone and remove its instruction buffer. */
1576 static void __kprobes kill_kprobe(struct kprobe *p)
1577 {
1578         struct kprobe *kp;
1579
1580         p->flags |= KPROBE_FLAG_GONE;
1581         if (kprobe_aggrprobe(p)) {
1582                 /*
1583                  * If this is an aggr_kprobe, we have to list all the
1584                  * chained probes and mark them GONE.
1585                  */
1586                 list_for_each_entry_rcu(kp, &p->list, list)
1587                         kp->flags |= KPROBE_FLAG_GONE;
1588                 p->post_handler = NULL;
1589                 p->break_handler = NULL;
1590                 kill_optimized_kprobe(p);
1591         }
1592         /*
1593          * Here, we can remove insn_slot safely, because no thread calls
1594          * the original probed function (which will be freed soon) any more.
1595          */
1596         arch_remove_kprobe(p);
1597 }
1598
1599 /* Disable one kprobe */
1600 int __kprobes disable_kprobe(struct kprobe *kp)
1601 {
1602         int ret = 0;
1603         struct kprobe *p;
1604
1605         mutex_lock(&kprobe_mutex);
1606
1607         /* Check whether specified probe is valid. */
1608         p = __get_valid_kprobe(kp);
1609         if (unlikely(p == NULL)) {
1610                 ret = -EINVAL;
1611                 goto out;
1612         }
1613
1614         /* If the probe is already disabled (or gone), just return */
1615         if (kprobe_disabled(kp))
1616                 goto out;
1617
1618         kp->flags |= KPROBE_FLAG_DISABLED;
1619         if (p != kp)
1620                 /* When kp != p, p is always enabled. */
1621                 try_to_disable_aggr_kprobe(p);
1622
1623         if (!kprobes_all_disarmed && kprobe_disabled(p))
1624                 disarm_kprobe(p);
1625 out:
1626         mutex_unlock(&kprobe_mutex);
1627         return ret;
1628 }
1629 EXPORT_SYMBOL_GPL(disable_kprobe);
1630
1631 /* Enable one kprobe */
1632 int __kprobes enable_kprobe(struct kprobe *kp)
1633 {
1634         int ret = 0;
1635         struct kprobe *p;
1636
1637         mutex_lock(&kprobe_mutex);
1638
1639         /* Check whether specified probe is valid. */
1640         p = __get_valid_kprobe(kp);
1641         if (unlikely(p == NULL)) {
1642                 ret = -EINVAL;
1643                 goto out;
1644         }
1645
1646         if (kprobe_gone(kp)) {
1647                 /* This kprobe has gone, we couldn't enable it. */
1648                 ret = -EINVAL;
1649                 goto out;
1650         }
1651
1652         if (p != kp)
1653                 kp->flags &= ~KPROBE_FLAG_DISABLED;
1654
1655         if (!kprobes_all_disarmed && kprobe_disabled(p)) {
1656                 p->flags &= ~KPROBE_FLAG_DISABLED;
1657                 arm_kprobe(p);
1658         }
1659 out:
1660         mutex_unlock(&kprobe_mutex);
1661         return ret;
1662 }
1663 EXPORT_SYMBOL_GPL(enable_kprobe);
1664
1665 void __kprobes dump_kprobe(struct kprobe *kp)
1666 {
1667         printk(KERN_WARNING "Dumping kprobe:\n");
1668         printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
1669                kp->symbol_name, kp->addr, kp->offset);
1670 }
1671
1672 /* Module notifier call back, checking kprobes on the module */
1673 static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1674                                              unsigned long val, void *data)
1675 {
1676         struct module *mod = data;
1677         struct hlist_head *head;
1678         struct hlist_node *node;
1679         struct kprobe *p;
1680         unsigned int i;
1681         int checkcore = (val == MODULE_STATE_GOING);
1682
1683         if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1684                 return NOTIFY_DONE;
1685
1686         /*
1687          * When MODULE_STATE_GOING was notified, both of module .text and
1688          * .init.text sections would be freed. When MODULE_STATE_LIVE was
1689          * notified, only .init.text section would be freed. We need to
1690          * disable kprobes which have been inserted in the sections.
1691          */
1692         mutex_lock(&kprobe_mutex);
1693         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1694                 head = &kprobe_table[i];
1695                 hlist_for_each_entry_rcu(p, node, head, hlist)
1696                         if (within_module_init((unsigned long)p->addr, mod) ||
1697                             (checkcore &&
1698                              within_module_core((unsigned long)p->addr, mod))) {
1699                                 /*
1700                                  * The vaddr this probe is installed will soon
1701                                  * be vfreed buy not synced to disk. Hence,
1702                                  * disarming the breakpoint isn't needed.
1703                                  */
1704                                 kill_kprobe(p);
1705                         }
1706         }
1707         mutex_unlock(&kprobe_mutex);
1708         return NOTIFY_DONE;
1709 }
1710
1711 static struct notifier_block kprobe_module_nb = {
1712         .notifier_call = kprobes_module_callback,
1713         .priority = 0
1714 };
1715
1716 static int __init init_kprobes(void)
1717 {
1718         int i, err = 0;
1719         unsigned long offset = 0, size = 0;
1720         char *modname, namebuf[128];
1721         const char *symbol_name;
1722         void *addr;
1723         struct kprobe_blackpoint *kb;
1724
1725         /* FIXME allocate the probe table, currently defined statically */
1726         /* initialize all list heads */
1727         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1728                 INIT_HLIST_HEAD(&kprobe_table[i]);
1729                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1730                 spin_lock_init(&(kretprobe_table_locks[i].lock));
1731         }
1732
1733         /*
1734          * Lookup and populate the kprobe_blacklist.
1735          *
1736          * Unlike the kretprobe blacklist, we'll need to determine
1737          * the range of addresses that belong to the said functions,
1738          * since a kprobe need not necessarily be at the beginning
1739          * of a function.
1740          */
1741         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1742                 kprobe_lookup_name(kb->name, addr);
1743                 if (!addr)
1744                         continue;
1745
1746                 kb->start_addr = (unsigned long)addr;
1747                 symbol_name = kallsyms_lookup(kb->start_addr,
1748                                 &size, &offset, &modname, namebuf);
1749                 if (!symbol_name)
1750                         kb->range = 0;
1751                 else
1752                         kb->range = size;
1753         }
1754
1755         if (kretprobe_blacklist_size) {
1756                 /* lookup the function address from its name */
1757                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1758                         kprobe_lookup_name(kretprobe_blacklist[i].name,
1759                                            kretprobe_blacklist[i].addr);
1760                         if (!kretprobe_blacklist[i].addr)
1761                                 printk("kretprobe: lookup failed: %s\n",
1762                                        kretprobe_blacklist[i].name);
1763                 }
1764         }
1765
1766 #if defined(CONFIG_OPTPROBES)
1767 #if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
1768         /* Init kprobe_optinsn_slots */
1769         kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
1770 #endif
1771         /* By default, kprobes can be optimized */
1772         kprobes_allow_optimization = true;
1773 #endif
1774
1775         /* By default, kprobes are armed */
1776         kprobes_all_disarmed = false;
1777
1778         err = arch_init_kprobes();
1779         if (!err)
1780                 err = register_die_notifier(&kprobe_exceptions_nb);
1781         if (!err)
1782                 err = register_module_notifier(&kprobe_module_nb);
1783
1784         kprobes_initialized = (err == 0);
1785
1786         if (!err)
1787                 init_test_probes();
1788         return err;
1789 }
1790
1791 #ifdef CONFIG_DEBUG_FS
1792 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
1793                 const char *sym, int offset, char *modname, struct kprobe *pp)
1794 {
1795         char *kprobe_type;
1796
1797         if (p->pre_handler == pre_handler_kretprobe)
1798                 kprobe_type = "r";
1799         else if (p->pre_handler == setjmp_pre_handler)
1800                 kprobe_type = "j";
1801         else
1802                 kprobe_type = "k";
1803
1804         if (sym)
1805                 seq_printf(pi, "%p  %s  %s+0x%x  %s ",
1806                         p->addr, kprobe_type, sym, offset,
1807                         (modname ? modname : " "));
1808         else
1809                 seq_printf(pi, "%p  %s  %p ",
1810                         p->addr, kprobe_type, p->addr);
1811
1812         if (!pp)
1813                 pp = p;
1814         seq_printf(pi, "%s%s%s\n",
1815                 (kprobe_gone(p) ? "[GONE]" : ""),
1816                 ((kprobe_disabled(p) && !kprobe_gone(p)) ?  "[DISABLED]" : ""),
1817                 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""));
1818 }
1819
1820 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
1821 {
1822         return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
1823 }
1824
1825 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
1826 {
1827         (*pos)++;
1828         if (*pos >= KPROBE_TABLE_SIZE)
1829                 return NULL;
1830         return pos;
1831 }
1832
1833 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
1834 {
1835         /* Nothing to do */
1836 }
1837
1838 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
1839 {
1840         struct hlist_head *head;
1841         struct hlist_node *node;
1842         struct kprobe *p, *kp;
1843         const char *sym = NULL;
1844         unsigned int i = *(loff_t *) v;
1845         unsigned long offset = 0;
1846         char *modname, namebuf[128];
1847
1848         head = &kprobe_table[i];
1849         preempt_disable();
1850         hlist_for_each_entry_rcu(p, node, head, hlist) {
1851                 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
1852                                         &offset, &modname, namebuf);
1853                 if (kprobe_aggrprobe(p)) {
1854                         list_for_each_entry_rcu(kp, &p->list, list)
1855                                 report_probe(pi, kp, sym, offset, modname, p);
1856                 } else
1857                         report_probe(pi, p, sym, offset, modname, NULL);
1858         }
1859         preempt_enable();
1860         return 0;
1861 }
1862
1863 static const struct seq_operations kprobes_seq_ops = {
1864         .start = kprobe_seq_start,
1865         .next  = kprobe_seq_next,
1866         .stop  = kprobe_seq_stop,
1867         .show  = show_kprobe_addr
1868 };
1869
1870 static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
1871 {
1872         return seq_open(filp, &kprobes_seq_ops);
1873 }
1874
1875 static const struct file_operations debugfs_kprobes_operations = {
1876         .open           = kprobes_open,
1877         .read           = seq_read,
1878         .llseek         = seq_lseek,
1879         .release        = seq_release,
1880 };
1881
1882 static void __kprobes arm_all_kprobes(void)
1883 {
1884         struct hlist_head *head;
1885         struct hlist_node *node;
1886         struct kprobe *p;
1887         unsigned int i;
1888
1889         mutex_lock(&kprobe_mutex);
1890
1891         /* If kprobes are armed, just return */
1892         if (!kprobes_all_disarmed)
1893                 goto already_enabled;
1894
1895         /* Arming kprobes doesn't optimize kprobe itself */
1896         mutex_lock(&text_mutex);
1897         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1898                 head = &kprobe_table[i];
1899                 hlist_for_each_entry_rcu(p, node, head, hlist)
1900                         if (!kprobe_disabled(p))
1901                                 __arm_kprobe(p);
1902         }
1903         mutex_unlock(&text_mutex);
1904
1905         kprobes_all_disarmed = false;
1906         printk(KERN_INFO "Kprobes globally enabled\n");
1907
1908 already_enabled:
1909         mutex_unlock(&kprobe_mutex);
1910         return;
1911 }
1912
1913 static void __kprobes disarm_all_kprobes(void)
1914 {
1915         struct hlist_head *head;
1916         struct hlist_node *node;
1917         struct kprobe *p;
1918         unsigned int i;
1919
1920         mutex_lock(&kprobe_mutex);
1921
1922         /* If kprobes are already disarmed, just return */
1923         if (kprobes_all_disarmed)
1924                 goto already_disabled;
1925
1926         kprobes_all_disarmed = true;
1927         printk(KERN_INFO "Kprobes globally disabled\n");
1928
1929         /*
1930          * Here we call get_online_cpus() for avoiding text_mutex deadlock,
1931          * because disarming may also unoptimize kprobes.
1932          */
1933         get_online_cpus();
1934         mutex_lock(&text_mutex);
1935         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1936                 head = &kprobe_table[i];
1937                 hlist_for_each_entry_rcu(p, node, head, hlist) {
1938                         if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
1939                                 __disarm_kprobe(p);
1940                 }
1941         }
1942
1943         mutex_unlock(&text_mutex);
1944         put_online_cpus();
1945         mutex_unlock(&kprobe_mutex);
1946         /* Allow all currently running kprobes to complete */
1947         synchronize_sched();
1948         return;
1949
1950 already_disabled:
1951         mutex_unlock(&kprobe_mutex);
1952         return;
1953 }
1954
1955 /*
1956  * XXX: The debugfs bool file interface doesn't allow for callbacks
1957  * when the bool state is switched. We can reuse that facility when
1958  * available
1959  */
1960 static ssize_t read_enabled_file_bool(struct file *file,
1961                char __user *user_buf, size_t count, loff_t *ppos)
1962 {
1963         char buf[3];
1964
1965         if (!kprobes_all_disarmed)
1966                 buf[0] = '1';
1967         else
1968                 buf[0] = '0';
1969         buf[1] = '\n';
1970         buf[2] = 0x00;
1971         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1972 }
1973
1974 static ssize_t write_enabled_file_bool(struct file *file,
1975                const char __user *user_buf, size_t count, loff_t *ppos)
1976 {
1977         char buf[32];
1978         int buf_size;
1979
1980         buf_size = min(count, (sizeof(buf)-1));
1981         if (copy_from_user(buf, user_buf, buf_size))
1982                 return -EFAULT;
1983
1984         switch (buf[0]) {
1985         case 'y':
1986         case 'Y':
1987         case '1':
1988                 arm_all_kprobes();
1989                 break;
1990         case 'n':
1991         case 'N':
1992         case '0':
1993                 disarm_all_kprobes();
1994                 break;
1995         }
1996
1997         return count;
1998 }
1999
2000 static const struct file_operations fops_kp = {
2001         .read =         read_enabled_file_bool,
2002         .write =        write_enabled_file_bool,
2003         .llseek =       default_llseek,
2004 };
2005
2006 static int __kprobes debugfs_kprobe_init(void)
2007 {
2008         struct dentry *dir, *file;
2009         unsigned int value = 1;
2010
2011         dir = debugfs_create_dir("kprobes", NULL);
2012         if (!dir)
2013                 return -ENOMEM;
2014
2015         file = debugfs_create_file("list", 0444, dir, NULL,
2016                                 &debugfs_kprobes_operations);
2017         if (!file) {
2018                 debugfs_remove(dir);
2019                 return -ENOMEM;
2020         }
2021
2022         file = debugfs_create_file("enabled", 0600, dir,
2023                                         &value, &fops_kp);
2024         if (!file) {
2025                 debugfs_remove(dir);
2026                 return -ENOMEM;
2027         }
2028
2029         return 0;
2030 }
2031
2032 late_initcall(debugfs_kprobe_init);
2033 #endif /* CONFIG_DEBUG_FS */
2034
2035 module_init(init_kprobes);
2036
2037 /* defined in arch/.../kernel/kprobes.c */
2038 EXPORT_SYMBOL_GPL(jprobe_return);