PM QOS update
[linux-2.6.git] / kernel / kprobes.c
1 /*
2  *  Kernel Probes (KProbes)
3  *  kernel/kprobes.c
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18  *
19  * Copyright (C) IBM Corporation, 2002, 2004
20  *
21  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22  *              Probes initial implementation (includes suggestions from
23  *              Rusty Russell).
24  * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25  *              hlists and exceptions notifier as suggested by Andi Kleen.
26  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27  *              interface to access function arguments.
28  * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29  *              exceptions notifier to be first on the priority list.
30  * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31  *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32  *              <prasanna@in.ibm.com> added function-return probes.
33  */
34 #include <linux/kprobes.h>
35 #include <linux/hash.h>
36 #include <linux/init.h>
37 #include <linux/slab.h>
38 #include <linux/stddef.h>
39 #include <linux/module.h>
40 #include <linux/moduleloader.h>
41 #include <linux/kallsyms.h>
42 #include <linux/freezer.h>
43 #include <linux/seq_file.h>
44 #include <linux/debugfs.h>
45 #include <linux/sysctl.h>
46 #include <linux/kdebug.h>
47 #include <linux/memory.h>
48 #include <linux/ftrace.h>
49 #include <linux/cpu.h>
50
51 #include <asm-generic/sections.h>
52 #include <asm/cacheflush.h>
53 #include <asm/errno.h>
54 #include <asm/uaccess.h>
55
56 #define KPROBE_HASH_BITS 6
57 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
58
59
60 /*
61  * Some oddball architectures like 64bit powerpc have function descriptors
62  * so this must be overridable.
63  */
64 #ifndef kprobe_lookup_name
65 #define kprobe_lookup_name(name, addr) \
66         addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
67 #endif
68
69 static int kprobes_initialized;
70 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
71 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
72
73 /* NOTE: change this value only with kprobe_mutex held */
74 static bool kprobes_all_disarmed;
75
76 static DEFINE_MUTEX(kprobe_mutex);      /* Protects kprobe_table */
77 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
78 static struct {
79         spinlock_t lock ____cacheline_aligned_in_smp;
80 } kretprobe_table_locks[KPROBE_TABLE_SIZE];
81
82 static spinlock_t *kretprobe_table_lock_ptr(unsigned long hash)
83 {
84         return &(kretprobe_table_locks[hash].lock);
85 }
86
87 /*
88  * Normally, functions that we'd want to prohibit kprobes in, are marked
89  * __kprobes. But, there are cases where such functions already belong to
90  * a different section (__sched for preempt_schedule)
91  *
92  * For such cases, we now have a blacklist
93  */
94 static struct kprobe_blackpoint kprobe_blacklist[] = {
95         {"preempt_schedule",},
96         {"native_get_debugreg",},
97         {"irq_entries_start",},
98         {"common_interrupt",},
99         {"mcount",},    /* mcount can be called from everywhere */
100         {NULL}    /* Terminator */
101 };
102
103 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
104 /*
105  * kprobe->ainsn.insn points to the copy of the instruction to be
106  * single-stepped. x86_64, POWER4 and above have no-exec support and
107  * stepping on the instruction on a vmalloced/kmalloced/data page
108  * is a recipe for disaster
109  */
110 struct kprobe_insn_page {
111         struct list_head list;
112         kprobe_opcode_t *insns;         /* Page of instruction slots */
113         int nused;
114         int ngarbage;
115         char slot_used[];
116 };
117
118 #define KPROBE_INSN_PAGE_SIZE(slots)                    \
119         (offsetof(struct kprobe_insn_page, slot_used) + \
120          (sizeof(char) * (slots)))
121
122 struct kprobe_insn_cache {
123         struct list_head pages; /* list of kprobe_insn_page */
124         size_t insn_size;       /* size of instruction slot */
125         int nr_garbage;
126 };
127
128 static int slots_per_page(struct kprobe_insn_cache *c)
129 {
130         return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
131 }
132
133 enum kprobe_slot_state {
134         SLOT_CLEAN = 0,
135         SLOT_DIRTY = 1,
136         SLOT_USED = 2,
137 };
138
139 static DEFINE_MUTEX(kprobe_insn_mutex); /* Protects kprobe_insn_slots */
140 static struct kprobe_insn_cache kprobe_insn_slots = {
141         .pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
142         .insn_size = MAX_INSN_SIZE,
143         .nr_garbage = 0,
144 };
145 static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c);
146
147 /**
148  * __get_insn_slot() - Find a slot on an executable page for an instruction.
149  * We allocate an executable page if there's no room on existing ones.
150  */
151 static kprobe_opcode_t __kprobes *__get_insn_slot(struct kprobe_insn_cache *c)
152 {
153         struct kprobe_insn_page *kip;
154
155  retry:
156         list_for_each_entry(kip, &c->pages, list) {
157                 if (kip->nused < slots_per_page(c)) {
158                         int i;
159                         for (i = 0; i < slots_per_page(c); i++) {
160                                 if (kip->slot_used[i] == SLOT_CLEAN) {
161                                         kip->slot_used[i] = SLOT_USED;
162                                         kip->nused++;
163                                         return kip->insns + (i * c->insn_size);
164                                 }
165                         }
166                         /* kip->nused is broken. Fix it. */
167                         kip->nused = slots_per_page(c);
168                         WARN_ON(1);
169                 }
170         }
171
172         /* If there are any garbage slots, collect it and try again. */
173         if (c->nr_garbage && collect_garbage_slots(c) == 0)
174                 goto retry;
175
176         /* All out of space.  Need to allocate a new page. */
177         kip = kmalloc(KPROBE_INSN_PAGE_SIZE(slots_per_page(c)), GFP_KERNEL);
178         if (!kip)
179                 return NULL;
180
181         /*
182          * Use module_alloc so this page is within +/- 2GB of where the
183          * kernel image and loaded module images reside. This is required
184          * so x86_64 can correctly handle the %rip-relative fixups.
185          */
186         kip->insns = module_alloc(PAGE_SIZE);
187         if (!kip->insns) {
188                 kfree(kip);
189                 return NULL;
190         }
191         INIT_LIST_HEAD(&kip->list);
192         memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
193         kip->slot_used[0] = SLOT_USED;
194         kip->nused = 1;
195         kip->ngarbage = 0;
196         list_add(&kip->list, &c->pages);
197         return kip->insns;
198 }
199
200
201 kprobe_opcode_t __kprobes *get_insn_slot(void)
202 {
203         kprobe_opcode_t *ret = NULL;
204
205         mutex_lock(&kprobe_insn_mutex);
206         ret = __get_insn_slot(&kprobe_insn_slots);
207         mutex_unlock(&kprobe_insn_mutex);
208
209         return ret;
210 }
211
212 /* Return 1 if all garbages are collected, otherwise 0. */
213 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
214 {
215         kip->slot_used[idx] = SLOT_CLEAN;
216         kip->nused--;
217         if (kip->nused == 0) {
218                 /*
219                  * Page is no longer in use.  Free it unless
220                  * it's the last one.  We keep the last one
221                  * so as not to have to set it up again the
222                  * next time somebody inserts a probe.
223                  */
224                 if (!list_is_singular(&kip->list)) {
225                         list_del(&kip->list);
226                         module_free(NULL, kip->insns);
227                         kfree(kip);
228                 }
229                 return 1;
230         }
231         return 0;
232 }
233
234 static int __kprobes collect_garbage_slots(struct kprobe_insn_cache *c)
235 {
236         struct kprobe_insn_page *kip, *next;
237
238         /* Ensure no-one is interrupted on the garbages */
239         synchronize_sched();
240
241         list_for_each_entry_safe(kip, next, &c->pages, list) {
242                 int i;
243                 if (kip->ngarbage == 0)
244                         continue;
245                 kip->ngarbage = 0;      /* we will collect all garbages */
246                 for (i = 0; i < slots_per_page(c); i++) {
247                         if (kip->slot_used[i] == SLOT_DIRTY &&
248                             collect_one_slot(kip, i))
249                                 break;
250                 }
251         }
252         c->nr_garbage = 0;
253         return 0;
254 }
255
256 static void __kprobes __free_insn_slot(struct kprobe_insn_cache *c,
257                                        kprobe_opcode_t *slot, int dirty)
258 {
259         struct kprobe_insn_page *kip;
260
261         list_for_each_entry(kip, &c->pages, list) {
262                 long idx = ((long)slot - (long)kip->insns) /
263                                 (c->insn_size * sizeof(kprobe_opcode_t));
264                 if (idx >= 0 && idx < slots_per_page(c)) {
265                         WARN_ON(kip->slot_used[idx] != SLOT_USED);
266                         if (dirty) {
267                                 kip->slot_used[idx] = SLOT_DIRTY;
268                                 kip->ngarbage++;
269                                 if (++c->nr_garbage > slots_per_page(c))
270                                         collect_garbage_slots(c);
271                         } else
272                                 collect_one_slot(kip, idx);
273                         return;
274                 }
275         }
276         /* Could not free this slot. */
277         WARN_ON(1);
278 }
279
280 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
281 {
282         mutex_lock(&kprobe_insn_mutex);
283         __free_insn_slot(&kprobe_insn_slots, slot, dirty);
284         mutex_unlock(&kprobe_insn_mutex);
285 }
286 #ifdef CONFIG_OPTPROBES
287 /* For optimized_kprobe buffer */
288 static DEFINE_MUTEX(kprobe_optinsn_mutex); /* Protects kprobe_optinsn_slots */
289 static struct kprobe_insn_cache kprobe_optinsn_slots = {
290         .pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
291         /* .insn_size is initialized later */
292         .nr_garbage = 0,
293 };
294 /* Get a slot for optimized_kprobe buffer */
295 kprobe_opcode_t __kprobes *get_optinsn_slot(void)
296 {
297         kprobe_opcode_t *ret = NULL;
298
299         mutex_lock(&kprobe_optinsn_mutex);
300         ret = __get_insn_slot(&kprobe_optinsn_slots);
301         mutex_unlock(&kprobe_optinsn_mutex);
302
303         return ret;
304 }
305
306 void __kprobes free_optinsn_slot(kprobe_opcode_t * slot, int dirty)
307 {
308         mutex_lock(&kprobe_optinsn_mutex);
309         __free_insn_slot(&kprobe_optinsn_slots, slot, dirty);
310         mutex_unlock(&kprobe_optinsn_mutex);
311 }
312 #endif
313 #endif
314
315 /* We have preemption disabled.. so it is safe to use __ versions */
316 static inline void set_kprobe_instance(struct kprobe *kp)
317 {
318         __get_cpu_var(kprobe_instance) = kp;
319 }
320
321 static inline void reset_kprobe_instance(void)
322 {
323         __get_cpu_var(kprobe_instance) = NULL;
324 }
325
326 /*
327  * This routine is called either:
328  *      - under the kprobe_mutex - during kprobe_[un]register()
329  *                              OR
330  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
331  */
332 struct kprobe __kprobes *get_kprobe(void *addr)
333 {
334         struct hlist_head *head;
335         struct hlist_node *node;
336         struct kprobe *p;
337
338         head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
339         hlist_for_each_entry_rcu(p, node, head, hlist) {
340                 if (p->addr == addr)
341                         return p;
342         }
343
344         return NULL;
345 }
346
347 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
348
349 /* Return true if the kprobe is an aggregator */
350 static inline int kprobe_aggrprobe(struct kprobe *p)
351 {
352         return p->pre_handler == aggr_pre_handler;
353 }
354
355 /*
356  * Keep all fields in the kprobe consistent
357  */
358 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
359 {
360         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
361         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
362 }
363
364 #ifdef CONFIG_OPTPROBES
365 /* NOTE: change this value only with kprobe_mutex held */
366 static bool kprobes_allow_optimization;
367
368 /*
369  * Call all pre_handler on the list, but ignores its return value.
370  * This must be called from arch-dep optimized caller.
371  */
372 void __kprobes opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
373 {
374         struct kprobe *kp;
375
376         list_for_each_entry_rcu(kp, &p->list, list) {
377                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
378                         set_kprobe_instance(kp);
379                         kp->pre_handler(kp, regs);
380                 }
381                 reset_kprobe_instance();
382         }
383 }
384
385 /* Return true(!0) if the kprobe is ready for optimization. */
386 static inline int kprobe_optready(struct kprobe *p)
387 {
388         struct optimized_kprobe *op;
389
390         if (kprobe_aggrprobe(p)) {
391                 op = container_of(p, struct optimized_kprobe, kp);
392                 return arch_prepared_optinsn(&op->optinsn);
393         }
394
395         return 0;
396 }
397
398 /*
399  * Return an optimized kprobe whose optimizing code replaces
400  * instructions including addr (exclude breakpoint).
401  */
402 struct kprobe *__kprobes get_optimized_kprobe(unsigned long addr)
403 {
404         int i;
405         struct kprobe *p = NULL;
406         struct optimized_kprobe *op;
407
408         /* Don't check i == 0, since that is a breakpoint case. */
409         for (i = 1; !p && i < MAX_OPTIMIZED_LENGTH; i++)
410                 p = get_kprobe((void *)(addr - i));
411
412         if (p && kprobe_optready(p)) {
413                 op = container_of(p, struct optimized_kprobe, kp);
414                 if (arch_within_optimized_kprobe(op, addr))
415                         return p;
416         }
417
418         return NULL;
419 }
420
421 /* Optimization staging list, protected by kprobe_mutex */
422 static LIST_HEAD(optimizing_list);
423
424 static void kprobe_optimizer(struct work_struct *work);
425 static DECLARE_DELAYED_WORK(optimizing_work, kprobe_optimizer);
426 #define OPTIMIZE_DELAY 5
427
428 /* Kprobe jump optimizer */
429 static __kprobes void kprobe_optimizer(struct work_struct *work)
430 {
431         struct optimized_kprobe *op, *tmp;
432
433         /* Lock modules while optimizing kprobes */
434         mutex_lock(&module_mutex);
435         mutex_lock(&kprobe_mutex);
436         if (kprobes_all_disarmed || !kprobes_allow_optimization)
437                 goto end;
438
439         /*
440          * Wait for quiesence period to ensure all running interrupts
441          * are done. Because optprobe may modify multiple instructions
442          * there is a chance that Nth instruction is interrupted. In that
443          * case, running interrupt can return to 2nd-Nth byte of jump
444          * instruction. This wait is for avoiding it.
445          */
446         synchronize_sched();
447
448         /*
449          * The optimization/unoptimization refers online_cpus via
450          * stop_machine() and cpu-hotplug modifies online_cpus.
451          * And same time, text_mutex will be held in cpu-hotplug and here.
452          * This combination can cause a deadlock (cpu-hotplug try to lock
453          * text_mutex but stop_machine can not be done because online_cpus
454          * has been changed)
455          * To avoid this deadlock, we need to call get_online_cpus()
456          * for preventing cpu-hotplug outside of text_mutex locking.
457          */
458         get_online_cpus();
459         mutex_lock(&text_mutex);
460         list_for_each_entry_safe(op, tmp, &optimizing_list, list) {
461                 WARN_ON(kprobe_disabled(&op->kp));
462                 if (arch_optimize_kprobe(op) < 0)
463                         op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
464                 list_del_init(&op->list);
465         }
466         mutex_unlock(&text_mutex);
467         put_online_cpus();
468 end:
469         mutex_unlock(&kprobe_mutex);
470         mutex_unlock(&module_mutex);
471 }
472
473 /* Optimize kprobe if p is ready to be optimized */
474 static __kprobes void optimize_kprobe(struct kprobe *p)
475 {
476         struct optimized_kprobe *op;
477
478         /* Check if the kprobe is disabled or not ready for optimization. */
479         if (!kprobe_optready(p) || !kprobes_allow_optimization ||
480             (kprobe_disabled(p) || kprobes_all_disarmed))
481                 return;
482
483         /* Both of break_handler and post_handler are not supported. */
484         if (p->break_handler || p->post_handler)
485                 return;
486
487         op = container_of(p, struct optimized_kprobe, kp);
488
489         /* Check there is no other kprobes at the optimized instructions */
490         if (arch_check_optimized_kprobe(op) < 0)
491                 return;
492
493         /* Check if it is already optimized. */
494         if (op->kp.flags & KPROBE_FLAG_OPTIMIZED)
495                 return;
496
497         op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
498         list_add(&op->list, &optimizing_list);
499         if (!delayed_work_pending(&optimizing_work))
500                 schedule_delayed_work(&optimizing_work, OPTIMIZE_DELAY);
501 }
502
503 /* Unoptimize a kprobe if p is optimized */
504 static __kprobes void unoptimize_kprobe(struct kprobe *p)
505 {
506         struct optimized_kprobe *op;
507
508         if ((p->flags & KPROBE_FLAG_OPTIMIZED) && kprobe_aggrprobe(p)) {
509                 op = container_of(p, struct optimized_kprobe, kp);
510                 if (!list_empty(&op->list))
511                         /* Dequeue from the optimization queue */
512                         list_del_init(&op->list);
513                 else
514                         /* Replace jump with break */
515                         arch_unoptimize_kprobe(op);
516                 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
517         }
518 }
519
520 /* Remove optimized instructions */
521 static void __kprobes kill_optimized_kprobe(struct kprobe *p)
522 {
523         struct optimized_kprobe *op;
524
525         op = container_of(p, struct optimized_kprobe, kp);
526         if (!list_empty(&op->list)) {
527                 /* Dequeue from the optimization queue */
528                 list_del_init(&op->list);
529                 op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
530         }
531         /* Don't unoptimize, because the target code will be freed. */
532         arch_remove_optimized_kprobe(op);
533 }
534
535 /* Try to prepare optimized instructions */
536 static __kprobes void prepare_optimized_kprobe(struct kprobe *p)
537 {
538         struct optimized_kprobe *op;
539
540         op = container_of(p, struct optimized_kprobe, kp);
541         arch_prepare_optimized_kprobe(op);
542 }
543
544 /* Free optimized instructions and optimized_kprobe */
545 static __kprobes void free_aggr_kprobe(struct kprobe *p)
546 {
547         struct optimized_kprobe *op;
548
549         op = container_of(p, struct optimized_kprobe, kp);
550         arch_remove_optimized_kprobe(op);
551         kfree(op);
552 }
553
554 /* Allocate new optimized_kprobe and try to prepare optimized instructions */
555 static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
556 {
557         struct optimized_kprobe *op;
558
559         op = kzalloc(sizeof(struct optimized_kprobe), GFP_KERNEL);
560         if (!op)
561                 return NULL;
562
563         INIT_LIST_HEAD(&op->list);
564         op->kp.addr = p->addr;
565         arch_prepare_optimized_kprobe(op);
566
567         return &op->kp;
568 }
569
570 static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
571
572 /*
573  * Prepare an optimized_kprobe and optimize it
574  * NOTE: p must be a normal registered kprobe
575  */
576 static __kprobes void try_to_optimize_kprobe(struct kprobe *p)
577 {
578         struct kprobe *ap;
579         struct optimized_kprobe *op;
580
581         ap = alloc_aggr_kprobe(p);
582         if (!ap)
583                 return;
584
585         op = container_of(ap, struct optimized_kprobe, kp);
586         if (!arch_prepared_optinsn(&op->optinsn)) {
587                 /* If failed to setup optimizing, fallback to kprobe */
588                 free_aggr_kprobe(ap);
589                 return;
590         }
591
592         init_aggr_kprobe(ap, p);
593         optimize_kprobe(ap);
594 }
595
596 #ifdef CONFIG_SYSCTL
597 static void __kprobes optimize_all_kprobes(void)
598 {
599         struct hlist_head *head;
600         struct hlist_node *node;
601         struct kprobe *p;
602         unsigned int i;
603
604         /* If optimization is already allowed, just return */
605         if (kprobes_allow_optimization)
606                 return;
607
608         kprobes_allow_optimization = true;
609         mutex_lock(&text_mutex);
610         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
611                 head = &kprobe_table[i];
612                 hlist_for_each_entry_rcu(p, node, head, hlist)
613                         if (!kprobe_disabled(p))
614                                 optimize_kprobe(p);
615         }
616         mutex_unlock(&text_mutex);
617         printk(KERN_INFO "Kprobes globally optimized\n");
618 }
619
620 static void __kprobes unoptimize_all_kprobes(void)
621 {
622         struct hlist_head *head;
623         struct hlist_node *node;
624         struct kprobe *p;
625         unsigned int i;
626
627         /* If optimization is already prohibited, just return */
628         if (!kprobes_allow_optimization)
629                 return;
630
631         kprobes_allow_optimization = false;
632         printk(KERN_INFO "Kprobes globally unoptimized\n");
633         get_online_cpus();      /* For avoiding text_mutex deadlock */
634         mutex_lock(&text_mutex);
635         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
636                 head = &kprobe_table[i];
637                 hlist_for_each_entry_rcu(p, node, head, hlist) {
638                         if (!kprobe_disabled(p))
639                                 unoptimize_kprobe(p);
640                 }
641         }
642
643         mutex_unlock(&text_mutex);
644         put_online_cpus();
645         /* Allow all currently running kprobes to complete */
646         synchronize_sched();
647 }
648
649 int sysctl_kprobes_optimization;
650 int proc_kprobes_optimization_handler(struct ctl_table *table, int write,
651                                       void __user *buffer, size_t *length,
652                                       loff_t *ppos)
653 {
654         int ret;
655
656         mutex_lock(&kprobe_mutex);
657         sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
658         ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
659
660         if (sysctl_kprobes_optimization)
661                 optimize_all_kprobes();
662         else
663                 unoptimize_all_kprobes();
664         mutex_unlock(&kprobe_mutex);
665
666         return ret;
667 }
668 #endif /* CONFIG_SYSCTL */
669
670 static void __kprobes __arm_kprobe(struct kprobe *p)
671 {
672         struct kprobe *old_p;
673
674         /* Check collision with other optimized kprobes */
675         old_p = get_optimized_kprobe((unsigned long)p->addr);
676         if (unlikely(old_p))
677                 unoptimize_kprobe(old_p); /* Fallback to unoptimized kprobe */
678
679         arch_arm_kprobe(p);
680         optimize_kprobe(p);     /* Try to optimize (add kprobe to a list) */
681 }
682
683 static void __kprobes __disarm_kprobe(struct kprobe *p)
684 {
685         struct kprobe *old_p;
686
687         unoptimize_kprobe(p);   /* Try to unoptimize */
688         arch_disarm_kprobe(p);
689
690         /* If another kprobe was blocked, optimize it. */
691         old_p = get_optimized_kprobe((unsigned long)p->addr);
692         if (unlikely(old_p))
693                 optimize_kprobe(old_p);
694 }
695
696 #else /* !CONFIG_OPTPROBES */
697
698 #define optimize_kprobe(p)                      do {} while (0)
699 #define unoptimize_kprobe(p)                    do {} while (0)
700 #define kill_optimized_kprobe(p)                do {} while (0)
701 #define prepare_optimized_kprobe(p)             do {} while (0)
702 #define try_to_optimize_kprobe(p)               do {} while (0)
703 #define __arm_kprobe(p)                         arch_arm_kprobe(p)
704 #define __disarm_kprobe(p)                      arch_disarm_kprobe(p)
705
706 static __kprobes void free_aggr_kprobe(struct kprobe *p)
707 {
708         kfree(p);
709 }
710
711 static __kprobes struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
712 {
713         return kzalloc(sizeof(struct kprobe), GFP_KERNEL);
714 }
715 #endif /* CONFIG_OPTPROBES */
716
717 /* Arm a kprobe with text_mutex */
718 static void __kprobes arm_kprobe(struct kprobe *kp)
719 {
720         /*
721          * Here, since __arm_kprobe() doesn't use stop_machine(),
722          * this doesn't cause deadlock on text_mutex. So, we don't
723          * need get_online_cpus().
724          */
725         mutex_lock(&text_mutex);
726         __arm_kprobe(kp);
727         mutex_unlock(&text_mutex);
728 }
729
730 /* Disarm a kprobe with text_mutex */
731 static void __kprobes disarm_kprobe(struct kprobe *kp)
732 {
733         get_online_cpus();      /* For avoiding text_mutex deadlock */
734         mutex_lock(&text_mutex);
735         __disarm_kprobe(kp);
736         mutex_unlock(&text_mutex);
737         put_online_cpus();
738 }
739
740 /*
741  * Aggregate handlers for multiple kprobes support - these handlers
742  * take care of invoking the individual kprobe handlers on p->list
743  */
744 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
745 {
746         struct kprobe *kp;
747
748         list_for_each_entry_rcu(kp, &p->list, list) {
749                 if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
750                         set_kprobe_instance(kp);
751                         if (kp->pre_handler(kp, regs))
752                                 return 1;
753                 }
754                 reset_kprobe_instance();
755         }
756         return 0;
757 }
758
759 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
760                                         unsigned long flags)
761 {
762         struct kprobe *kp;
763
764         list_for_each_entry_rcu(kp, &p->list, list) {
765                 if (kp->post_handler && likely(!kprobe_disabled(kp))) {
766                         set_kprobe_instance(kp);
767                         kp->post_handler(kp, regs, flags);
768                         reset_kprobe_instance();
769                 }
770         }
771 }
772
773 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
774                                         int trapnr)
775 {
776         struct kprobe *cur = __get_cpu_var(kprobe_instance);
777
778         /*
779          * if we faulted "during" the execution of a user specified
780          * probe handler, invoke just that probe's fault handler
781          */
782         if (cur && cur->fault_handler) {
783                 if (cur->fault_handler(cur, regs, trapnr))
784                         return 1;
785         }
786         return 0;
787 }
788
789 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
790 {
791         struct kprobe *cur = __get_cpu_var(kprobe_instance);
792         int ret = 0;
793
794         if (cur && cur->break_handler) {
795                 if (cur->break_handler(cur, regs))
796                         ret = 1;
797         }
798         reset_kprobe_instance();
799         return ret;
800 }
801
802 /* Walks the list and increments nmissed count for multiprobe case */
803 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
804 {
805         struct kprobe *kp;
806         if (!kprobe_aggrprobe(p)) {
807                 p->nmissed++;
808         } else {
809                 list_for_each_entry_rcu(kp, &p->list, list)
810                         kp->nmissed++;
811         }
812         return;
813 }
814
815 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
816                                 struct hlist_head *head)
817 {
818         struct kretprobe *rp = ri->rp;
819
820         /* remove rp inst off the rprobe_inst_table */
821         hlist_del(&ri->hlist);
822         INIT_HLIST_NODE(&ri->hlist);
823         if (likely(rp)) {
824                 spin_lock(&rp->lock);
825                 hlist_add_head(&ri->hlist, &rp->free_instances);
826                 spin_unlock(&rp->lock);
827         } else
828                 /* Unregistering */
829                 hlist_add_head(&ri->hlist, head);
830 }
831
832 void __kprobes kretprobe_hash_lock(struct task_struct *tsk,
833                          struct hlist_head **head, unsigned long *flags)
834 {
835         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
836         spinlock_t *hlist_lock;
837
838         *head = &kretprobe_inst_table[hash];
839         hlist_lock = kretprobe_table_lock_ptr(hash);
840         spin_lock_irqsave(hlist_lock, *flags);
841 }
842
843 static void __kprobes kretprobe_table_lock(unsigned long hash,
844         unsigned long *flags)
845 {
846         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
847         spin_lock_irqsave(hlist_lock, *flags);
848 }
849
850 void __kprobes kretprobe_hash_unlock(struct task_struct *tsk,
851         unsigned long *flags)
852 {
853         unsigned long hash = hash_ptr(tsk, KPROBE_HASH_BITS);
854         spinlock_t *hlist_lock;
855
856         hlist_lock = kretprobe_table_lock_ptr(hash);
857         spin_unlock_irqrestore(hlist_lock, *flags);
858 }
859
860 void __kprobes kretprobe_table_unlock(unsigned long hash, unsigned long *flags)
861 {
862         spinlock_t *hlist_lock = kretprobe_table_lock_ptr(hash);
863         spin_unlock_irqrestore(hlist_lock, *flags);
864 }
865
866 /*
867  * This function is called from finish_task_switch when task tk becomes dead,
868  * so that we can recycle any function-return probe instances associated
869  * with this task. These left over instances represent probed functions
870  * that have been called but will never return.
871  */
872 void __kprobes kprobe_flush_task(struct task_struct *tk)
873 {
874         struct kretprobe_instance *ri;
875         struct hlist_head *head, empty_rp;
876         struct hlist_node *node, *tmp;
877         unsigned long hash, flags = 0;
878
879         if (unlikely(!kprobes_initialized))
880                 /* Early boot.  kretprobe_table_locks not yet initialized. */
881                 return;
882
883         hash = hash_ptr(tk, KPROBE_HASH_BITS);
884         head = &kretprobe_inst_table[hash];
885         kretprobe_table_lock(hash, &flags);
886         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
887                 if (ri->task == tk)
888                         recycle_rp_inst(ri, &empty_rp);
889         }
890         kretprobe_table_unlock(hash, &flags);
891         INIT_HLIST_HEAD(&empty_rp);
892         hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
893                 hlist_del(&ri->hlist);
894                 kfree(ri);
895         }
896 }
897
898 static inline void free_rp_inst(struct kretprobe *rp)
899 {
900         struct kretprobe_instance *ri;
901         struct hlist_node *pos, *next;
902
903         hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, hlist) {
904                 hlist_del(&ri->hlist);
905                 kfree(ri);
906         }
907 }
908
909 static void __kprobes cleanup_rp_inst(struct kretprobe *rp)
910 {
911         unsigned long flags, hash;
912         struct kretprobe_instance *ri;
913         struct hlist_node *pos, *next;
914         struct hlist_head *head;
915
916         /* No race here */
917         for (hash = 0; hash < KPROBE_TABLE_SIZE; hash++) {
918                 kretprobe_table_lock(hash, &flags);
919                 head = &kretprobe_inst_table[hash];
920                 hlist_for_each_entry_safe(ri, pos, next, head, hlist) {
921                         if (ri->rp == rp)
922                                 ri->rp = NULL;
923                 }
924                 kretprobe_table_unlock(hash, &flags);
925         }
926         free_rp_inst(rp);
927 }
928
929 /*
930 * Add the new probe to ap->list. Fail if this is the
931 * second jprobe at the address - two jprobes can't coexist
932 */
933 static int __kprobes add_new_kprobe(struct kprobe *ap, struct kprobe *p)
934 {
935         BUG_ON(kprobe_gone(ap) || kprobe_gone(p));
936
937         if (p->break_handler || p->post_handler)
938                 unoptimize_kprobe(ap);  /* Fall back to normal kprobe */
939
940         if (p->break_handler) {
941                 if (ap->break_handler)
942                         return -EEXIST;
943                 list_add_tail_rcu(&p->list, &ap->list);
944                 ap->break_handler = aggr_break_handler;
945         } else
946                 list_add_rcu(&p->list, &ap->list);
947         if (p->post_handler && !ap->post_handler)
948                 ap->post_handler = aggr_post_handler;
949
950         if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
951                 ap->flags &= ~KPROBE_FLAG_DISABLED;
952                 if (!kprobes_all_disarmed)
953                         /* Arm the breakpoint again. */
954                         __arm_kprobe(ap);
955         }
956         return 0;
957 }
958
959 /*
960  * Fill in the required fields of the "manager kprobe". Replace the
961  * earlier kprobe in the hlist with the manager kprobe
962  */
963 static void __kprobes init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
964 {
965         /* Copy p's insn slot to ap */
966         copy_kprobe(p, ap);
967         flush_insn_slot(ap);
968         ap->addr = p->addr;
969         ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
970         ap->pre_handler = aggr_pre_handler;
971         ap->fault_handler = aggr_fault_handler;
972         /* We don't care the kprobe which has gone. */
973         if (p->post_handler && !kprobe_gone(p))
974                 ap->post_handler = aggr_post_handler;
975         if (p->break_handler && !kprobe_gone(p))
976                 ap->break_handler = aggr_break_handler;
977
978         INIT_LIST_HEAD(&ap->list);
979         INIT_HLIST_NODE(&ap->hlist);
980
981         list_add_rcu(&p->list, &ap->list);
982         hlist_replace_rcu(&p->hlist, &ap->hlist);
983 }
984
985 /*
986  * This is the second or subsequent kprobe at the address - handle
987  * the intricacies
988  */
989 static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
990                                           struct kprobe *p)
991 {
992         int ret = 0;
993         struct kprobe *ap = old_p;
994
995         if (!kprobe_aggrprobe(old_p)) {
996                 /* If old_p is not an aggr_kprobe, create new aggr_kprobe. */
997                 ap = alloc_aggr_kprobe(old_p);
998                 if (!ap)
999                         return -ENOMEM;
1000                 init_aggr_kprobe(ap, old_p);
1001         }
1002
1003         if (kprobe_gone(ap)) {
1004                 /*
1005                  * Attempting to insert new probe at the same location that
1006                  * had a probe in the module vaddr area which already
1007                  * freed. So, the instruction slot has already been
1008                  * released. We need a new slot for the new probe.
1009                  */
1010                 ret = arch_prepare_kprobe(ap);
1011                 if (ret)
1012                         /*
1013                          * Even if fail to allocate new slot, don't need to
1014                          * free aggr_probe. It will be used next time, or
1015                          * freed by unregister_kprobe.
1016                          */
1017                         return ret;
1018
1019                 /* Prepare optimized instructions if possible. */
1020                 prepare_optimized_kprobe(ap);
1021
1022                 /*
1023                  * Clear gone flag to prevent allocating new slot again, and
1024                  * set disabled flag because it is not armed yet.
1025                  */
1026                 ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1027                             | KPROBE_FLAG_DISABLED;
1028         }
1029
1030         /* Copy ap's insn slot to p */
1031         copy_kprobe(ap, p);
1032         return add_new_kprobe(ap, p);
1033 }
1034
1035 /* Try to disable aggr_kprobe, and return 1 if succeeded.*/
1036 static int __kprobes try_to_disable_aggr_kprobe(struct kprobe *p)
1037 {
1038         struct kprobe *kp;
1039
1040         list_for_each_entry_rcu(kp, &p->list, list) {
1041                 if (!kprobe_disabled(kp))
1042                         /*
1043                          * There is an active probe on the list.
1044                          * We can't disable aggr_kprobe.
1045                          */
1046                         return 0;
1047         }
1048         p->flags |= KPROBE_FLAG_DISABLED;
1049         return 1;
1050 }
1051
1052 static int __kprobes in_kprobes_functions(unsigned long addr)
1053 {
1054         struct kprobe_blackpoint *kb;
1055
1056         if (addr >= (unsigned long)__kprobes_text_start &&
1057             addr < (unsigned long)__kprobes_text_end)
1058                 return -EINVAL;
1059         /*
1060          * If there exists a kprobe_blacklist, verify and
1061          * fail any probe registration in the prohibited area
1062          */
1063         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1064                 if (kb->start_addr) {
1065                         if (addr >= kb->start_addr &&
1066                             addr < (kb->start_addr + kb->range))
1067                                 return -EINVAL;
1068                 }
1069         }
1070         return 0;
1071 }
1072
1073 /*
1074  * If we have a symbol_name argument, look it up and add the offset field
1075  * to it. This way, we can specify a relative address to a symbol.
1076  */
1077 static kprobe_opcode_t __kprobes *kprobe_addr(struct kprobe *p)
1078 {
1079         kprobe_opcode_t *addr = p->addr;
1080         if (p->symbol_name) {
1081                 if (addr)
1082                         return NULL;
1083                 kprobe_lookup_name(p->symbol_name, addr);
1084         }
1085
1086         if (!addr)
1087                 return NULL;
1088         return (kprobe_opcode_t *)(((char *)addr) + p->offset);
1089 }
1090
1091 /* Check passed kprobe is valid and return kprobe in kprobe_table. */
1092 static struct kprobe * __kprobes __get_valid_kprobe(struct kprobe *p)
1093 {
1094         struct kprobe *old_p, *list_p;
1095
1096         old_p = get_kprobe(p->addr);
1097         if (unlikely(!old_p))
1098                 return NULL;
1099
1100         if (p != old_p) {
1101                 list_for_each_entry_rcu(list_p, &old_p->list, list)
1102                         if (list_p == p)
1103                         /* kprobe p is a valid probe */
1104                                 goto valid;
1105                 return NULL;
1106         }
1107 valid:
1108         return old_p;
1109 }
1110
1111 /* Return error if the kprobe is being re-registered */
1112 static inline int check_kprobe_rereg(struct kprobe *p)
1113 {
1114         int ret = 0;
1115         struct kprobe *old_p;
1116
1117         mutex_lock(&kprobe_mutex);
1118         old_p = __get_valid_kprobe(p);
1119         if (old_p)
1120                 ret = -EINVAL;
1121         mutex_unlock(&kprobe_mutex);
1122         return ret;
1123 }
1124
1125 int __kprobes register_kprobe(struct kprobe *p)
1126 {
1127         int ret = 0;
1128         struct kprobe *old_p;
1129         struct module *probed_mod;
1130         kprobe_opcode_t *addr;
1131
1132         addr = kprobe_addr(p);
1133         if (!addr)
1134                 return -EINVAL;
1135         p->addr = addr;
1136
1137         ret = check_kprobe_rereg(p);
1138         if (ret)
1139                 return ret;
1140
1141         preempt_disable();
1142         if (!kernel_text_address((unsigned long) p->addr) ||
1143             in_kprobes_functions((unsigned long) p->addr) ||
1144             ftrace_text_reserved(p->addr, p->addr)) {
1145                 preempt_enable();
1146                 return -EINVAL;
1147         }
1148
1149         /* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
1150         p->flags &= KPROBE_FLAG_DISABLED;
1151
1152         /*
1153          * Check if are we probing a module.
1154          */
1155         probed_mod = __module_text_address((unsigned long) p->addr);
1156         if (probed_mod) {
1157                 /*
1158                  * We must hold a refcount of the probed module while updating
1159                  * its code to prohibit unexpected unloading.
1160                  */
1161                 if (unlikely(!try_module_get(probed_mod))) {
1162                         preempt_enable();
1163                         return -EINVAL;
1164                 }
1165                 /*
1166                  * If the module freed .init.text, we couldn't insert
1167                  * kprobes in there.
1168                  */
1169                 if (within_module_init((unsigned long)p->addr, probed_mod) &&
1170                     probed_mod->state != MODULE_STATE_COMING) {
1171                         module_put(probed_mod);
1172                         preempt_enable();
1173                         return -EINVAL;
1174                 }
1175         }
1176         preempt_enable();
1177
1178         p->nmissed = 0;
1179         INIT_LIST_HEAD(&p->list);
1180         mutex_lock(&kprobe_mutex);
1181
1182         get_online_cpus();      /* For avoiding text_mutex deadlock. */
1183         mutex_lock(&text_mutex);
1184
1185         old_p = get_kprobe(p->addr);
1186         if (old_p) {
1187                 /* Since this may unoptimize old_p, locking text_mutex. */
1188                 ret = register_aggr_kprobe(old_p, p);
1189                 goto out;
1190         }
1191
1192         ret = arch_prepare_kprobe(p);
1193         if (ret)
1194                 goto out;
1195
1196         INIT_HLIST_NODE(&p->hlist);
1197         hlist_add_head_rcu(&p->hlist,
1198                        &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
1199
1200         if (!kprobes_all_disarmed && !kprobe_disabled(p))
1201                 __arm_kprobe(p);
1202
1203         /* Try to optimize kprobe */
1204         try_to_optimize_kprobe(p);
1205
1206 out:
1207         mutex_unlock(&text_mutex);
1208         put_online_cpus();
1209         mutex_unlock(&kprobe_mutex);
1210
1211         if (probed_mod)
1212                 module_put(probed_mod);
1213
1214         return ret;
1215 }
1216 EXPORT_SYMBOL_GPL(register_kprobe);
1217
1218 /*
1219  * Unregister a kprobe without a scheduler synchronization.
1220  */
1221 static int __kprobes __unregister_kprobe_top(struct kprobe *p)
1222 {
1223         struct kprobe *old_p, *list_p;
1224
1225         old_p = __get_valid_kprobe(p);
1226         if (old_p == NULL)
1227                 return -EINVAL;
1228
1229         if (old_p == p ||
1230             (kprobe_aggrprobe(old_p) &&
1231              list_is_singular(&old_p->list))) {
1232                 /*
1233                  * Only probe on the hash list. Disarm only if kprobes are
1234                  * enabled and not gone - otherwise, the breakpoint would
1235                  * already have been removed. We save on flushing icache.
1236                  */
1237                 if (!kprobes_all_disarmed && !kprobe_disabled(old_p))
1238                         disarm_kprobe(old_p);
1239                 hlist_del_rcu(&old_p->hlist);
1240         } else {
1241                 if (p->break_handler && !kprobe_gone(p))
1242                         old_p->break_handler = NULL;
1243                 if (p->post_handler && !kprobe_gone(p)) {
1244                         list_for_each_entry_rcu(list_p, &old_p->list, list) {
1245                                 if ((list_p != p) && (list_p->post_handler))
1246                                         goto noclean;
1247                         }
1248                         old_p->post_handler = NULL;
1249                 }
1250 noclean:
1251                 list_del_rcu(&p->list);
1252                 if (!kprobe_disabled(old_p)) {
1253                         try_to_disable_aggr_kprobe(old_p);
1254                         if (!kprobes_all_disarmed) {
1255                                 if (kprobe_disabled(old_p))
1256                                         disarm_kprobe(old_p);
1257                                 else
1258                                         /* Try to optimize this probe again */
1259                                         optimize_kprobe(old_p);
1260                         }
1261                 }
1262         }
1263         return 0;
1264 }
1265
1266 static void __kprobes __unregister_kprobe_bottom(struct kprobe *p)
1267 {
1268         struct kprobe *old_p;
1269
1270         if (list_empty(&p->list))
1271                 arch_remove_kprobe(p);
1272         else if (list_is_singular(&p->list)) {
1273                 /* "p" is the last child of an aggr_kprobe */
1274                 old_p = list_entry(p->list.next, struct kprobe, list);
1275                 list_del(&p->list);
1276                 arch_remove_kprobe(old_p);
1277                 free_aggr_kprobe(old_p);
1278         }
1279 }
1280
1281 int __kprobes register_kprobes(struct kprobe **kps, int num)
1282 {
1283         int i, ret = 0;
1284
1285         if (num <= 0)
1286                 return -EINVAL;
1287         for (i = 0; i < num; i++) {
1288                 ret = register_kprobe(kps[i]);
1289                 if (ret < 0) {
1290                         if (i > 0)
1291                                 unregister_kprobes(kps, i);
1292                         break;
1293                 }
1294         }
1295         return ret;
1296 }
1297 EXPORT_SYMBOL_GPL(register_kprobes);
1298
1299 void __kprobes unregister_kprobe(struct kprobe *p)
1300 {
1301         unregister_kprobes(&p, 1);
1302 }
1303 EXPORT_SYMBOL_GPL(unregister_kprobe);
1304
1305 void __kprobes unregister_kprobes(struct kprobe **kps, int num)
1306 {
1307         int i;
1308
1309         if (num <= 0)
1310                 return;
1311         mutex_lock(&kprobe_mutex);
1312         for (i = 0; i < num; i++)
1313                 if (__unregister_kprobe_top(kps[i]) < 0)
1314                         kps[i]->addr = NULL;
1315         mutex_unlock(&kprobe_mutex);
1316
1317         synchronize_sched();
1318         for (i = 0; i < num; i++)
1319                 if (kps[i]->addr)
1320                         __unregister_kprobe_bottom(kps[i]);
1321 }
1322 EXPORT_SYMBOL_GPL(unregister_kprobes);
1323
1324 static struct notifier_block kprobe_exceptions_nb = {
1325         .notifier_call = kprobe_exceptions_notify,
1326         .priority = 0x7fffffff /* we need to be notified first */
1327 };
1328
1329 unsigned long __weak arch_deref_entry_point(void *entry)
1330 {
1331         return (unsigned long)entry;
1332 }
1333
1334 int __kprobes register_jprobes(struct jprobe **jps, int num)
1335 {
1336         struct jprobe *jp;
1337         int ret = 0, i;
1338
1339         if (num <= 0)
1340                 return -EINVAL;
1341         for (i = 0; i < num; i++) {
1342                 unsigned long addr;
1343                 jp = jps[i];
1344                 addr = arch_deref_entry_point(jp->entry);
1345
1346                 if (!kernel_text_address(addr))
1347                         ret = -EINVAL;
1348                 else {
1349                         /* Todo: Verify probepoint is a function entry point */
1350                         jp->kp.pre_handler = setjmp_pre_handler;
1351                         jp->kp.break_handler = longjmp_break_handler;
1352                         ret = register_kprobe(&jp->kp);
1353                 }
1354                 if (ret < 0) {
1355                         if (i > 0)
1356                                 unregister_jprobes(jps, i);
1357                         break;
1358                 }
1359         }
1360         return ret;
1361 }
1362 EXPORT_SYMBOL_GPL(register_jprobes);
1363
1364 int __kprobes register_jprobe(struct jprobe *jp)
1365 {
1366         return register_jprobes(&jp, 1);
1367 }
1368 EXPORT_SYMBOL_GPL(register_jprobe);
1369
1370 void __kprobes unregister_jprobe(struct jprobe *jp)
1371 {
1372         unregister_jprobes(&jp, 1);
1373 }
1374 EXPORT_SYMBOL_GPL(unregister_jprobe);
1375
1376 void __kprobes unregister_jprobes(struct jprobe **jps, int num)
1377 {
1378         int i;
1379
1380         if (num <= 0)
1381                 return;
1382         mutex_lock(&kprobe_mutex);
1383         for (i = 0; i < num; i++)
1384                 if (__unregister_kprobe_top(&jps[i]->kp) < 0)
1385                         jps[i]->kp.addr = NULL;
1386         mutex_unlock(&kprobe_mutex);
1387
1388         synchronize_sched();
1389         for (i = 0; i < num; i++) {
1390                 if (jps[i]->kp.addr)
1391                         __unregister_kprobe_bottom(&jps[i]->kp);
1392         }
1393 }
1394 EXPORT_SYMBOL_GPL(unregister_jprobes);
1395
1396 #ifdef CONFIG_KRETPROBES
1397 /*
1398  * This kprobe pre_handler is registered with every kretprobe. When probe
1399  * hits it will set up the return probe.
1400  */
1401 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1402                                            struct pt_regs *regs)
1403 {
1404         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
1405         unsigned long hash, flags = 0;
1406         struct kretprobe_instance *ri;
1407
1408         /*TODO: consider to only swap the RA after the last pre_handler fired */
1409         hash = hash_ptr(current, KPROBE_HASH_BITS);
1410         spin_lock_irqsave(&rp->lock, flags);
1411         if (!hlist_empty(&rp->free_instances)) {
1412                 ri = hlist_entry(rp->free_instances.first,
1413                                 struct kretprobe_instance, hlist);
1414                 hlist_del(&ri->hlist);
1415                 spin_unlock_irqrestore(&rp->lock, flags);
1416
1417                 ri->rp = rp;
1418                 ri->task = current;
1419
1420                 if (rp->entry_handler && rp->entry_handler(ri, regs))
1421                         return 0;
1422
1423                 arch_prepare_kretprobe(ri, regs);
1424
1425                 /* XXX(hch): why is there no hlist_move_head? */
1426                 INIT_HLIST_NODE(&ri->hlist);
1427                 kretprobe_table_lock(hash, &flags);
1428                 hlist_add_head(&ri->hlist, &kretprobe_inst_table[hash]);
1429                 kretprobe_table_unlock(hash, &flags);
1430         } else {
1431                 rp->nmissed++;
1432                 spin_unlock_irqrestore(&rp->lock, flags);
1433         }
1434         return 0;
1435 }
1436
1437 int __kprobes register_kretprobe(struct kretprobe *rp)
1438 {
1439         int ret = 0;
1440         struct kretprobe_instance *inst;
1441         int i;
1442         void *addr;
1443
1444         if (kretprobe_blacklist_size) {
1445                 addr = kprobe_addr(&rp->kp);
1446                 if (!addr)
1447                         return -EINVAL;
1448
1449                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1450                         if (kretprobe_blacklist[i].addr == addr)
1451                                 return -EINVAL;
1452                 }
1453         }
1454
1455         rp->kp.pre_handler = pre_handler_kretprobe;
1456         rp->kp.post_handler = NULL;
1457         rp->kp.fault_handler = NULL;
1458         rp->kp.break_handler = NULL;
1459
1460         /* Pre-allocate memory for max kretprobe instances */
1461         if (rp->maxactive <= 0) {
1462 #ifdef CONFIG_PREEMPT
1463                 rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
1464 #else
1465                 rp->maxactive = num_possible_cpus();
1466 #endif
1467         }
1468         spin_lock_init(&rp->lock);
1469         INIT_HLIST_HEAD(&rp->free_instances);
1470         for (i = 0; i < rp->maxactive; i++) {
1471                 inst = kmalloc(sizeof(struct kretprobe_instance) +
1472                                rp->data_size, GFP_KERNEL);
1473                 if (inst == NULL) {
1474                         free_rp_inst(rp);
1475                         return -ENOMEM;
1476                 }
1477                 INIT_HLIST_NODE(&inst->hlist);
1478                 hlist_add_head(&inst->hlist, &rp->free_instances);
1479         }
1480
1481         rp->nmissed = 0;
1482         /* Establish function entry probe point */
1483         ret = register_kprobe(&rp->kp);
1484         if (ret != 0)
1485                 free_rp_inst(rp);
1486         return ret;
1487 }
1488 EXPORT_SYMBOL_GPL(register_kretprobe);
1489
1490 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1491 {
1492         int ret = 0, i;
1493
1494         if (num <= 0)
1495                 return -EINVAL;
1496         for (i = 0; i < num; i++) {
1497                 ret = register_kretprobe(rps[i]);
1498                 if (ret < 0) {
1499                         if (i > 0)
1500                                 unregister_kretprobes(rps, i);
1501                         break;
1502                 }
1503         }
1504         return ret;
1505 }
1506 EXPORT_SYMBOL_GPL(register_kretprobes);
1507
1508 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1509 {
1510         unregister_kretprobes(&rp, 1);
1511 }
1512 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1513
1514 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1515 {
1516         int i;
1517
1518         if (num <= 0)
1519                 return;
1520         mutex_lock(&kprobe_mutex);
1521         for (i = 0; i < num; i++)
1522                 if (__unregister_kprobe_top(&rps[i]->kp) < 0)
1523                         rps[i]->kp.addr = NULL;
1524         mutex_unlock(&kprobe_mutex);
1525
1526         synchronize_sched();
1527         for (i = 0; i < num; i++) {
1528                 if (rps[i]->kp.addr) {
1529                         __unregister_kprobe_bottom(&rps[i]->kp);
1530                         cleanup_rp_inst(rps[i]);
1531                 }
1532         }
1533 }
1534 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1535
1536 #else /* CONFIG_KRETPROBES */
1537 int __kprobes register_kretprobe(struct kretprobe *rp)
1538 {
1539         return -ENOSYS;
1540 }
1541 EXPORT_SYMBOL_GPL(register_kretprobe);
1542
1543 int __kprobes register_kretprobes(struct kretprobe **rps, int num)
1544 {
1545         return -ENOSYS;
1546 }
1547 EXPORT_SYMBOL_GPL(register_kretprobes);
1548
1549 void __kprobes unregister_kretprobe(struct kretprobe *rp)
1550 {
1551 }
1552 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1553
1554 void __kprobes unregister_kretprobes(struct kretprobe **rps, int num)
1555 {
1556 }
1557 EXPORT_SYMBOL_GPL(unregister_kretprobes);
1558
1559 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
1560                                            struct pt_regs *regs)
1561 {
1562         return 0;
1563 }
1564
1565 #endif /* CONFIG_KRETPROBES */
1566
1567 /* Set the kprobe gone and remove its instruction buffer. */
1568 static void __kprobes kill_kprobe(struct kprobe *p)
1569 {
1570         struct kprobe *kp;
1571
1572         p->flags |= KPROBE_FLAG_GONE;
1573         if (kprobe_aggrprobe(p)) {
1574                 /*
1575                  * If this is an aggr_kprobe, we have to list all the
1576                  * chained probes and mark them GONE.
1577                  */
1578                 list_for_each_entry_rcu(kp, &p->list, list)
1579                         kp->flags |= KPROBE_FLAG_GONE;
1580                 p->post_handler = NULL;
1581                 p->break_handler = NULL;
1582                 kill_optimized_kprobe(p);
1583         }
1584         /*
1585          * Here, we can remove insn_slot safely, because no thread calls
1586          * the original probed function (which will be freed soon) any more.
1587          */
1588         arch_remove_kprobe(p);
1589 }
1590
1591 void __kprobes dump_kprobe(struct kprobe *kp)
1592 {
1593         printk(KERN_WARNING "Dumping kprobe:\n");
1594         printk(KERN_WARNING "Name: %s\nAddress: %p\nOffset: %x\n",
1595                kp->symbol_name, kp->addr, kp->offset);
1596 }
1597
1598 /* Module notifier call back, checking kprobes on the module */
1599 static int __kprobes kprobes_module_callback(struct notifier_block *nb,
1600                                              unsigned long val, void *data)
1601 {
1602         struct module *mod = data;
1603         struct hlist_head *head;
1604         struct hlist_node *node;
1605         struct kprobe *p;
1606         unsigned int i;
1607         int checkcore = (val == MODULE_STATE_GOING);
1608
1609         if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
1610                 return NOTIFY_DONE;
1611
1612         /*
1613          * When MODULE_STATE_GOING was notified, both of module .text and
1614          * .init.text sections would be freed. When MODULE_STATE_LIVE was
1615          * notified, only .init.text section would be freed. We need to
1616          * disable kprobes which have been inserted in the sections.
1617          */
1618         mutex_lock(&kprobe_mutex);
1619         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1620                 head = &kprobe_table[i];
1621                 hlist_for_each_entry_rcu(p, node, head, hlist)
1622                         if (within_module_init((unsigned long)p->addr, mod) ||
1623                             (checkcore &&
1624                              within_module_core((unsigned long)p->addr, mod))) {
1625                                 /*
1626                                  * The vaddr this probe is installed will soon
1627                                  * be vfreed buy not synced to disk. Hence,
1628                                  * disarming the breakpoint isn't needed.
1629                                  */
1630                                 kill_kprobe(p);
1631                         }
1632         }
1633         mutex_unlock(&kprobe_mutex);
1634         return NOTIFY_DONE;
1635 }
1636
1637 static struct notifier_block kprobe_module_nb = {
1638         .notifier_call = kprobes_module_callback,
1639         .priority = 0
1640 };
1641
1642 static int __init init_kprobes(void)
1643 {
1644         int i, err = 0;
1645         unsigned long offset = 0, size = 0;
1646         char *modname, namebuf[128];
1647         const char *symbol_name;
1648         void *addr;
1649         struct kprobe_blackpoint *kb;
1650
1651         /* FIXME allocate the probe table, currently defined statically */
1652         /* initialize all list heads */
1653         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1654                 INIT_HLIST_HEAD(&kprobe_table[i]);
1655                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
1656                 spin_lock_init(&(kretprobe_table_locks[i].lock));
1657         }
1658
1659         /*
1660          * Lookup and populate the kprobe_blacklist.
1661          *
1662          * Unlike the kretprobe blacklist, we'll need to determine
1663          * the range of addresses that belong to the said functions,
1664          * since a kprobe need not necessarily be at the beginning
1665          * of a function.
1666          */
1667         for (kb = kprobe_blacklist; kb->name != NULL; kb++) {
1668                 kprobe_lookup_name(kb->name, addr);
1669                 if (!addr)
1670                         continue;
1671
1672                 kb->start_addr = (unsigned long)addr;
1673                 symbol_name = kallsyms_lookup(kb->start_addr,
1674                                 &size, &offset, &modname, namebuf);
1675                 if (!symbol_name)
1676                         kb->range = 0;
1677                 else
1678                         kb->range = size;
1679         }
1680
1681         if (kretprobe_blacklist_size) {
1682                 /* lookup the function address from its name */
1683                 for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
1684                         kprobe_lookup_name(kretprobe_blacklist[i].name,
1685                                            kretprobe_blacklist[i].addr);
1686                         if (!kretprobe_blacklist[i].addr)
1687                                 printk("kretprobe: lookup failed: %s\n",
1688                                        kretprobe_blacklist[i].name);
1689                 }
1690         }
1691
1692 #if defined(CONFIG_OPTPROBES)
1693 #if defined(__ARCH_WANT_KPROBES_INSN_SLOT)
1694         /* Init kprobe_optinsn_slots */
1695         kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
1696 #endif
1697         /* By default, kprobes can be optimized */
1698         kprobes_allow_optimization = true;
1699 #endif
1700
1701         /* By default, kprobes are armed */
1702         kprobes_all_disarmed = false;
1703
1704         err = arch_init_kprobes();
1705         if (!err)
1706                 err = register_die_notifier(&kprobe_exceptions_nb);
1707         if (!err)
1708                 err = register_module_notifier(&kprobe_module_nb);
1709
1710         kprobes_initialized = (err == 0);
1711
1712         if (!err)
1713                 init_test_probes();
1714         return err;
1715 }
1716
1717 #ifdef CONFIG_DEBUG_FS
1718 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
1719                 const char *sym, int offset, char *modname, struct kprobe *pp)
1720 {
1721         char *kprobe_type;
1722
1723         if (p->pre_handler == pre_handler_kretprobe)
1724                 kprobe_type = "r";
1725         else if (p->pre_handler == setjmp_pre_handler)
1726                 kprobe_type = "j";
1727         else
1728                 kprobe_type = "k";
1729
1730         if (sym)
1731                 seq_printf(pi, "%p  %s  %s+0x%x  %s ",
1732                         p->addr, kprobe_type, sym, offset,
1733                         (modname ? modname : " "));
1734         else
1735                 seq_printf(pi, "%p  %s  %p ",
1736                         p->addr, kprobe_type, p->addr);
1737
1738         if (!pp)
1739                 pp = p;
1740         seq_printf(pi, "%s%s%s\n",
1741                 (kprobe_gone(p) ? "[GONE]" : ""),
1742                 ((kprobe_disabled(p) && !kprobe_gone(p)) ?  "[DISABLED]" : ""),
1743                 (kprobe_optimized(pp) ? "[OPTIMIZED]" : ""));
1744 }
1745
1746 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
1747 {
1748         return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
1749 }
1750
1751 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
1752 {
1753         (*pos)++;
1754         if (*pos >= KPROBE_TABLE_SIZE)
1755                 return NULL;
1756         return pos;
1757 }
1758
1759 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
1760 {
1761         /* Nothing to do */
1762 }
1763
1764 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
1765 {
1766         struct hlist_head *head;
1767         struct hlist_node *node;
1768         struct kprobe *p, *kp;
1769         const char *sym = NULL;
1770         unsigned int i = *(loff_t *) v;
1771         unsigned long offset = 0;
1772         char *modname, namebuf[128];
1773
1774         head = &kprobe_table[i];
1775         preempt_disable();
1776         hlist_for_each_entry_rcu(p, node, head, hlist) {
1777                 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
1778                                         &offset, &modname, namebuf);
1779                 if (kprobe_aggrprobe(p)) {
1780                         list_for_each_entry_rcu(kp, &p->list, list)
1781                                 report_probe(pi, kp, sym, offset, modname, p);
1782                 } else
1783                         report_probe(pi, p, sym, offset, modname, NULL);
1784         }
1785         preempt_enable();
1786         return 0;
1787 }
1788
1789 static const struct seq_operations kprobes_seq_ops = {
1790         .start = kprobe_seq_start,
1791         .next  = kprobe_seq_next,
1792         .stop  = kprobe_seq_stop,
1793         .show  = show_kprobe_addr
1794 };
1795
1796 static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
1797 {
1798         return seq_open(filp, &kprobes_seq_ops);
1799 }
1800
1801 static const struct file_operations debugfs_kprobes_operations = {
1802         .open           = kprobes_open,
1803         .read           = seq_read,
1804         .llseek         = seq_lseek,
1805         .release        = seq_release,
1806 };
1807
1808 /* Disable one kprobe */
1809 int __kprobes disable_kprobe(struct kprobe *kp)
1810 {
1811         int ret = 0;
1812         struct kprobe *p;
1813
1814         mutex_lock(&kprobe_mutex);
1815
1816         /* Check whether specified probe is valid. */
1817         p = __get_valid_kprobe(kp);
1818         if (unlikely(p == NULL)) {
1819                 ret = -EINVAL;
1820                 goto out;
1821         }
1822
1823         /* If the probe is already disabled (or gone), just return */
1824         if (kprobe_disabled(kp))
1825                 goto out;
1826
1827         kp->flags |= KPROBE_FLAG_DISABLED;
1828         if (p != kp)
1829                 /* When kp != p, p is always enabled. */
1830                 try_to_disable_aggr_kprobe(p);
1831
1832         if (!kprobes_all_disarmed && kprobe_disabled(p))
1833                 disarm_kprobe(p);
1834 out:
1835         mutex_unlock(&kprobe_mutex);
1836         return ret;
1837 }
1838 EXPORT_SYMBOL_GPL(disable_kprobe);
1839
1840 /* Enable one kprobe */
1841 int __kprobes enable_kprobe(struct kprobe *kp)
1842 {
1843         int ret = 0;
1844         struct kprobe *p;
1845
1846         mutex_lock(&kprobe_mutex);
1847
1848         /* Check whether specified probe is valid. */
1849         p = __get_valid_kprobe(kp);
1850         if (unlikely(p == NULL)) {
1851                 ret = -EINVAL;
1852                 goto out;
1853         }
1854
1855         if (kprobe_gone(kp)) {
1856                 /* This kprobe has gone, we couldn't enable it. */
1857                 ret = -EINVAL;
1858                 goto out;
1859         }
1860
1861         if (p != kp)
1862                 kp->flags &= ~KPROBE_FLAG_DISABLED;
1863
1864         if (!kprobes_all_disarmed && kprobe_disabled(p)) {
1865                 p->flags &= ~KPROBE_FLAG_DISABLED;
1866                 arm_kprobe(p);
1867         }
1868 out:
1869         mutex_unlock(&kprobe_mutex);
1870         return ret;
1871 }
1872 EXPORT_SYMBOL_GPL(enable_kprobe);
1873
1874 static void __kprobes arm_all_kprobes(void)
1875 {
1876         struct hlist_head *head;
1877         struct hlist_node *node;
1878         struct kprobe *p;
1879         unsigned int i;
1880
1881         mutex_lock(&kprobe_mutex);
1882
1883         /* If kprobes are armed, just return */
1884         if (!kprobes_all_disarmed)
1885                 goto already_enabled;
1886
1887         /* Arming kprobes doesn't optimize kprobe itself */
1888         mutex_lock(&text_mutex);
1889         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1890                 head = &kprobe_table[i];
1891                 hlist_for_each_entry_rcu(p, node, head, hlist)
1892                         if (!kprobe_disabled(p))
1893                                 __arm_kprobe(p);
1894         }
1895         mutex_unlock(&text_mutex);
1896
1897         kprobes_all_disarmed = false;
1898         printk(KERN_INFO "Kprobes globally enabled\n");
1899
1900 already_enabled:
1901         mutex_unlock(&kprobe_mutex);
1902         return;
1903 }
1904
1905 static void __kprobes disarm_all_kprobes(void)
1906 {
1907         struct hlist_head *head;
1908         struct hlist_node *node;
1909         struct kprobe *p;
1910         unsigned int i;
1911
1912         mutex_lock(&kprobe_mutex);
1913
1914         /* If kprobes are already disarmed, just return */
1915         if (kprobes_all_disarmed)
1916                 goto already_disabled;
1917
1918         kprobes_all_disarmed = true;
1919         printk(KERN_INFO "Kprobes globally disabled\n");
1920
1921         /*
1922          * Here we call get_online_cpus() for avoiding text_mutex deadlock,
1923          * because disarming may also unoptimize kprobes.
1924          */
1925         get_online_cpus();
1926         mutex_lock(&text_mutex);
1927         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1928                 head = &kprobe_table[i];
1929                 hlist_for_each_entry_rcu(p, node, head, hlist) {
1930                         if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p))
1931                                 __disarm_kprobe(p);
1932                 }
1933         }
1934
1935         mutex_unlock(&text_mutex);
1936         put_online_cpus();
1937         mutex_unlock(&kprobe_mutex);
1938         /* Allow all currently running kprobes to complete */
1939         synchronize_sched();
1940         return;
1941
1942 already_disabled:
1943         mutex_unlock(&kprobe_mutex);
1944         return;
1945 }
1946
1947 /*
1948  * XXX: The debugfs bool file interface doesn't allow for callbacks
1949  * when the bool state is switched. We can reuse that facility when
1950  * available
1951  */
1952 static ssize_t read_enabled_file_bool(struct file *file,
1953                char __user *user_buf, size_t count, loff_t *ppos)
1954 {
1955         char buf[3];
1956
1957         if (!kprobes_all_disarmed)
1958                 buf[0] = '1';
1959         else
1960                 buf[0] = '0';
1961         buf[1] = '\n';
1962         buf[2] = 0x00;
1963         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
1964 }
1965
1966 static ssize_t write_enabled_file_bool(struct file *file,
1967                const char __user *user_buf, size_t count, loff_t *ppos)
1968 {
1969         char buf[32];
1970         int buf_size;
1971
1972         buf_size = min(count, (sizeof(buf)-1));
1973         if (copy_from_user(buf, user_buf, buf_size))
1974                 return -EFAULT;
1975
1976         switch (buf[0]) {
1977         case 'y':
1978         case 'Y':
1979         case '1':
1980                 arm_all_kprobes();
1981                 break;
1982         case 'n':
1983         case 'N':
1984         case '0':
1985                 disarm_all_kprobes();
1986                 break;
1987         }
1988
1989         return count;
1990 }
1991
1992 static const struct file_operations fops_kp = {
1993         .read =         read_enabled_file_bool,
1994         .write =        write_enabled_file_bool,
1995 };
1996
1997 static int __kprobes debugfs_kprobe_init(void)
1998 {
1999         struct dentry *dir, *file;
2000         unsigned int value = 1;
2001
2002         dir = debugfs_create_dir("kprobes", NULL);
2003         if (!dir)
2004                 return -ENOMEM;
2005
2006         file = debugfs_create_file("list", 0444, dir, NULL,
2007                                 &debugfs_kprobes_operations);
2008         if (!file) {
2009                 debugfs_remove(dir);
2010                 return -ENOMEM;
2011         }
2012
2013         file = debugfs_create_file("enabled", 0600, dir,
2014                                         &value, &fops_kp);
2015         if (!file) {
2016                 debugfs_remove(dir);
2017                 return -ENOMEM;
2018         }
2019
2020         return 0;
2021 }
2022
2023 late_initcall(debugfs_kprobe_init);
2024 #endif /* CONFIG_DEBUG_FS */
2025
2026 module_init(init_kprobes);
2027
2028 /* defined in arch/.../kernel/kprobes.c */
2029 EXPORT_SYMBOL_GPL(jprobe_return);