f9798ff7899f4a610823fd641fe5e2dde0e746f7
[linux-2.6.git] / kernel / kprobes.c
1 /*
2  *  Kernel Probes (KProbes)
3  *  kernel/kprobes.c
4  *
5  * This program is free software; you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * This program is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with this program; if not, write to the Free Software
17  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18  *
19  * Copyright (C) IBM Corporation, 2002, 2004
20  *
21  * 2002-Oct     Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
22  *              Probes initial implementation (includes suggestions from
23  *              Rusty Russell).
24  * 2004-Aug     Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
25  *              hlists and exceptions notifier as suggested by Andi Kleen.
26  * 2004-July    Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
27  *              interface to access function arguments.
28  * 2004-Sep     Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
29  *              exceptions notifier to be first on the priority list.
30  * 2005-May     Hien Nguyen <hien@us.ibm.com>, Jim Keniston
31  *              <jkenisto@us.ibm.com> and Prasanna S Panchamukhi
32  *              <prasanna@in.ibm.com> added function-return probes.
33  */
34 #include <linux/kprobes.h>
35 #include <linux/hash.h>
36 #include <linux/init.h>
37 #include <linux/slab.h>
38 #include <linux/stddef.h>
39 #include <linux/module.h>
40 #include <linux/moduleloader.h>
41 #include <linux/kallsyms.h>
42 #include <linux/freezer.h>
43 #include <linux/seq_file.h>
44 #include <linux/debugfs.h>
45 #include <linux/kdebug.h>
46
47 #include <asm-generic/sections.h>
48 #include <asm/cacheflush.h>
49 #include <asm/errno.h>
50 #include <asm/uaccess.h>
51
52 #define KPROBE_HASH_BITS 6
53 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
54
55
56 /*
57  * Some oddball architectures like 64bit powerpc have function descriptors
58  * so this must be overridable.
59  */
60 #ifndef kprobe_lookup_name
61 #define kprobe_lookup_name(name, addr) \
62         addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
63 #endif
64
65 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
66 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
67
68 /* NOTE: change this value only with kprobe_mutex held */
69 static bool kprobe_enabled;
70
71 DEFINE_MUTEX(kprobe_mutex);             /* Protects kprobe_table */
72 DEFINE_SPINLOCK(kretprobe_lock);        /* Protects kretprobe_inst_table */
73 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
74
75 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
76 /*
77  * kprobe->ainsn.insn points to the copy of the instruction to be
78  * single-stepped. x86_64, POWER4 and above have no-exec support and
79  * stepping on the instruction on a vmalloced/kmalloced/data page
80  * is a recipe for disaster
81  */
82 #define INSNS_PER_PAGE  (PAGE_SIZE/(MAX_INSN_SIZE * sizeof(kprobe_opcode_t)))
83
84 struct kprobe_insn_page {
85         struct hlist_node hlist;
86         kprobe_opcode_t *insns;         /* Page of instruction slots */
87         char slot_used[INSNS_PER_PAGE];
88         int nused;
89         int ngarbage;
90 };
91
92 enum kprobe_slot_state {
93         SLOT_CLEAN = 0,
94         SLOT_DIRTY = 1,
95         SLOT_USED = 2,
96 };
97
98 static struct hlist_head kprobe_insn_pages;
99 static int kprobe_garbage_slots;
100 static int collect_garbage_slots(void);
101
102 static int __kprobes check_safety(void)
103 {
104         int ret = 0;
105 #if defined(CONFIG_PREEMPT) && defined(CONFIG_PM)
106         ret = freeze_processes();
107         if (ret == 0) {
108                 struct task_struct *p, *q;
109                 do_each_thread(p, q) {
110                         if (p != current && p->state == TASK_RUNNING &&
111                             p->pid != 0) {
112                                 printk("Check failed: %s is running\n",p->comm);
113                                 ret = -1;
114                                 goto loop_end;
115                         }
116                 } while_each_thread(p, q);
117         }
118 loop_end:
119         thaw_processes();
120 #else
121         synchronize_sched();
122 #endif
123         return ret;
124 }
125
126 /**
127  * get_insn_slot() - Find a slot on an executable page for an instruction.
128  * We allocate an executable page if there's no room on existing ones.
129  */
130 kprobe_opcode_t __kprobes *get_insn_slot(void)
131 {
132         struct kprobe_insn_page *kip;
133         struct hlist_node *pos;
134
135  retry:
136         hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
137                 if (kip->nused < INSNS_PER_PAGE) {
138                         int i;
139                         for (i = 0; i < INSNS_PER_PAGE; i++) {
140                                 if (kip->slot_used[i] == SLOT_CLEAN) {
141                                         kip->slot_used[i] = SLOT_USED;
142                                         kip->nused++;
143                                         return kip->insns + (i * MAX_INSN_SIZE);
144                                 }
145                         }
146                         /* Surprise!  No unused slots.  Fix kip->nused. */
147                         kip->nused = INSNS_PER_PAGE;
148                 }
149         }
150
151         /* If there are any garbage slots, collect it and try again. */
152         if (kprobe_garbage_slots && collect_garbage_slots() == 0) {
153                 goto retry;
154         }
155         /* All out of space.  Need to allocate a new page. Use slot 0. */
156         kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL);
157         if (!kip)
158                 return NULL;
159
160         /*
161          * Use module_alloc so this page is within +/- 2GB of where the
162          * kernel image and loaded module images reside. This is required
163          * so x86_64 can correctly handle the %rip-relative fixups.
164          */
165         kip->insns = module_alloc(PAGE_SIZE);
166         if (!kip->insns) {
167                 kfree(kip);
168                 return NULL;
169         }
170         INIT_HLIST_NODE(&kip->hlist);
171         hlist_add_head(&kip->hlist, &kprobe_insn_pages);
172         memset(kip->slot_used, SLOT_CLEAN, INSNS_PER_PAGE);
173         kip->slot_used[0] = SLOT_USED;
174         kip->nused = 1;
175         kip->ngarbage = 0;
176         return kip->insns;
177 }
178
179 /* Return 1 if all garbages are collected, otherwise 0. */
180 static int __kprobes collect_one_slot(struct kprobe_insn_page *kip, int idx)
181 {
182         kip->slot_used[idx] = SLOT_CLEAN;
183         kip->nused--;
184         if (kip->nused == 0) {
185                 /*
186                  * Page is no longer in use.  Free it unless
187                  * it's the last one.  We keep the last one
188                  * so as not to have to set it up again the
189                  * next time somebody inserts a probe.
190                  */
191                 hlist_del(&kip->hlist);
192                 if (hlist_empty(&kprobe_insn_pages)) {
193                         INIT_HLIST_NODE(&kip->hlist);
194                         hlist_add_head(&kip->hlist,
195                                        &kprobe_insn_pages);
196                 } else {
197                         module_free(NULL, kip->insns);
198                         kfree(kip);
199                 }
200                 return 1;
201         }
202         return 0;
203 }
204
205 static int __kprobes collect_garbage_slots(void)
206 {
207         struct kprobe_insn_page *kip;
208         struct hlist_node *pos, *next;
209
210         /* Ensure no-one is preepmted on the garbages */
211         if (check_safety() != 0)
212                 return -EAGAIN;
213
214         hlist_for_each_entry_safe(kip, pos, next, &kprobe_insn_pages, hlist) {
215                 int i;
216                 if (kip->ngarbage == 0)
217                         continue;
218                 kip->ngarbage = 0;      /* we will collect all garbages */
219                 for (i = 0; i < INSNS_PER_PAGE; i++) {
220                         if (kip->slot_used[i] == SLOT_DIRTY &&
221                             collect_one_slot(kip, i))
222                                 break;
223                 }
224         }
225         kprobe_garbage_slots = 0;
226         return 0;
227 }
228
229 void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty)
230 {
231         struct kprobe_insn_page *kip;
232         struct hlist_node *pos;
233
234         hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) {
235                 if (kip->insns <= slot &&
236                     slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) {
237                         int i = (slot - kip->insns) / MAX_INSN_SIZE;
238                         if (dirty) {
239                                 kip->slot_used[i] = SLOT_DIRTY;
240                                 kip->ngarbage++;
241                         } else {
242                                 collect_one_slot(kip, i);
243                         }
244                         break;
245                 }
246         }
247
248         if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE)
249                 collect_garbage_slots();
250 }
251 #endif
252
253 /* We have preemption disabled.. so it is safe to use __ versions */
254 static inline void set_kprobe_instance(struct kprobe *kp)
255 {
256         __get_cpu_var(kprobe_instance) = kp;
257 }
258
259 static inline void reset_kprobe_instance(void)
260 {
261         __get_cpu_var(kprobe_instance) = NULL;
262 }
263
264 /*
265  * This routine is called either:
266  *      - under the kprobe_mutex - during kprobe_[un]register()
267  *                              OR
268  *      - with preemption disabled - from arch/xxx/kernel/kprobes.c
269  */
270 struct kprobe __kprobes *get_kprobe(void *addr)
271 {
272         struct hlist_head *head;
273         struct hlist_node *node;
274         struct kprobe *p;
275
276         head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
277         hlist_for_each_entry_rcu(p, node, head, hlist) {
278                 if (p->addr == addr)
279                         return p;
280         }
281         return NULL;
282 }
283
284 /*
285  * Aggregate handlers for multiple kprobes support - these handlers
286  * take care of invoking the individual kprobe handlers on p->list
287  */
288 static int __kprobes aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
289 {
290         struct kprobe *kp;
291
292         list_for_each_entry_rcu(kp, &p->list, list) {
293                 if (kp->pre_handler) {
294                         set_kprobe_instance(kp);
295                         if (kp->pre_handler(kp, regs))
296                                 return 1;
297                 }
298                 reset_kprobe_instance();
299         }
300         return 0;
301 }
302
303 static void __kprobes aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
304                                         unsigned long flags)
305 {
306         struct kprobe *kp;
307
308         list_for_each_entry_rcu(kp, &p->list, list) {
309                 if (kp->post_handler) {
310                         set_kprobe_instance(kp);
311                         kp->post_handler(kp, regs, flags);
312                         reset_kprobe_instance();
313                 }
314         }
315 }
316
317 static int __kprobes aggr_fault_handler(struct kprobe *p, struct pt_regs *regs,
318                                         int trapnr)
319 {
320         struct kprobe *cur = __get_cpu_var(kprobe_instance);
321
322         /*
323          * if we faulted "during" the execution of a user specified
324          * probe handler, invoke just that probe's fault handler
325          */
326         if (cur && cur->fault_handler) {
327                 if (cur->fault_handler(cur, regs, trapnr))
328                         return 1;
329         }
330         return 0;
331 }
332
333 static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
334 {
335         struct kprobe *cur = __get_cpu_var(kprobe_instance);
336         int ret = 0;
337
338         if (cur && cur->break_handler) {
339                 if (cur->break_handler(cur, regs))
340                         ret = 1;
341         }
342         reset_kprobe_instance();
343         return ret;
344 }
345
346 /* Walks the list and increments nmissed count for multiprobe case */
347 void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
348 {
349         struct kprobe *kp;
350         if (p->pre_handler != aggr_pre_handler) {
351                 p->nmissed++;
352         } else {
353                 list_for_each_entry_rcu(kp, &p->list, list)
354                         kp->nmissed++;
355         }
356         return;
357 }
358
359 /* Called with kretprobe_lock held */
360 void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
361                                 struct hlist_head *head)
362 {
363         /* remove rp inst off the rprobe_inst_table */
364         hlist_del(&ri->hlist);
365         if (ri->rp) {
366                 /* remove rp inst off the used list */
367                 hlist_del(&ri->uflist);
368                 /* put rp inst back onto the free list */
369                 INIT_HLIST_NODE(&ri->uflist);
370                 hlist_add_head(&ri->uflist, &ri->rp->free_instances);
371         } else
372                 /* Unregistering */
373                 hlist_add_head(&ri->hlist, head);
374 }
375
376 struct hlist_head __kprobes *kretprobe_inst_table_head(struct task_struct *tsk)
377 {
378         return &kretprobe_inst_table[hash_ptr(tsk, KPROBE_HASH_BITS)];
379 }
380
381 /*
382  * This function is called from finish_task_switch when task tk becomes dead,
383  * so that we can recycle any function-return probe instances associated
384  * with this task. These left over instances represent probed functions
385  * that have been called but will never return.
386  */
387 void __kprobes kprobe_flush_task(struct task_struct *tk)
388 {
389         struct kretprobe_instance *ri;
390         struct hlist_head *head, empty_rp;
391         struct hlist_node *node, *tmp;
392         unsigned long flags = 0;
393
394         INIT_HLIST_HEAD(&empty_rp);
395         spin_lock_irqsave(&kretprobe_lock, flags);
396         head = kretprobe_inst_table_head(tk);
397         hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
398                 if (ri->task == tk)
399                         recycle_rp_inst(ri, &empty_rp);
400         }
401         spin_unlock_irqrestore(&kretprobe_lock, flags);
402
403         hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
404                 hlist_del(&ri->hlist);
405                 kfree(ri);
406         }
407 }
408
409 static inline void free_rp_inst(struct kretprobe *rp)
410 {
411         struct kretprobe_instance *ri;
412         struct hlist_node *pos, *next;
413
414         hlist_for_each_entry_safe(ri, pos, next, &rp->free_instances, uflist) {
415                 hlist_del(&ri->uflist);
416                 kfree(ri);
417         }
418 }
419
420 /*
421  * Keep all fields in the kprobe consistent
422  */
423 static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
424 {
425         memcpy(&p->opcode, &old_p->opcode, sizeof(kprobe_opcode_t));
426         memcpy(&p->ainsn, &old_p->ainsn, sizeof(struct arch_specific_insn));
427 }
428
429 /*
430 * Add the new probe to old_p->list. Fail if this is the
431 * second jprobe at the address - two jprobes can't coexist
432 */
433 static int __kprobes add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
434 {
435         if (p->break_handler) {
436                 if (old_p->break_handler)
437                         return -EEXIST;
438                 list_add_tail_rcu(&p->list, &old_p->list);
439                 old_p->break_handler = aggr_break_handler;
440         } else
441                 list_add_rcu(&p->list, &old_p->list);
442         if (p->post_handler && !old_p->post_handler)
443                 old_p->post_handler = aggr_post_handler;
444         return 0;
445 }
446
447 /*
448  * Fill in the required fields of the "manager kprobe". Replace the
449  * earlier kprobe in the hlist with the manager kprobe
450  */
451 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
452 {
453         copy_kprobe(p, ap);
454         flush_insn_slot(ap);
455         ap->addr = p->addr;
456         ap->pre_handler = aggr_pre_handler;
457         ap->fault_handler = aggr_fault_handler;
458         if (p->post_handler)
459                 ap->post_handler = aggr_post_handler;
460         if (p->break_handler)
461                 ap->break_handler = aggr_break_handler;
462
463         INIT_LIST_HEAD(&ap->list);
464         list_add_rcu(&p->list, &ap->list);
465
466         hlist_replace_rcu(&p->hlist, &ap->hlist);
467 }
468
469 /*
470  * This is the second or subsequent kprobe at the address - handle
471  * the intricacies
472  */
473 static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
474                                           struct kprobe *p)
475 {
476         int ret = 0;
477         struct kprobe *ap;
478
479         if (old_p->pre_handler == aggr_pre_handler) {
480                 copy_kprobe(old_p, p);
481                 ret = add_new_kprobe(old_p, p);
482         } else {
483                 ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
484                 if (!ap)
485                         return -ENOMEM;
486                 add_aggr_kprobe(ap, old_p);
487                 copy_kprobe(ap, p);
488                 ret = add_new_kprobe(ap, p);
489         }
490         return ret;
491 }
492
493 static int __kprobes in_kprobes_functions(unsigned long addr)
494 {
495         if (addr >= (unsigned long)__kprobes_text_start &&
496             addr < (unsigned long)__kprobes_text_end)
497                 return -EINVAL;
498         return 0;
499 }
500
501 static int __kprobes __register_kprobe(struct kprobe *p,
502         unsigned long called_from)
503 {
504         int ret = 0;
505         struct kprobe *old_p;
506         struct module *probed_mod;
507
508         /*
509          * If we have a symbol_name argument look it up,
510          * and add it to the address.  That way the addr
511          * field can either be global or relative to a symbol.
512          */
513         if (p->symbol_name) {
514                 if (p->addr)
515                         return -EINVAL;
516                 kprobe_lookup_name(p->symbol_name, p->addr);
517         }
518
519         if (!p->addr)
520                 return -EINVAL;
521         p->addr = (kprobe_opcode_t *)(((char *)p->addr)+ p->offset);
522
523         if (!kernel_text_address((unsigned long) p->addr) ||
524             in_kprobes_functions((unsigned long) p->addr))
525                 return -EINVAL;
526
527         p->mod_refcounted = 0;
528
529         /*
530          * Check if are we probing a module.
531          */
532         probed_mod = module_text_address((unsigned long) p->addr);
533         if (probed_mod) {
534                 struct module *calling_mod = module_text_address(called_from);
535                 /*
536                  * We must allow modules to probe themself and in this case
537                  * avoid incrementing the module refcount, so as to allow
538                  * unloading of self probing modules.
539                  */
540                 if (calling_mod && calling_mod != probed_mod) {
541                         if (unlikely(!try_module_get(probed_mod)))
542                                 return -EINVAL;
543                         p->mod_refcounted = 1;
544                 } else
545                         probed_mod = NULL;
546         }
547
548         p->nmissed = 0;
549         mutex_lock(&kprobe_mutex);
550         old_p = get_kprobe(p->addr);
551         if (old_p) {
552                 ret = register_aggr_kprobe(old_p, p);
553                 goto out;
554         }
555
556         ret = arch_prepare_kprobe(p);
557         if (ret)
558                 goto out;
559
560         INIT_HLIST_NODE(&p->hlist);
561         hlist_add_head_rcu(&p->hlist,
562                        &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
563
564         if (kprobe_enabled)
565                 arch_arm_kprobe(p);
566
567 out:
568         mutex_unlock(&kprobe_mutex);
569
570         if (ret && probed_mod)
571                 module_put(probed_mod);
572         return ret;
573 }
574
575 int __kprobes register_kprobe(struct kprobe *p)
576 {
577         return __register_kprobe(p, (unsigned long)__builtin_return_address(0));
578 }
579
580 void __kprobes unregister_kprobe(struct kprobe *p)
581 {
582         struct module *mod;
583         struct kprobe *old_p, *list_p;
584         int cleanup_p;
585
586         mutex_lock(&kprobe_mutex);
587         old_p = get_kprobe(p->addr);
588         if (unlikely(!old_p)) {
589                 mutex_unlock(&kprobe_mutex);
590                 return;
591         }
592         if (p != old_p) {
593                 list_for_each_entry_rcu(list_p, &old_p->list, list)
594                         if (list_p == p)
595                         /* kprobe p is a valid probe */
596                                 goto valid_p;
597                 mutex_unlock(&kprobe_mutex);
598                 return;
599         }
600 valid_p:
601         if (old_p == p ||
602             (old_p->pre_handler == aggr_pre_handler &&
603              p->list.next == &old_p->list && p->list.prev == &old_p->list)) {
604                 /*
605                  * Only probe on the hash list. Disarm only if kprobes are
606                  * enabled - otherwise, the breakpoint would already have
607                  * been removed. We save on flushing icache.
608                  */
609                 if (kprobe_enabled)
610                         arch_disarm_kprobe(p);
611                 hlist_del_rcu(&old_p->hlist);
612                 cleanup_p = 1;
613         } else {
614                 list_del_rcu(&p->list);
615                 cleanup_p = 0;
616         }
617
618         mutex_unlock(&kprobe_mutex);
619
620         synchronize_sched();
621         if (p->mod_refcounted) {
622                 mod = module_text_address((unsigned long)p->addr);
623                 if (mod)
624                         module_put(mod);
625         }
626
627         if (cleanup_p) {
628                 if (p != old_p) {
629                         list_del_rcu(&p->list);
630                         kfree(old_p);
631                 }
632                 arch_remove_kprobe(p);
633         } else {
634                 mutex_lock(&kprobe_mutex);
635                 if (p->break_handler)
636                         old_p->break_handler = NULL;
637                 if (p->post_handler){
638                         list_for_each_entry_rcu(list_p, &old_p->list, list){
639                                 if (list_p->post_handler){
640                                         cleanup_p = 2;
641                                         break;
642                                 }
643                         }
644                         if (cleanup_p == 0)
645                                 old_p->post_handler = NULL;
646                 }
647                 mutex_unlock(&kprobe_mutex);
648         }
649 }
650
651 static struct notifier_block kprobe_exceptions_nb = {
652         .notifier_call = kprobe_exceptions_notify,
653         .priority = 0x7fffffff /* we need to be notified first */
654 };
655
656 unsigned long __weak arch_deref_entry_point(void *entry)
657 {
658         return (unsigned long)entry;
659 }
660
661 int __kprobes register_jprobe(struct jprobe *jp)
662 {
663         unsigned long addr = arch_deref_entry_point(jp->entry);
664
665         if (!kernel_text_address(addr))
666                 return -EINVAL;
667
668         /* Todo: Verify probepoint is a function entry point */
669         jp->kp.pre_handler = setjmp_pre_handler;
670         jp->kp.break_handler = longjmp_break_handler;
671
672         return __register_kprobe(&jp->kp,
673                 (unsigned long)__builtin_return_address(0));
674 }
675
676 void __kprobes unregister_jprobe(struct jprobe *jp)
677 {
678         unregister_kprobe(&jp->kp);
679 }
680
681 #ifdef ARCH_SUPPORTS_KRETPROBES
682
683 /*
684  * This kprobe pre_handler is registered with every kretprobe. When probe
685  * hits it will set up the return probe.
686  */
687 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
688                                            struct pt_regs *regs)
689 {
690         struct kretprobe *rp = container_of(p, struct kretprobe, kp);
691         unsigned long flags = 0;
692
693         /*TODO: consider to only swap the RA after the last pre_handler fired */
694         spin_lock_irqsave(&kretprobe_lock, flags);
695         if (!hlist_empty(&rp->free_instances)) {
696                 struct kretprobe_instance *ri;
697
698                 ri = hlist_entry(rp->free_instances.first,
699                                  struct kretprobe_instance, uflist);
700                 ri->rp = rp;
701                 ri->task = current;
702                 arch_prepare_kretprobe(ri, regs);
703
704                 /* XXX(hch): why is there no hlist_move_head? */
705                 hlist_del(&ri->uflist);
706                 hlist_add_head(&ri->uflist, &ri->rp->used_instances);
707                 hlist_add_head(&ri->hlist, kretprobe_inst_table_head(ri->task));
708         } else
709                 rp->nmissed++;
710         spin_unlock_irqrestore(&kretprobe_lock, flags);
711         return 0;
712 }
713
714 int __kprobes register_kretprobe(struct kretprobe *rp)
715 {
716         int ret = 0;
717         struct kretprobe_instance *inst;
718         int i;
719
720         rp->kp.pre_handler = pre_handler_kretprobe;
721         rp->kp.post_handler = NULL;
722         rp->kp.fault_handler = NULL;
723         rp->kp.break_handler = NULL;
724
725         /* Pre-allocate memory for max kretprobe instances */
726         if (rp->maxactive <= 0) {
727 #ifdef CONFIG_PREEMPT
728                 rp->maxactive = max(10, 2 * NR_CPUS);
729 #else
730                 rp->maxactive = NR_CPUS;
731 #endif
732         }
733         INIT_HLIST_HEAD(&rp->used_instances);
734         INIT_HLIST_HEAD(&rp->free_instances);
735         for (i = 0; i < rp->maxactive; i++) {
736                 inst = kmalloc(sizeof(struct kretprobe_instance), GFP_KERNEL);
737                 if (inst == NULL) {
738                         free_rp_inst(rp);
739                         return -ENOMEM;
740                 }
741                 INIT_HLIST_NODE(&inst->uflist);
742                 hlist_add_head(&inst->uflist, &rp->free_instances);
743         }
744
745         rp->nmissed = 0;
746         /* Establish function entry probe point */
747         if ((ret = __register_kprobe(&rp->kp,
748                 (unsigned long)__builtin_return_address(0))) != 0)
749                 free_rp_inst(rp);
750         return ret;
751 }
752
753 #else /* ARCH_SUPPORTS_KRETPROBES */
754
755 int __kprobes register_kretprobe(struct kretprobe *rp)
756 {
757         return -ENOSYS;
758 }
759
760 static int __kprobes pre_handler_kretprobe(struct kprobe *p,
761                                            struct pt_regs *regs)
762 {
763         return 0;
764 }
765
766 #endif /* ARCH_SUPPORTS_KRETPROBES */
767
768 void __kprobes unregister_kretprobe(struct kretprobe *rp)
769 {
770         unsigned long flags;
771         struct kretprobe_instance *ri;
772         struct hlist_node *pos, *next;
773
774         unregister_kprobe(&rp->kp);
775
776         /* No race here */
777         spin_lock_irqsave(&kretprobe_lock, flags);
778         hlist_for_each_entry_safe(ri, pos, next, &rp->used_instances, uflist) {
779                 ri->rp = NULL;
780                 hlist_del(&ri->uflist);
781         }
782         spin_unlock_irqrestore(&kretprobe_lock, flags);
783         free_rp_inst(rp);
784 }
785
786 static int __init init_kprobes(void)
787 {
788         int i, err = 0;
789
790         /* FIXME allocate the probe table, currently defined statically */
791         /* initialize all list heads */
792         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
793                 INIT_HLIST_HEAD(&kprobe_table[i]);
794                 INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
795         }
796
797         /* By default, kprobes are enabled */
798         kprobe_enabled = true;
799
800         err = arch_init_kprobes();
801         if (!err)
802                 err = register_die_notifier(&kprobe_exceptions_nb);
803
804         return err;
805 }
806
807 #ifdef CONFIG_DEBUG_FS
808 static void __kprobes report_probe(struct seq_file *pi, struct kprobe *p,
809                 const char *sym, int offset,char *modname)
810 {
811         char *kprobe_type;
812
813         if (p->pre_handler == pre_handler_kretprobe)
814                 kprobe_type = "r";
815         else if (p->pre_handler == setjmp_pre_handler)
816                 kprobe_type = "j";
817         else
818                 kprobe_type = "k";
819         if (sym)
820                 seq_printf(pi, "%p  %s  %s+0x%x  %s\n", p->addr, kprobe_type,
821                         sym, offset, (modname ? modname : " "));
822         else
823                 seq_printf(pi, "%p  %s  %p\n", p->addr, kprobe_type, p->addr);
824 }
825
826 static void __kprobes *kprobe_seq_start(struct seq_file *f, loff_t *pos)
827 {
828         return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
829 }
830
831 static void __kprobes *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
832 {
833         (*pos)++;
834         if (*pos >= KPROBE_TABLE_SIZE)
835                 return NULL;
836         return pos;
837 }
838
839 static void __kprobes kprobe_seq_stop(struct seq_file *f, void *v)
840 {
841         /* Nothing to do */
842 }
843
844 static int __kprobes show_kprobe_addr(struct seq_file *pi, void *v)
845 {
846         struct hlist_head *head;
847         struct hlist_node *node;
848         struct kprobe *p, *kp;
849         const char *sym = NULL;
850         unsigned int i = *(loff_t *) v;
851         unsigned long offset = 0;
852         char *modname, namebuf[128];
853
854         head = &kprobe_table[i];
855         preempt_disable();
856         hlist_for_each_entry_rcu(p, node, head, hlist) {
857                 sym = kallsyms_lookup((unsigned long)p->addr, NULL,
858                                         &offset, &modname, namebuf);
859                 if (p->pre_handler == aggr_pre_handler) {
860                         list_for_each_entry_rcu(kp, &p->list, list)
861                                 report_probe(pi, kp, sym, offset, modname);
862                 } else
863                         report_probe(pi, p, sym, offset, modname);
864         }
865         preempt_enable();
866         return 0;
867 }
868
869 static struct seq_operations kprobes_seq_ops = {
870         .start = kprobe_seq_start,
871         .next  = kprobe_seq_next,
872         .stop  = kprobe_seq_stop,
873         .show  = show_kprobe_addr
874 };
875
876 static int __kprobes kprobes_open(struct inode *inode, struct file *filp)
877 {
878         return seq_open(filp, &kprobes_seq_ops);
879 }
880
881 static struct file_operations debugfs_kprobes_operations = {
882         .open           = kprobes_open,
883         .read           = seq_read,
884         .llseek         = seq_lseek,
885         .release        = seq_release,
886 };
887
888 static void __kprobes enable_all_kprobes(void)
889 {
890         struct hlist_head *head;
891         struct hlist_node *node;
892         struct kprobe *p;
893         unsigned int i;
894
895         mutex_lock(&kprobe_mutex);
896
897         /* If kprobes are already enabled, just return */
898         if (kprobe_enabled)
899                 goto already_enabled;
900
901         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
902                 head = &kprobe_table[i];
903                 hlist_for_each_entry_rcu(p, node, head, hlist)
904                         arch_arm_kprobe(p);
905         }
906
907         kprobe_enabled = true;
908         printk(KERN_INFO "Kprobes globally enabled\n");
909
910 already_enabled:
911         mutex_unlock(&kprobe_mutex);
912         return;
913 }
914
915 static void __kprobes disable_all_kprobes(void)
916 {
917         struct hlist_head *head;
918         struct hlist_node *node;
919         struct kprobe *p;
920         unsigned int i;
921
922         mutex_lock(&kprobe_mutex);
923
924         /* If kprobes are already disabled, just return */
925         if (!kprobe_enabled)
926                 goto already_disabled;
927
928         kprobe_enabled = false;
929         printk(KERN_INFO "Kprobes globally disabled\n");
930         for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
931                 head = &kprobe_table[i];
932                 hlist_for_each_entry_rcu(p, node, head, hlist) {
933                         if (!arch_trampoline_kprobe(p))
934                                 arch_disarm_kprobe(p);
935                 }
936         }
937
938         mutex_unlock(&kprobe_mutex);
939         /* Allow all currently running kprobes to complete */
940         synchronize_sched();
941         return;
942
943 already_disabled:
944         mutex_unlock(&kprobe_mutex);
945         return;
946 }
947
948 /*
949  * XXX: The debugfs bool file interface doesn't allow for callbacks
950  * when the bool state is switched. We can reuse that facility when
951  * available
952  */
953 static ssize_t read_enabled_file_bool(struct file *file,
954                char __user *user_buf, size_t count, loff_t *ppos)
955 {
956         char buf[3];
957
958         if (kprobe_enabled)
959                 buf[0] = '1';
960         else
961                 buf[0] = '0';
962         buf[1] = '\n';
963         buf[2] = 0x00;
964         return simple_read_from_buffer(user_buf, count, ppos, buf, 2);
965 }
966
967 static ssize_t write_enabled_file_bool(struct file *file,
968                const char __user *user_buf, size_t count, loff_t *ppos)
969 {
970         char buf[32];
971         int buf_size;
972
973         buf_size = min(count, (sizeof(buf)-1));
974         if (copy_from_user(buf, user_buf, buf_size))
975                 return -EFAULT;
976
977         switch (buf[0]) {
978         case 'y':
979         case 'Y':
980         case '1':
981                 enable_all_kprobes();
982                 break;
983         case 'n':
984         case 'N':
985         case '0':
986                 disable_all_kprobes();
987                 break;
988         }
989
990         return count;
991 }
992
993 static struct file_operations fops_kp = {
994         .read =         read_enabled_file_bool,
995         .write =        write_enabled_file_bool,
996 };
997
998 static int __kprobes debugfs_kprobe_init(void)
999 {
1000         struct dentry *dir, *file;
1001         unsigned int value = 1;
1002
1003         dir = debugfs_create_dir("kprobes", NULL);
1004         if (!dir)
1005                 return -ENOMEM;
1006
1007         file = debugfs_create_file("list", 0444, dir, NULL,
1008                                 &debugfs_kprobes_operations);
1009         if (!file) {
1010                 debugfs_remove(dir);
1011                 return -ENOMEM;
1012         }
1013
1014         file = debugfs_create_file("enabled", 0600, dir,
1015                                         &value, &fops_kp);
1016         if (!file) {
1017                 debugfs_remove(dir);
1018                 return -ENOMEM;
1019         }
1020
1021         return 0;
1022 }
1023
1024 late_initcall(debugfs_kprobe_init);
1025 #endif /* CONFIG_DEBUG_FS */
1026
1027 module_init(init_kprobes);
1028
1029 EXPORT_SYMBOL_GPL(register_kprobe);
1030 EXPORT_SYMBOL_GPL(unregister_kprobe);
1031 EXPORT_SYMBOL_GPL(register_jprobe);
1032 EXPORT_SYMBOL_GPL(unregister_jprobe);
1033 #ifdef CONFIG_KPROBES
1034 EXPORT_SYMBOL_GPL(jprobe_return);
1035 #endif
1036
1037 #ifdef CONFIG_KPROBES
1038 EXPORT_SYMBOL_GPL(register_kretprobe);
1039 EXPORT_SYMBOL_GPL(unregister_kretprobe);
1040 #endif