[PATCH] slab: remove kmem_cache_t
[linux-2.6.git] / kernel / kprobes.c
index cfef426..610c837 100644 (file)
@@ -37,6 +37,7 @@
 #include <linux/slab.h>
 #include <linux/module.h>
 #include <linux/moduleloader.h>
+#include <linux/kallsyms.h>
 #include <asm-generic/sections.h>
 #include <asm/cacheflush.h>
 #include <asm/errno.h>
 #define KPROBE_HASH_BITS 6
 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
 
+
+/*
+ * Some oddball architectures like 64bit powerpc have function descriptors
+ * so this must be overridable.
+ */
+#ifndef kprobe_lookup_name
+#define kprobe_lookup_name(name, addr) \
+       addr = ((kprobe_opcode_t *)(kallsyms_lookup_name(name)))
+#endif
+
 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
 static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE];
+static atomic_t kprobe_count;
 
-static DEFINE_SPINLOCK(kprobe_lock);   /* Protects kprobe_table */
+DEFINE_MUTEX(kprobe_mutex);            /* Protects kprobe_table */
 DEFINE_SPINLOCK(kretprobe_lock);       /* Protects kretprobe_inst_table */
 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL;
 
+static struct notifier_block kprobe_page_fault_nb = {
+       .notifier_call = kprobe_exceptions_notify,
+       .priority = 0x7fffffff /* we need to notified first */
+};
+
+#ifdef __ARCH_WANT_KPROBES_INSN_SLOT
 /*
  * kprobe->ainsn.insn points to the copy of the instruction to be
  * single-stepped. x86_64, POWER4 and above have no-exec support and
@@ -151,6 +169,7 @@ void __kprobes free_insn_slot(kprobe_opcode_t *slot)
                }
        }
 }
+#endif
 
 /* We have preemption disabled.. so it is safe to use __ versions */
 static inline void set_kprobe_instance(struct kprobe *kp)
@@ -165,9 +184,9 @@ static inline void reset_kprobe_instance(void)
 
 /*
  * This routine is called either:
- *     - under the kprobe_lock spinlock - during kprobe_[un]register()
+ *     - under the kprobe_mutex - during kprobe_[un]register()
  *                             OR
- *     - under an rcu_read_lock() - from arch/xxx/kernel/kprobes.c
+ *     - with preemption disabled - from arch/xxx/kernel/kprobes.c
  */
 struct kprobe __kprobes *get_kprobe(void *addr)
 {
@@ -246,6 +265,19 @@ static int __kprobes aggr_break_handler(struct kprobe *p, struct pt_regs *regs)
        return ret;
 }
 
+/* Walks the list and increments nmissed count for multiprobe case */
+void __kprobes kprobes_inc_nmissed_count(struct kprobe *p)
+{
+       struct kprobe *kp;
+       if (p->pre_handler != aggr_pre_handler) {
+               p->nmissed++;
+       } else {
+               list_for_each_entry_rcu(kp, &p->list, list)
+                       kp->nmissed++;
+       }
+       return;
+}
+
 /* Called with kretprobe_lock held */
 struct kretprobe_instance __kprobes *get_free_rp_inst(struct kretprobe *rp)
 {
@@ -287,7 +319,8 @@ void __kprobes add_rp_inst(struct kretprobe_instance *ri)
 }
 
 /* Called with kretprobe_lock held */
-void __kprobes recycle_rp_inst(struct kretprobe_instance *ri)
+void __kprobes recycle_rp_inst(struct kretprobe_instance *ri,
+                               struct hlist_head *head)
 {
        /* remove rp inst off the rprobe_inst_table */
        hlist_del(&ri->hlist);
@@ -299,7 +332,7 @@ void __kprobes recycle_rp_inst(struct kretprobe_instance *ri)
                hlist_add_head(&ri->uflist, &ri->rp->free_instances);
        } else
                /* Unregistering */
-               kfree(ri);
+               hlist_add_head(&ri->hlist, head);
 }
 
 struct hlist_head __kprobes *kretprobe_inst_table_head(struct task_struct *tsk)
@@ -308,42 +341,31 @@ struct hlist_head __kprobes *kretprobe_inst_table_head(struct task_struct *tsk)
 }
 
 /*
- * This function is called from exit_thread or flush_thread when task tk's
- * stack is being recycled so that we can recycle any function-return probe
- * instances associated with this task. These left over instances represent
- * probed functions that have been called but will never return.
+ * This function is called from finish_task_switch when task tk becomes dead,
+ * so that we can recycle any function-return probe instances associated
+ * with this task. These left over instances represent probed functions
+ * that have been called but will never return.
  */
 void __kprobes kprobe_flush_task(struct task_struct *tk)
 {
-        struct kretprobe_instance *ri;
-        struct hlist_head *head;
+       struct kretprobe_instance *ri;
+       struct hlist_head *head, empty_rp;
        struct hlist_node *node, *tmp;
        unsigned long flags = 0;
 
+       INIT_HLIST_HEAD(&empty_rp);
        spin_lock_irqsave(&kretprobe_lock, flags);
-        head = kretprobe_inst_table_head(current);
-        hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
-                if (ri->task == tk)
-                        recycle_rp_inst(ri);
-        }
+       head = kretprobe_inst_table_head(tk);
+       hlist_for_each_entry_safe(ri, node, tmp, head, hlist) {
+               if (ri->task == tk)
+                       recycle_rp_inst(ri, &empty_rp);
+       }
        spin_unlock_irqrestore(&kretprobe_lock, flags);
-}
 
-/*
- * This kprobe pre_handler is registered with every kretprobe. When probe
- * hits it will set up the return probe.
- */
-static int __kprobes pre_handler_kretprobe(struct kprobe *p,
-                                          struct pt_regs *regs)
-{
-       struct kretprobe *rp = container_of(p, struct kretprobe, kp);
-       unsigned long flags = 0;
-
-       /*TODO: consider to only swap the RA after the last pre_handler fired */
-       spin_lock_irqsave(&kretprobe_lock, flags);
-       arch_prepare_kretprobe(rp, regs);
-       spin_unlock_irqrestore(&kretprobe_lock, flags);
-       return 0;
+       hlist_for_each_entry_safe(ri, node, tmp, &empty_rp, hlist) {
+               hlist_del(&ri->hlist);
+               kfree(ri);
+       }
 }
 
 static inline void free_rp_inst(struct kretprobe *rp)
@@ -370,16 +392,15 @@ static inline void copy_kprobe(struct kprobe *old_p, struct kprobe *p)
 */
 static int __kprobes add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
 {
-        struct kprobe *kp;
-
        if (p->break_handler) {
-               list_for_each_entry_rcu(kp, &old_p->list, list) {
-                       if (kp->break_handler)
-                               return -EEXIST;
-               }
+               if (old_p->break_handler)
+                       return -EEXIST;
                list_add_tail_rcu(&p->list, &old_p->list);
+               old_p->break_handler = aggr_break_handler;
        } else
                list_add_rcu(&p->list, &old_p->list);
+       if (p->post_handler && !old_p->post_handler)
+               old_p->post_handler = aggr_post_handler;
        return 0;
 }
 
@@ -390,25 +411,24 @@ static int __kprobes add_new_kprobe(struct kprobe *old_p, struct kprobe *p)
 static inline void add_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
 {
        copy_kprobe(p, ap);
+       flush_insn_slot(ap);
        ap->addr = p->addr;
        ap->pre_handler = aggr_pre_handler;
-       ap->post_handler = aggr_post_handler;
        ap->fault_handler = aggr_fault_handler;
-       ap->break_handler = aggr_break_handler;
+       if (p->post_handler)
+               ap->post_handler = aggr_post_handler;
+       if (p->break_handler)
+               ap->break_handler = aggr_break_handler;
 
        INIT_LIST_HEAD(&ap->list);
        list_add_rcu(&p->list, &ap->list);
 
-       INIT_HLIST_NODE(&ap->hlist);
-       hlist_del_rcu(&p->hlist);
-       hlist_add_head_rcu(&ap->hlist,
-               &kprobe_table[hash_ptr(ap->addr, KPROBE_HASH_BITS)]);
+       hlist_replace_rcu(&p->hlist, &ap->hlist);
 }
 
 /*
  * This is the second or subsequent kprobe at the address - handle
  * the intricacies
- * TODO: Move kcalloc outside the spin_lock
  */
 static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
                                          struct kprobe *p)
@@ -420,7 +440,7 @@ static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
                copy_kprobe(old_p, p);
                ret = add_new_kprobe(old_p, p);
        } else {
-               ap = kcalloc(1, sizeof(struct kprobe), GFP_ATOMIC);
+               ap = kzalloc(sizeof(struct kprobe), GFP_KERNEL);
                if (!ap)
                        return -ENOMEM;
                add_aggr_kprobe(ap, old_p);
@@ -430,25 +450,6 @@ static int __kprobes register_aggr_kprobe(struct kprobe *old_p,
        return ret;
 }
 
-/* kprobe removal house-keeping routines */
-static inline void cleanup_kprobe(struct kprobe *p, unsigned long flags)
-{
-       arch_disarm_kprobe(p);
-       hlist_del_rcu(&p->hlist);
-       spin_unlock_irqrestore(&kprobe_lock, flags);
-       arch_remove_kprobe(p);
-}
-
-static inline void cleanup_aggr_kprobe(struct kprobe *old_p,
-               struct kprobe *p, unsigned long flags)
-{
-       list_del_rcu(&p->list);
-       if (list_empty(&old_p->list))
-               cleanup_kprobe(old_p, flags);
-       else
-               spin_unlock_irqrestore(&kprobe_lock, flags);
-}
-
 static int __kprobes in_kprobes_functions(unsigned long addr)
 {
        if (addr >= (unsigned long)__kprobes_text_start
@@ -457,74 +458,173 @@ static int __kprobes in_kprobes_functions(unsigned long addr)
        return 0;
 }
 
-int __kprobes register_kprobe(struct kprobe *p)
+static int __kprobes __register_kprobe(struct kprobe *p,
+       unsigned long called_from)
 {
        int ret = 0;
-       unsigned long flags = 0;
        struct kprobe *old_p;
+       struct module *probed_mod;
 
-       if ((ret = in_kprobes_functions((unsigned long) p->addr)) != 0)
-               return ret;
-       if ((ret = arch_prepare_kprobe(p)) != 0)
-               goto rm_kprobe;
+       /*
+        * If we have a symbol_name argument look it up,
+        * and add it to the address.  That way the addr
+        * field can either be global or relative to a symbol.
+        */
+       if (p->symbol_name) {
+               if (p->addr)
+                       return -EINVAL;
+               kprobe_lookup_name(p->symbol_name, p->addr);
+       }
+
+       if (!p->addr)
+               return -EINVAL;
+       p->addr = (kprobe_opcode_t *)(((char *)p->addr)+ p->offset);
+
+       if ((!kernel_text_address((unsigned long) p->addr)) ||
+               in_kprobes_functions((unsigned long) p->addr))
+               return -EINVAL;
+
+       p->mod_refcounted = 0;
+       /* Check are we probing a module */
+       if ((probed_mod = module_text_address((unsigned long) p->addr))) {
+               struct module *calling_mod = module_text_address(called_from);
+               /* We must allow modules to probe themself and
+                * in this case avoid incrementing the module refcount,
+                * so as to allow unloading of self probing modules.
+                */
+               if (calling_mod && (calling_mod != probed_mod)) {
+                       if (unlikely(!try_module_get(probed_mod)))
+                               return -EINVAL;
+                       p->mod_refcounted = 1;
+               } else
+                       probed_mod = NULL;
+       }
 
        p->nmissed = 0;
-       spin_lock_irqsave(&kprobe_lock, flags);
+       mutex_lock(&kprobe_mutex);
        old_p = get_kprobe(p->addr);
        if (old_p) {
                ret = register_aggr_kprobe(old_p, p);
+               if (!ret)
+                       atomic_inc(&kprobe_count);
                goto out;
        }
 
-       arch_copy_kprobe(p);
+       if ((ret = arch_prepare_kprobe(p)) != 0)
+               goto out;
+
        INIT_HLIST_NODE(&p->hlist);
        hlist_add_head_rcu(&p->hlist,
                       &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
 
-       arch_arm_kprobe(p);
+       if (atomic_add_return(1, &kprobe_count) == \
+                               (ARCH_INACTIVE_KPROBE_COUNT + 1))
+               register_page_fault_notifier(&kprobe_page_fault_nb);
+
+       arch_arm_kprobe(p);
 
 out:
-       spin_unlock_irqrestore(&kprobe_lock, flags);
-rm_kprobe:
-       if (ret == -EEXIST)
-               arch_remove_kprobe(p);
+       mutex_unlock(&kprobe_mutex);
+
+       if (ret && probed_mod)
+               module_put(probed_mod);
        return ret;
 }
 
+int __kprobes register_kprobe(struct kprobe *p)
+{
+       return __register_kprobe(p,
+               (unsigned long)__builtin_return_address(0));
+}
+
 void __kprobes unregister_kprobe(struct kprobe *p)
 {
-       unsigned long flags;
-       struct kprobe *old_p;
+       struct module *mod;
+       struct kprobe *old_p, *list_p;
+       int cleanup_p;
 
-       spin_lock_irqsave(&kprobe_lock, flags);
+       mutex_lock(&kprobe_mutex);
        old_p = get_kprobe(p->addr);
-       if (old_p) {
-               /* cleanup_*_kprobe() does the spin_unlock_irqrestore */
-               if (old_p->pre_handler == aggr_pre_handler)
-                       cleanup_aggr_kprobe(old_p, p, flags);
-               else
-                       cleanup_kprobe(p, flags);
-
-               synchronize_sched();
-               if (old_p->pre_handler == aggr_pre_handler &&
-                               list_empty(&old_p->list))
+       if (unlikely(!old_p)) {
+               mutex_unlock(&kprobe_mutex);
+               return;
+       }
+       if (p != old_p) {
+               list_for_each_entry_rcu(list_p, &old_p->list, list)
+                       if (list_p == p)
+                       /* kprobe p is a valid probe */
+                               goto valid_p;
+               mutex_unlock(&kprobe_mutex);
+               return;
+       }
+valid_p:
+       if ((old_p == p) || ((old_p->pre_handler == aggr_pre_handler) &&
+               (p->list.next == &old_p->list) &&
+               (p->list.prev == &old_p->list))) {
+               /* Only probe on the hash list */
+               arch_disarm_kprobe(p);
+               hlist_del_rcu(&old_p->hlist);
+               cleanup_p = 1;
+       } else {
+               list_del_rcu(&p->list);
+               cleanup_p = 0;
+       }
+
+       mutex_unlock(&kprobe_mutex);
+
+       synchronize_sched();
+       if (p->mod_refcounted &&
+           (mod = module_text_address((unsigned long)p->addr)))
+               module_put(mod);
+
+       if (cleanup_p) {
+               if (p != old_p) {
+                       list_del_rcu(&p->list);
                        kfree(old_p);
-       } else
-               spin_unlock_irqrestore(&kprobe_lock, flags);
+               }
+               arch_remove_kprobe(p);
+       } else {
+               mutex_lock(&kprobe_mutex);
+               if (p->break_handler)
+                       old_p->break_handler = NULL;
+               if (p->post_handler){
+                       list_for_each_entry_rcu(list_p, &old_p->list, list){
+                               if (list_p->post_handler){
+                                       cleanup_p = 2;
+                                       break;
+                               }
+                       }
+                       if (cleanup_p == 0)
+                               old_p->post_handler = NULL;
+               }
+               mutex_unlock(&kprobe_mutex);
+       }
+
+       /* Call unregister_page_fault_notifier()
+        * if no probes are active
+        */
+       mutex_lock(&kprobe_mutex);
+       if (atomic_add_return(-1, &kprobe_count) == \
+                               ARCH_INACTIVE_KPROBE_COUNT)
+               unregister_page_fault_notifier(&kprobe_page_fault_nb);
+       mutex_unlock(&kprobe_mutex);
+       return;
 }
 
 static struct notifier_block kprobe_exceptions_nb = {
        .notifier_call = kprobe_exceptions_notify,
-       .priority = 0x7fffffff /* we need to notified first */
+       .priority = 0x7fffffff /* we need to be notified first */
 };
 
+
 int __kprobes register_jprobe(struct jprobe *jp)
 {
        /* Todo: Verify probepoint is a function entry point */
        jp->kp.pre_handler = setjmp_pre_handler;
        jp->kp.break_handler = longjmp_break_handler;
 
-       return register_kprobe(&jp->kp);
+       return __register_kprobe(&jp->kp,
+               (unsigned long)__builtin_return_address(0));
 }
 
 void __kprobes unregister_jprobe(struct jprobe *jp)
@@ -534,6 +634,23 @@ void __kprobes unregister_jprobe(struct jprobe *jp)
 
 #ifdef ARCH_SUPPORTS_KRETPROBES
 
+/*
+ * This kprobe pre_handler is registered with every kretprobe. When probe
+ * hits it will set up the return probe.
+ */
+static int __kprobes pre_handler_kretprobe(struct kprobe *p,
+                                          struct pt_regs *regs)
+{
+       struct kretprobe *rp = container_of(p, struct kretprobe, kp);
+       unsigned long flags = 0;
+
+       /*TODO: consider to only swap the RA after the last pre_handler fired */
+       spin_lock_irqsave(&kretprobe_lock, flags);
+       arch_prepare_kretprobe(rp, regs);
+       spin_unlock_irqrestore(&kretprobe_lock, flags);
+       return 0;
+}
+
 int __kprobes register_kretprobe(struct kretprobe *rp)
 {
        int ret = 0;
@@ -541,6 +658,9 @@ int __kprobes register_kretprobe(struct kretprobe *rp)
        int i;
 
        rp->kp.pre_handler = pre_handler_kretprobe;
+       rp->kp.post_handler = NULL;
+       rp->kp.fault_handler = NULL;
+       rp->kp.break_handler = NULL;
 
        /* Pre-allocate memory for max kretprobe instances */
        if (rp->maxactive <= 0) {
@@ -564,7 +684,8 @@ int __kprobes register_kretprobe(struct kretprobe *rp)
 
        rp->nmissed = 0;
        /* Establish function entry probe point */
-       if ((ret = register_kprobe(&rp->kp)) != 0)
+       if ((ret = __register_kprobe(&rp->kp,
+               (unsigned long)__builtin_return_address(0))) != 0)
                free_rp_inst(rp);
        return ret;
 }
@@ -586,12 +707,12 @@ void __kprobes unregister_kretprobe(struct kretprobe *rp)
        unregister_kprobe(&rp->kp);
        /* No race here */
        spin_lock_irqsave(&kretprobe_lock, flags);
-       free_rp_inst(rp);
        while ((ri = get_used_rp_inst(rp)) != NULL) {
                ri->rp = NULL;
                hlist_del(&ri->uflist);
        }
        spin_unlock_irqrestore(&kretprobe_lock, flags);
+       free_rp_inst(rp);
 }
 
 static int __init init_kprobes(void)
@@ -604,6 +725,7 @@ static int __init init_kprobes(void)
                INIT_HLIST_HEAD(&kprobe_table[i]);
                INIT_HLIST_HEAD(&kretprobe_inst_table[i]);
        }
+       atomic_set(&kprobe_count, 0);
 
        err = arch_init_kprobes();
        if (!err)