x86: Fix alternatives and kprobes to remap write-protected kernel text

[linux-2.6.git] / arch / i386 / kernel / alternative.c

diff --git a/arch/i386/kernel/alternative.c b/arch/i386/kernel/alternative.c
index 915b6c4d9baf52608b0b4ef28e7460be7e6daaca..206ea2ca63cc5beb221c238ff81172244de262b9 100644 (file)
--- a/arch/i386/kernel/alternative.c
+++ b/arch/i386/kernel/alternative.c
@@ -2,12 +2,15 @@
 #include <linux/sched.h>
 #include <linux/spinlock.h>
 #include <linux/list.h>
+#include <linux/kprobes.h>
+#include <linux/mm.h>
+#include <linux/vmalloc.h>
 #include <asm/alternative.h>
 #include <asm/sections.h>
+#include <asm/pgtable.h>
 
-static int noreplace_smp     = 0;
-static int smp_alt_once      = 0;
-static int debug_alternative = 0;
+#ifdef CONFIG_HOTPLUG_CPU
+static int smp_alt_once;
 
 static int __init bootonly(char *str)
 {
@@ -15,6 +18,11 @@ static int __init bootonly(char *str)
        return 1;
 }
 __setup("smp-alt-boot", bootonly);
+#else
+#define smp_alt_once 1
+#endif
+
+static int debug_alternative;
 
 static int __init debug_alt(char *str)
 {
@@ -23,6 +31,8 @@ static int __init debug_alt(char *str)
 }
 __setup("debug-alternative", debug_alt);
 
+static int noreplace_smp;
+
 static int __init setup_noreplace_smp(char *str)
 {
        noreplace_smp = 1;
@@ -30,6 +40,16 @@ static int __init setup_noreplace_smp(char *str)
 }
 __setup("noreplace-smp", setup_noreplace_smp);
 
+#ifdef CONFIG_PARAVIRT
+static int noreplace_paravirt = 0;
+
+static int __init setup_noreplace_paravirt(char *str)
+{
+       noreplace_paravirt = 1;
+       return 1;
+}
+__setup("noreplace-paravirt", setup_noreplace_paravirt);
+#endif
 
 #define DPRINTK(fmt, args...) if (debug_alternative) \
        printk(KERN_DEBUG fmt, args)
@@ -134,7 +154,7 @@ static void nop_out(void *insns, unsigned int len)
                unsigned int noplen = len;
                if (noplen > ASM_NOP_MAX)
                        noplen = ASM_NOP_MAX;
-               memcpy(insns, noptable[noplen], noplen);
+               text_poke(insns, noptable[noplen], noplen);
                insns += noplen;
                len -= noplen;
        }
@@ -186,7 +206,7 @@ static void alternatives_smp_lock(u8 **start, u8 **end, u8 *text, u8 *text_end)
                        continue;
                if (*ptr > text_end)
                        continue;
-               **ptr = 0xf0; /* lock prefix */
+               text_poke(*ptr, ((unsigned char []){0xf0}), 1); /* add lock prefix */
        };
 }
 
@@ -325,9 +345,13 @@ void alternatives_smp_switch(int smp)
 #endif
 
 #ifdef CONFIG_PARAVIRT
-void apply_paravirt(struct paravirt_patch *start, struct paravirt_patch *end)
+void apply_paravirt(struct paravirt_patch_site *start,
+                   struct paravirt_patch_site *end)
 {
-       struct paravirt_patch *p;
+       struct paravirt_patch_site *p;
+
+       if (noreplace_paravirt)
+               return;
 
        for (p = start; p < end; p++) {
                unsigned int used;
@@ -335,14 +359,13 @@ void apply_paravirt(struct paravirt_patch *start, struct paravirt_patch *end)
                used = paravirt_ops.patch(p->instrtype, p->clobbers, p->instr,
                                          p->len);
 
+               BUG_ON(used > p->len);
+
                /* Pad the rest with nops */
                nop_out(p->instr + used, p->len - used);
        }
-
-       /* Sync to be conservative, in case we patched following instructions */
-       sync_core();
 }
-extern struct paravirt_patch __start_parainstructions[],
+extern struct paravirt_patch_site __start_parainstructions[],
        __stop_parainstructions[];
 #endif /* CONFIG_PARAVIRT */
 
@@ -359,8 +382,6 @@ void __init alternative_instructions(void)
 #ifdef CONFIG_HOTPLUG_CPU
        if (num_possible_cpus() < 2)
                smp_alt_once = 1;
-#else
-       smp_alt_once = 1;
 #endif
 
 #ifdef CONFIG_SMP
@@ -373,8 +394,8 @@ void __init alternative_instructions(void)
                                                _text, _etext);
                }
                free_init_pages("SMP alternatives",
-                               __pa_symbol(&__smp_locks),
-                               __pa_symbol(&__smp_locks_end));
+                               (unsigned long)__smp_locks,
+                               (unsigned long)__smp_locks_end);
        } else {
                alternatives_smp_module_add(NULL, "core kernel",
                                            __smp_locks, __smp_locks_end,
@@ -382,6 +403,34 @@ void __init alternative_instructions(void)
                alternatives_smp_switch(0);
        }
 #endif
-       apply_paravirt(__start_parainstructions, __stop_parainstructions);
+       apply_paravirt(__parainstructions, __parainstructions_end);
        local_irq_restore(flags);
 }
+
+/*
+ * Warning:
+ * When you use this code to patch more than one byte of an instruction
+ * you need to make sure that other CPUs cannot execute this code in parallel.
+ * Also no thread must be currently preempted in the middle of these instructions.
+ * And on the local CPU you need to be protected again NMI or MCE handlers
+ * seeing an inconsistent instruction while you patch.
+ */
+void __kprobes text_poke(void *oaddr, unsigned char *opcode, int len)
+{
+        u8 *addr = oaddr;
+       if (!pte_write(*lookup_address((unsigned long)addr))) {
+               struct page *p[2] = { virt_to_page(addr), virt_to_page(addr+PAGE_SIZE) };
+               addr = vmap(p, 2, VM_MAP, PAGE_KERNEL);
+               if (!addr)
+                       return;
+               addr += ((unsigned long)oaddr) % PAGE_SIZE;
+       }
+       memcpy(addr, opcode, len);
+       sync_core();
+       /* Not strictly needed, but can speed CPU recovery up. Ignore cross cacheline
+          case. */
+       if (cpu_has_clflush)
+               asm("clflush (%0) " :: "r" (oaddr) : "memory");
+       if (addr != oaddr)
+               vunmap(addr);
+}