perf: Remove the nmi parameter from the swevent and overflow interface
[linux-2.6.git] / arch / sh / mm / fault_32.c
index 33b43d2..7bebd04 100644 (file)
@@ -2,7 +2,7 @@
  * Page fault handler for SH with an MMU.
  *
  *  Copyright (C) 1999  Niibe Yutaka
- *  Copyright (C) 2003 - 2007  Paul Mundt
+ *  Copyright (C) 2003 - 2009  Paul Mundt
  *
  *  Based on linux/arch/i386/mm/fault.c:
  *   Copyright (C) 1995  Linus Torvalds
 #include <linux/mm.h>
 #include <linux/hardirq.h>
 #include <linux/kprobes.h>
+#include <linux/perf_event.h>
+#include <asm/io_trapped.h>
 #include <asm/system.h>
 #include <asm/mmu_context.h>
 #include <asm/tlbflush.h>
-#include <asm/kgdb.h>
+
+static inline int notify_page_fault(struct pt_regs *regs, int trap)
+{
+       int ret = 0;
+
+       if (kprobes_built_in() && !user_mode(regs)) {
+               preempt_disable();
+               if (kprobe_running() && kprobe_fault_handler(regs, trap))
+                       ret = 1;
+               preempt_enable();
+       }
+
+       return ret;
+}
+
+static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
+{
+       unsigned index = pgd_index(address);
+       pgd_t *pgd_k;
+       pud_t *pud, *pud_k;
+       pmd_t *pmd, *pmd_k;
+
+       pgd += index;
+       pgd_k = init_mm.pgd + index;
+
+       if (!pgd_present(*pgd_k))
+               return NULL;
+
+       pud = pud_offset(pgd, address);
+       pud_k = pud_offset(pgd_k, address);
+       if (!pud_present(*pud_k))
+               return NULL;
+
+       if (!pud_present(*pud))
+           set_pud(pud, *pud_k);
+
+       pmd = pmd_offset(pud, address);
+       pmd_k = pmd_offset(pud_k, address);
+       if (!pmd_present(*pmd_k))
+               return NULL;
+
+       if (!pmd_present(*pmd))
+               set_pmd(pmd, *pmd_k);
+       else {
+               /*
+                * The page tables are fully synchronised so there must
+                * be another reason for the fault. Return NULL here to
+                * signal that we have not taken care of the fault.
+                */
+               BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
+               return NULL;
+       }
+
+       return pmd_k;
+}
+
+/*
+ * Handle a fault on the vmalloc or module mapping area
+ */
+static noinline int vmalloc_fault(unsigned long address)
+{
+       pgd_t *pgd_k;
+       pmd_t *pmd_k;
+       pte_t *pte_k;
+
+       /* Make sure we are in vmalloc/module/P3 area: */
+       if (!(address >= VMALLOC_START && address < P3_ADDR_MAX))
+               return -1;
+
+       /*
+        * Synchronize this task's top level page-table
+        * with the 'reference' page table.
+        *
+        * Do _not_ use "current" here. We might be inside
+        * an interrupt in the middle of a task switch..
+        */
+       pgd_k = get_TTB();
+       pmd_k = vmalloc_sync_one(pgd_k, address);
+       if (!pmd_k)
+               return -1;
+
+       pte_k = pte_offset_kernel(pmd_k, address);
+       if (!pte_present(*pte_k))
+               return -1;
+
+       return 0;
+}
+
+static int fault_in_kernel_space(unsigned long address)
+{
+       return address >= TASK_SIZE;
+}
 
 /*
  * This routine handles page faults.  It determines the address,
@@ -29,6 +122,7 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
                                        unsigned long writeaccess,
                                        unsigned long address)
 {
+       unsigned long vec;
        struct task_struct *tsk;
        struct mm_struct *mm;
        struct vm_area_struct * vma;
@@ -36,60 +130,41 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
        int fault;
        siginfo_t info;
 
-       trace_hardirqs_on();
-       local_irq_enable();
-
-#ifdef CONFIG_SH_KGDB
-       if (kgdb_nofault && kgdb_bus_err_hook)
-               kgdb_bus_err_hook();
-#endif
-
        tsk = current;
        mm = tsk->mm;
        si_code = SEGV_MAPERR;
+       vec = lookup_exception_vector();
 
-       if (unlikely(address >= TASK_SIZE)) {
-               /*
-                * Synchronize this task's top level page-table
-                * with the 'reference' page table.
-                *
-                * Do _not_ use "tsk" here. We might be inside
-                * an interrupt in the middle of a task switch..
-                */
-               int offset = pgd_index(address);
-               pgd_t *pgd, *pgd_k;
-               pud_t *pud, *pud_k;
-               pmd_t *pmd, *pmd_k;
-
-               pgd = get_TTB() + offset;
-               pgd_k = swapper_pg_dir + offset;
-
-               /* This will never happen with the folded page table. */
-               if (!pgd_present(*pgd)) {
-                       if (!pgd_present(*pgd_k))
-                               goto bad_area_nosemaphore;
-                       set_pgd(pgd, *pgd_k);
+       /*
+        * We fault-in kernel-space virtual memory on-demand. The
+        * 'reference' page table is init_mm.pgd.
+        *
+        * NOTE! We MUST NOT take any locks for this case. We may
+        * be in an interrupt or a critical region, and should
+        * only copy the information from the master page table,
+        * nothing more.
+        */
+       if (unlikely(fault_in_kernel_space(address))) {
+               if (vmalloc_fault(address) >= 0)
+                       return;
+               if (notify_page_fault(regs, vec))
                        return;
-               }
-
-               pud = pud_offset(pgd, address);
-               pud_k = pud_offset(pgd_k, address);
-               if (pud_present(*pud) || !pud_present(*pud_k))
-                       goto bad_area_nosemaphore;
-               set_pud(pud, *pud_k);
 
-               pmd = pmd_offset(pud, address);
-               pmd_k = pmd_offset(pud_k, address);
-               if (pmd_present(*pmd) || !pmd_present(*pmd_k))
-                       goto bad_area_nosemaphore;
-               set_pmd(pmd, *pmd_k);
+               goto bad_area_nosemaphore;
+       }
 
+       if (unlikely(notify_page_fault(regs, vec)))
                return;
-       }
+
+       /* Only enable interrupts if they were on before the fault */
+       if ((regs->sr & SR_IMASK) != SR_IMASK)
+               local_irq_enable();
+
+       perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address);
 
        /*
-        * If we're in an interrupt or have no user
-        * context, we must not take the fault..
+        * If we're in an interrupt, have no user context or are running
+        * in an atomic region then we must not take the fault:
         */
        if (in_atomic() || !mm)
                goto no_context;
@@ -105,10 +180,11 @@ asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
                goto bad_area;
        if (expand_stack(vma, address))
                goto bad_area;
-/*
- * Ok, we have a good vm_area for this memory access, so
- * we can handle it..
- */
+
+       /*
+        * Ok, we have a good vm_area for this memory access, so
+        * we can handle it..
+        */
 good_area:
        si_code = SEGV_ACCERR;
        if (writeaccess) {
@@ -124,8 +200,7 @@ good_area:
         * make sure we exit gracefully rather than endlessly redo
         * the fault.
         */
-survive:
-       fault = handle_mm_fault(mm, vma, address, writeaccess);
+       fault = handle_mm_fault(mm, vma, address, writeaccess ? FAULT_FLAG_WRITE : 0);
        if (unlikely(fault & VM_FAULT_ERROR)) {
                if (fault & VM_FAULT_OOM)
                        goto out_of_memory;
@@ -133,18 +208,23 @@ survive:
                        goto do_sigbus;
                BUG();
        }
-       if (fault & VM_FAULT_MAJOR)
+       if (fault & VM_FAULT_MAJOR) {
                tsk->maj_flt++;
-       else
+               perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1,
+                                    regs, address);
+       } else {
                tsk->min_flt++;
+               perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1,
+                                    regs, address);
+       }
 
        up_read(&mm->mmap_sem);
        return;
 
-/*
- * Something tried to access memory that isn't in our memory map..
- * Fix it, but check if it's kernel or user first..
- */
+       /*
+        * Something tried to access memory that isn't in our memory map..
+        * Fix it, but check if it's kernel or user first..
+        */
 bad_area:
        up_read(&mm->mmap_sem);
 
@@ -163,6 +243,8 @@ no_context:
        if (fixup_exception(regs))
                return;
 
+       if (handle_trapped_io(regs, address))
+               return;
 /*
  * Oops. The kernel tried to access some bad page. We'll have to
  * terminate things with extreme prejudice.
@@ -207,15 +289,10 @@ no_context:
  */
 out_of_memory:
        up_read(&mm->mmap_sem);
-       if (is_global_init(current)) {
-               yield();
-               down_read(&mm->mmap_sem);
-               goto survive;
-       }
-       printk("VM: killing process %s\n", tsk->comm);
-       if (user_mode(regs))
-               do_group_exit(SIGKILL);
-       goto no_context;
+       if (!user_mode(regs))
+               goto no_context;
+       pagefault_out_of_memory();
+       return;
 
 do_sigbus:
        up_read(&mm->mmap_sem);
@@ -235,23 +312,12 @@ do_sigbus:
                goto no_context;
 }
 
-#ifdef CONFIG_SH_STORE_QUEUES
-/*
- * This is a special case for the SH-4 store queues, as pages for this
- * space still need to be faulted in before it's possible to flush the
- * store queue cache for writeout to the remapped region.
- */
-#define P3_ADDR_MAX            (P4SEG_STORE_QUE + 0x04000000)
-#else
-#define P3_ADDR_MAX            P4SEG
-#endif
-
 /*
  * Called with interrupts disabled.
  */
-asmlinkage int __kprobes __do_page_fault(struct pt_regs *regs,
-                                        unsigned long writeaccess,
-                                        unsigned long address)
+asmlinkage int __kprobes
+handle_tlbmiss(struct pt_regs *regs, unsigned long writeaccess,
+              unsigned long address)
 {
        pgd_t *pgd;
        pud_t *pud;
@@ -259,11 +325,6 @@ asmlinkage int __kprobes __do_page_fault(struct pt_regs *regs,
        pte_t *pte;
        pte_t entry;
 
-#ifdef CONFIG_SH_KGDB
-       if (kgdb_nofault && kgdb_bus_err_hook)
-               kgdb_bus_err_hook();
-#endif
-
        /*
         * We don't take page faults for P1, P2, and parts of P4, these
         * are always mapped, whether it be due to legacy behaviour in
@@ -284,7 +345,6 @@ asmlinkage int __kprobes __do_page_fault(struct pt_regs *regs,
        pmd = pmd_offset(pud, address);
        if (pmd_none_or_clear_bad(pmd))
                return 1;
-
        pte = pte_offset_kernel(pmd, address);
        entry = *pte;
        if (unlikely(pte_none(entry) || pte_not_present(entry)))
@@ -297,7 +357,18 @@ asmlinkage int __kprobes __do_page_fault(struct pt_regs *regs,
        entry = pte_mkyoung(entry);
 
        set_pte(pte, entry);
-       update_mmu_cache(NULL, address, entry);
+
+#if defined(CONFIG_CPU_SH4) && !defined(CONFIG_SMP)
+       /*
+        * SH-4 does not set MMUCR.RC to the corresponding TLB entry in
+        * the case of an initial page write exception, so we need to
+        * flush it in order to avoid potential TLB entry duplication.
+        */
+       if (writeaccess == 2)
+               local_flush_tlb_one(get_asid(), address & PAGE_MASK);
+#endif
+
+       update_mmu_cache(NULL, address, pte);
 
        return 0;
 }