arm: errata: Workaround for Cortex-A15 erratum 798181 (TLBI/DSB operations)
Catalin Marinas [Tue, 19 Feb 2013 16:34:29 +0000 (16:34 +0000)]
On Cortex-A15 (r0p0..r3p2) the TLBI/DSB are not adequately shooting down
all use of the old entries. This patch implements the erratum workaround
which consists of:

1. Dummy TLBIMVAIS and DSB on the CPU doing the TLBI operation.
2. Send IPI to the CPUs that are running the same mm (and ASID) as the
   one being invalidated (or all the online CPUs for global pages).
3. CPU receiving the IPI executes a DMB and CLREX (part of the exception
   return code already).

The switch_mm() code includes a DMB operation since the IPI is only sent
to CPUs running the same ASID.

Change-Id: Ideb7f479910f7d4bf25182c84eb5e71691c42a93
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Bo Yan <byan@nvidia.com>
Reviewed-on: http://git-master/r/209830
Reviewed-by: Mrutyunjay Sawant <msawant@nvidia.com>
Tested-by: Mrutyunjay Sawant <msawant@nvidia.com>

arch/arm/Kconfig
arch/arm/include/asm/highmem.h
arch/arm/include/asm/mmu_context.h
arch/arm/include/asm/tlbflush.h
arch/arm/kernel/smp_tlb.c
arch/arm/mm/context.c

index 6472d15..9bc8d84 100644 (file)
@@ -1451,6 +1451,16 @@ config PL310_ERRATA_769419
          on systems with an outer cache, the store buffer is drained
          explicitly.
 
+config ARM_ERRATA_798181
+       bool "ARM errata: TLBI/DSB failure on Cortex-A15"
+       depends on CPU_V7 && SMP
+       help
+         On Cortex-A15 (r0p0..r3p2) the TLBI*IS/DSB operations are not
+         adequately shooting down all use of the old entries. This
+         option enables the Linux kernel workaround for this erratum
+         which sends an IPI to the CPUs that are running the same mm
+         (and ASID) as the one being invalidated.
+
 config ARM_ERRATA_775420
        bool "ARM errata: A data cache maintenance operation which aborts, might lead to deadlock"
        depends on CPU_V7
index 8c5e828..91b99ab 100644 (file)
@@ -41,6 +41,13 @@ extern void kunmap_high(struct page *page);
 #endif
 #endif
 
+/*
+ * Needed to be able to broadcast the TLB invalidation for kmap.
+ */
+#ifdef CONFIG_ARM_ERRATA_798181
+#undef ARCH_NEEDS_KMAP_HIGH_GET
+#endif
+
 #ifdef ARCH_NEEDS_KMAP_HIGH_GET
 extern void *kmap_high_get(struct page *page);
 #else
index a0b3cac..20d1dfe 100644 (file)
@@ -125,6 +125,13 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next,
 #endif
                check_context(next);
                cpu_switch_mm(next->pgd, next);
+#ifdef CONFIG_ARM_ERRATA_798181
+               /*
+                * DMB required since we only send IPI to the other CPUs if
+                * they run the same mm as the one being invalidated.
+                */
+               dmb();
+#endif
                if (cache_is_vivt())
                        cpumask_clear_cpu(cpu, mm_cpumask(prev));
        }
index 85fe61e..9b4541e 100644 (file)
@@ -447,6 +447,21 @@ static inline void local_flush_tlb_kernel_page(unsigned long kaddr)
        }
 }
 
+#ifdef CONFIG_ARM_ERRATA_798181
+static inline void dummy_flush_tlb_a15_erratum(void)
+{
+       /*
+        * Dummy TLBIMVAIS. Using the unmapped address 0 and ASID 0.
+        */
+       asm("mcr p15, 0, %0, c8, c3, 1" : : "r" (0));
+       dsb();
+}
+#else
+static inline void dummy_flush_tlb_a15_erratum(void)
+{
+}
+#endif
+
 /*
  *     flush_pmd_entry
  *
index 02c5d2c..c92997e 100644 (file)
@@ -12,6 +12,7 @@
 
 #include <asm/smp_plat.h>
 #include <asm/tlbflush.h>
+#include <asm/mmu_context.h>
 
 /**********************************************************************/
 
@@ -64,12 +65,71 @@ static inline void ipi_flush_tlb_kernel_range(void *arg)
        local_flush_tlb_kernel_range(ta->ta_start, ta->ta_end);
 }
 
+#ifdef CONFIG_ARM_ERRATA_798181
+static int erratum_a15_798181(void)
+{
+       unsigned int midr = read_cpuid_id();
+
+       /* Cortex-A15 r0p0..r3p2 affected */
+       if ((midr & 0xff0ffff0) != 0x410fc0f0 || midr > 0x413fc0f2)
+               return 0;
+       return 1;
+}
+#else
+static int erratum_a15_798181(void)
+{
+       return 0;
+}
+#endif
+
+static void ipi_flush_tlb_a15_erratum(void *arg)
+{
+       dmb();
+}
+
+static void broadcast_tlb_a15_erratum(void)
+{
+       if (!erratum_a15_798181())
+               return;
+
+       dummy_flush_tlb_a15_erratum();
+       smp_call_function_many(cpu_online_mask, ipi_flush_tlb_a15_erratum,
+                              NULL, 1);
+}
+
+static void broadcast_tlb_mm_a15_erratum(struct mm_struct *mm)
+{
+       int cpu;
+       cpumask_t mask = { CPU_BITS_NONE };
+
+       if (!erratum_a15_798181())
+               return;
+
+       dummy_flush_tlb_a15_erratum();
+       for_each_online_cpu(cpu) {
+               if (cpu == smp_processor_id())
+                       continue;
+               /*
+                * We only need to send an IPI if the other CPUs are running
+                * the same mm (and ASID) as the one being invalidated. There
+                * is no need for locking around the current_mm check since
+                * the switch_mm() function has a dmb() for this erratum in
+                * case a context switch happens on another CPU after the
+                * condition below.
+                */
+               if (mm == per_cpu(current_mm, cpu))
+                       cpumask_set_cpu(cpu, &mask);
+       }
+       smp_call_function_many(&mask, ipi_flush_tlb_a15_erratum, NULL, 1);
+}
+
 void flush_tlb_all(void)
 {
        if (tlb_ops_need_broadcast())
                on_each_cpu(ipi_flush_tlb_all, NULL, 1);
        else
                local_flush_tlb_all();
+       broadcast_tlb_a15_erratum();
 }
 
 void flush_tlb_mm(struct mm_struct *mm)
@@ -78,6 +138,7 @@ void flush_tlb_mm(struct mm_struct *mm)
                on_each_cpu_mask(mm_cpumask(mm), ipi_flush_tlb_mm, mm, 1);
        else
                local_flush_tlb_mm(mm);
+       broadcast_tlb_mm_a15_erratum(mm);
 }
 
 void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
@@ -90,6 +151,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr)
                                        &ta, 1);
        } else
                local_flush_tlb_page(vma, uaddr);
+       broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 }
 
 void flush_tlb_kernel_page(unsigned long kaddr)
@@ -100,6 +162,7 @@ void flush_tlb_kernel_page(unsigned long kaddr)
                on_each_cpu(ipi_flush_tlb_kernel_page, &ta, 1);
        } else
                local_flush_tlb_kernel_page(kaddr);
+       broadcast_tlb_a15_erratum();
 }
 
 void flush_tlb_range(struct vm_area_struct *vma,
@@ -114,6 +177,7 @@ void flush_tlb_range(struct vm_area_struct *vma,
                                        &ta, 1);
        } else
                local_flush_tlb_range(vma, start, end);
+       broadcast_tlb_mm_a15_erratum(vma->vm_mm);
 }
 
 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -125,5 +189,6 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
                on_each_cpu(ipi_flush_tlb_kernel_range, &ta, 1);
        } else
                local_flush_tlb_kernel_range(start, end);
+       broadcast_tlb_a15_erratum();
 }
 
index aaa291f..11f3448 100644 (file)
@@ -65,6 +65,7 @@ static void flush_context(void)
 {
        cpu_set_reserved_ttbr0();
        local_flush_tlb_all();
+       dummy_flush_tlb_a15_erratum();
        if (icache_is_vivt_asid_tagged()) {
                __flush_icache_all();
                dsb();