Merge branch 'master' into for-next
Tejun Heo [Fri, 3 Jul 2009 22:13:18 +0000 (07:13 +0900)]
Pull linus#master to merge PER_CPU_DEF_ATTRIBUTES and alpha build fix
changes.  As alpha in percpu tree uses 'weak' attribute instead of
inline assembly, there's no need for __used attribute.

Conflicts:
arch/alpha/include/asm/percpu.h
arch/mn10300/kernel/vmlinux.lds.S
include/linux/percpu-defs.h

69 files changed:
Makefile
arch/alpha/include/asm/percpu.h
arch/alpha/include/asm/tlbflush.h
arch/alpha/kernel/vmlinux.lds.S
arch/arm/kernel/vmlinux.lds.S
arch/avr32/kernel/vmlinux.lds.S
arch/blackfin/kernel/vmlinux.lds.S
arch/blackfin/mm/sram-alloc.c
arch/cris/include/asm/mmu_context.h
arch/cris/kernel/vmlinux.lds.S
arch/cris/mm/fault.c
arch/frv/kernel/vmlinux.lds.S
arch/h8300/kernel/vmlinux.lds.S
arch/ia64/Kconfig
arch/ia64/kernel/smp.c
arch/ia64/kernel/vmlinux.lds.S
arch/ia64/sn/kernel/setup.c
arch/m32r/kernel/vmlinux.lds.S
arch/m68k/kernel/vmlinux-std.lds
arch/m68k/kernel/vmlinux-sun3.lds
arch/m68knommu/kernel/vmlinux.lds.S
arch/microblaze/kernel/vmlinux.lds.S
arch/mips/kernel/vmlinux.lds.S
arch/mn10300/kernel/vmlinux.lds.S
arch/parisc/kernel/vmlinux.lds.S
arch/powerpc/Kconfig
arch/powerpc/kernel/vmlinux.lds.S
arch/powerpc/mm/stab.c
arch/powerpc/platforms/ps3/smp.c
arch/s390/include/asm/percpu.h
arch/s390/kernel/vmlinux.lds.S
arch/sh/kernel/vmlinux.lds.S
arch/sparc/Kconfig
arch/sparc/kernel/vmlinux.lds.S
arch/um/kernel/dyn.lds.S
arch/um/kernel/uml.lds.S
arch/x86/Kconfig
arch/x86/kernel/cpu/cpu_debug.c
arch/x86/kernel/cpu/mcheck/mce.c
arch/x86/kernel/cpu/mcheck/mce_amd.c
arch/x86/kernel/cpu/perf_counter.c
arch/xtensa/kernel/vmlinux.lds.S
block/as-iosched.c
block/cfq-iosched.c
drivers/cpufreq/cpufreq_conservative.c
drivers/cpufreq/cpufreq_ondemand.c
drivers/xen/events.c
include/asm-generic/vmlinux.lds.h
include/linux/percpu-defs.h
include/linux/percpu.h
init/main.c
kernel/module.c
kernel/perf_counter.c
kernel/sched.c
kernel/trace/trace_events.c
lib/Kconfig.debug
mm/Makefile
mm/allocpercpu.c
mm/kmemleak-test.c
mm/page-writeback.c
mm/percpu.c
mm/quicklist.c
mm/slub.c
net/ipv4/syncookies.c
net/ipv6/syncookies.c
net/rds/ib_stats.c
net/rds/iw_stats.c
net/rds/page.c
scripts/module-common.lds [new file with mode: 0644]

index d1216fe..2f50338 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -327,7 +327,7 @@ CHECKFLAGS     := -D__linux__ -Dlinux -D__STDC__ -Dunix -D__unix__ \
 MODFLAGS       = -DMODULE
 CFLAGS_MODULE   = $(MODFLAGS)
 AFLAGS_MODULE   = $(MODFLAGS)
-LDFLAGS_MODULE  =
+LDFLAGS_MODULE  = -T $(srctree)/scripts/module-common.lds
 CFLAGS_KERNEL  =
 AFLAGS_KERNEL  =
 CFLAGS_GCOV    = -fprofile-arcs -ftest-coverage
index b663f1f..2c12378 100644 (file)
 #ifndef __ALPHA_PERCPU_H
 #define __ALPHA_PERCPU_H
 
-#include <linux/compiler.h>
-#include <linux/threads.h>
-#include <linux/percpu-defs.h>
-
-/*
- * Determine the real variable name from the name visible in the
- * kernel sources.
- */
-#define per_cpu_var(var) per_cpu__##var
-
-#ifdef CONFIG_SMP
-
-/*
- * per_cpu_offset() is the offset that has to be added to a
- * percpu variable to get to the instance for a certain processor.
- */
-extern unsigned long __per_cpu_offset[NR_CPUS];
-
-#define per_cpu_offset(x) (__per_cpu_offset[x])
-
-#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
-#ifdef CONFIG_DEBUG_PREEMPT
-#define my_cpu_offset per_cpu_offset(smp_processor_id())
-#else
-#define my_cpu_offset __my_cpu_offset
-#endif
-
-#ifndef MODULE
-#define SHIFT_PERCPU_PTR(var, offset) RELOC_HIDE(&per_cpu_var(var), (offset))
-#define PER_CPU_DEF_ATTRIBUTES
-#else
 /*
- * To calculate addresses of locally defined variables, GCC uses 32-bit
- * displacement from the GP. Which doesn't work for per cpu variables in
- * modules, as an offset to the kernel per cpu area is way above 4G.
+ * To calculate addresses of locally defined variables, GCC uses
+ * 32-bit displacement from the GP. Which doesn't work for per cpu
+ * variables in modules, as an offset to the kernel per cpu area is
+ * way above 4G.
  *
- * This forces allocation of a GOT entry for per cpu variable using
- * ldq instruction with a 'literal' relocation.
- */
-#define SHIFT_PERCPU_PTR(var, offset) ({               \
-       extern int simple_identifier_##var(void);       \
-       unsigned long __ptr, tmp_gp;                    \
-       asm (  "br      %1, 1f                        \n\
-       1:      ldgp    %1, 0(%1)                     \n\
-               ldq %0, per_cpu__" #var"(%1)\t!literal"         \
-               : "=&r"(__ptr), "=&r"(tmp_gp));         \
-       (typeof(&per_cpu_var(var)))(__ptr + (offset)); })
-
-#define PER_CPU_DEF_ATTRIBUTES __used
-
-#endif /* MODULE */
-
-/*
- * A percpu variable may point to a discarded regions. The following are
- * established ways to produce a usable pointer from the percpu variable
- * offset.
+ * Always use weak definitions for percpu variables in modules.
  */
-#define per_cpu(var, cpu) \
-       (*SHIFT_PERCPU_PTR(var, per_cpu_offset(cpu)))
-#define __get_cpu_var(var) \
-       (*SHIFT_PERCPU_PTR(var, my_cpu_offset))
-#define __raw_get_cpu_var(var) \
-       (*SHIFT_PERCPU_PTR(var, __my_cpu_offset))
-
-#else /* ! SMP */
-
-#define per_cpu(var, cpu)              (*((void)(cpu), &per_cpu_var(var)))
-#define __get_cpu_var(var)             per_cpu_var(var)
-#define __raw_get_cpu_var(var)         per_cpu_var(var)
-
-#define PER_CPU_DEF_ATTRIBUTES
-
-#endif /* SMP */
-
-#ifdef CONFIG_SMP
-#define PER_CPU_BASE_SECTION ".data.percpu"
-#else
-#define PER_CPU_BASE_SECTION ".data"
-#endif
-
-#ifdef CONFIG_SMP
-
-#ifdef MODULE
-#define PER_CPU_SHARED_ALIGNED_SECTION ""
-#else
-#define PER_CPU_SHARED_ALIGNED_SECTION ".shared_aligned"
-#endif
-#define PER_CPU_FIRST_SECTION ".first"
-
-#else
-
-#define PER_CPU_SHARED_ALIGNED_SECTION ""
-#define PER_CPU_FIRST_SECTION ""
-
+#if defined(MODULE) && defined(CONFIG_SMP)
+#define ARCH_NEEDS_WEAK_PER_CPU
 #endif
 
-#define PER_CPU_ATTRIBUTES
+#include <asm-generic/percpu.h>
 
 #endif /* __ALPHA_PERCPU_H */
index 9d87aaa..e89e0c2 100644 (file)
@@ -2,6 +2,7 @@
 #define _ALPHA_TLBFLUSH_H
 
 #include <linux/mm.h>
+#include <linux/sched.h>
 #include <asm/compiler.h>
 #include <asm/pgalloc.h>
 
index b9d6568..75fe1d6 100644 (file)
@@ -139,6 +139,7 @@ SECTIONS
                EXIT_TEXT
                EXIT_DATA
                *(.exitcall.exit)
+               *(.discard)
        }
 
        .mdebug 0 : {
index 6937102..5cc4812 100644 (file)
@@ -83,6 +83,7 @@ SECTIONS
                EXIT_TEXT
                EXIT_DATA
                *(.exitcall.exit)
+               *(.discard)
                *(.ARM.exidx.exit.text)
                *(.ARM.extab.exit.text)
 #ifndef CONFIG_HOTPLUG_CPU
index 7910d41..b832460 100644 (file)
@@ -131,6 +131,7 @@ SECTIONS
        /DISCARD/               : {
                EXIT_DATA
                *(.exitcall.exit)
+               *(.discard)
        }
 
        DWARF_DEBUG
index 6ac307c..6e8eabd 100644 (file)
@@ -280,5 +280,6 @@ SECTIONS
        /DISCARD/ :
        {
                *(.exitcall.exit)
+               *(.discard)
        }
 }
index 0bc3c4e..99e4dbb 100644 (file)
@@ -42,9 +42,9 @@
 #include <asm/mem_map.h>
 #include "blackfin_sram.h"
 
-static DEFINE_PER_CPU(spinlock_t, l1sram_lock) ____cacheline_aligned_in_smp;
-static DEFINE_PER_CPU(spinlock_t, l1_data_sram_lock) ____cacheline_aligned_in_smp;
-static DEFINE_PER_CPU(spinlock_t, l1_inst_sram_lock) ____cacheline_aligned_in_smp;
+static DEFINE_PER_CPU_SHARED_ALIGNED(spinlock_t, l1sram_lock);
+static DEFINE_PER_CPU_SHARED_ALIGNED(spinlock_t, l1_data_sram_lock);
+static DEFINE_PER_CPU_SHARED_ALIGNED(spinlock_t, l1_inst_sram_lock);
 static spinlock_t l2_sram_lock ____cacheline_aligned_in_smp;
 
 /* the data structure for L1 scratchpad and DATA SRAM */
index 72ba08d..1d45fd6 100644 (file)
@@ -17,7 +17,8 @@ extern void switch_mm(struct mm_struct *prev, struct mm_struct *next,
  * registers like cr3 on the i386
  */
 
-extern volatile DEFINE_PER_CPU(pgd_t *,current_pgd); /* defined in arch/cris/mm/fault.c */
+/* defined in arch/cris/mm/fault.c */
+DECLARE_PER_CPU(pgd_t *, current_pgd);
 
 static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
 {
index 0d2adfc..a3175eb 100644 (file)
@@ -145,6 +145,7 @@ SECTIONS
                EXIT_TEXT
                EXIT_DATA
                *(.exitcall.exit)
+               *(.discard)
         }
 
        dram_end = dram_start + (CONFIG_ETRAX_DRAM_SIZE - __CONFIG_ETRAX_VMEM_SIZE)*1024*1024;
index f925115..4a7cdd9 100644 (file)
@@ -29,7 +29,7 @@ extern void die_if_kernel(const char *, struct pt_regs *, long);
 
 /* current active page directory */
 
-volatile DEFINE_PER_CPU(pgd_t *,current_pgd);
+DEFINE_PER_CPU(pgd_t *, current_pgd);
 unsigned long cris_signal_return_page;
 
 /*
index 22d9787..64b5a5e 100644 (file)
@@ -177,6 +177,8 @@ SECTIONS
   .debug_ranges                0 : { *(.debug_ranges) }
 
   .comment 0 : { *(.comment) }
+
+  /DISCARD/ : { *(.discard) }
 }
 
 __kernel_image_size_no_bss = __bss_start - __kernel_image_start;
index 43a87b9..03d6c0d 100644 (file)
@@ -154,6 +154,7 @@ SECTIONS
        }
        /DISCARD/ : {
                *(.exitcall.exit)
+               *(.discard)
        }
         .romfs :       
        {
index 170042b..328d2f8 100644 (file)
@@ -89,6 +89,9 @@ config GENERIC_TIME_VSYSCALL
        bool
        default y
 
+config HAVE_LEGACY_PER_CPU_AREA
+       def_bool y
+
 config HAVE_SETUP_PER_CPU_AREA
        def_bool y
 
index f0c521b..93ebfea 100644 (file)
@@ -58,7 +58,8 @@ static struct local_tlb_flush_counts {
        unsigned int count;
 } __attribute__((__aligned__(32))) local_tlb_flush_counts[NR_CPUS];
 
-static DEFINE_PER_CPU(unsigned short, shadow_flush_counts[NR_CPUS]) ____cacheline_aligned;
+static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned short [NR_CPUS],
+                                    shadow_flush_counts);
 
 #define IPI_CALL_FUNC          0
 #define IPI_CPU_STOP           1
index 4a95e86..13d9589 100644 (file)
@@ -29,6 +29,7 @@ SECTIONS
        EXIT_TEXT
        EXIT_DATA
        *(.exitcall.exit)
+       *(.discard)
        *(.IA_64.unwind.exit.text)
        *(.IA_64.unwind_info.exit.text)
        }
index e456f06..ece1bf9 100644 (file)
@@ -71,7 +71,7 @@ EXPORT_SYMBOL(sn_rtc_cycles_per_second);
 DEFINE_PER_CPU(struct sn_hub_info_s, __sn_hub_info);
 EXPORT_PER_CPU_SYMBOL(__sn_hub_info);
 
-DEFINE_PER_CPU(short, __sn_cnodeid_to_nasid[MAX_COMPACT_NODES]);
+DEFINE_PER_CPU(short [MAX_COMPACT_NODES], __sn_cnodeid_to_nasid);
 EXPORT_PER_CPU_SYMBOL(__sn_cnodeid_to_nasid);
 
 DEFINE_PER_CPU(struct nodepda_s *, __sn_nodepda);
index 4179adf..480a499 100644 (file)
@@ -125,6 +125,7 @@ SECTIONS
        EXIT_TEXT
        EXIT_DATA
        *(.exitcall.exit)
+       *(.discard)
        }
 
   /* Stabs debugging sections.  */
index 01d212b..905a797 100644 (file)
@@ -87,6 +87,7 @@ SECTIONS
        EXIT_TEXT
        EXIT_DATA
        *(.exitcall.exit)
+       *(.discard)
        }
 
   /* Stabs debugging sections.  */
index c192f77..47d04be 100644 (file)
@@ -82,6 +82,7 @@ __init_begin = .;
        EXIT_TEXT
        EXIT_DATA
        *(.exitcall.exit)
+       *(.discard)
        }
 
   .crap : {
index b7fe505..68111a6 100644 (file)
@@ -188,6 +188,7 @@ SECTIONS {
                EXIT_TEXT
                EXIT_DATA
                *(.exitcall.exit)
+               *(.discard)
        }
 
        .bss : {
index d34d38d..a207543 100644 (file)
@@ -162,4 +162,6 @@ SECTIONS {
        }
        . = ALIGN(4096);
        _end = .;
+
+       /DISCARD/ : { *(.discard) }
 }
index 58738c8..4590160 100644 (file)
@@ -179,6 +179,7 @@ SECTIONS
        /* Sections to be discarded */
        /DISCARD/ : {
                *(.exitcall.exit)
+               *(.discard)
 
                /* ABI crap starts here */
                *(.MIPS.options)
index bcebcef..5609d49 100644 (file)
@@ -118,6 +118,7 @@ SECTIONS
   /* Sections to be discarded */
   /DISCARD/ : {
        EXIT_CALL
+       *(.discard)
        }
 
   STABS_DEBUG
index fd2cc4f..ccf5834 100644 (file)
@@ -240,6 +240,7 @@ SECTIONS
        /* Sections to be discarded */
        /DISCARD/ : {
                *(.exitcall.exit)
+               *(.discard)
 #ifdef CONFIG_64BIT
                /* temporary hack until binutils is fixed to not emit these
                 * for static binaries
index d00131c..61bbffa 100644 (file)
@@ -46,6 +46,9 @@ config GENERIC_HARDIRQS_NO__DO_IRQ
        bool
        default y
 
+config HAVE_LEGACY_PER_CPU_AREA
+       def_bool PPC64
+
 config HAVE_SETUP_PER_CPU_AREA
        def_bool PPC64
 
index 8ef8a14..7fca935 100644 (file)
@@ -40,6 +40,7 @@ SECTIONS
        /* Sections to be discarded. */
        /DISCARD/ : {
        *(.exitcall.exit)
+       *(.discard)
        EXIT_DATA
        }
 
index 98cd1dc..6e9b69c 100644 (file)
@@ -31,7 +31,7 @@ struct stab_entry {
 
 #define NR_STAB_CACHE_ENTRIES 8
 static DEFINE_PER_CPU(long, stab_cache_ptr);
-static DEFINE_PER_CPU(long, stab_cache[NR_STAB_CACHE_ENTRIES]);
+static DEFINE_PER_CPU(long [NR_STAB_CACHE_ENTRIES], stab_cache);
 
 /*
  * Create a segment table entry for the given esid/vsid pair.
index f6e04bc..51ffde4 100644 (file)
@@ -37,7 +37,7 @@
   */
 
 #define MSG_COUNT 4
-static DEFINE_PER_CPU(unsigned int, ps3_ipi_virqs[MSG_COUNT]);
+static DEFINE_PER_CPU(unsigned int [MSG_COUNT], ps3_ipi_virqs);
 
 static void do_message_pass(int target, int msg)
 {
index 408d60b..f7ad871 100644 (file)
@@ -1,37 +1,21 @@
 #ifndef __ARCH_S390_PERCPU__
 #define __ARCH_S390_PERCPU__
 
-#include <linux/compiler.h>
-#include <asm/lowcore.h>
-
 /*
  * s390 uses its own implementation for per cpu data, the offset of
  * the cpu local data area is cached in the cpu's lowcore memory.
- * For 64 bit module code s390 forces the use of a GOT slot for the
- * address of the per cpu variable. This is needed because the module
- * may be more than 4G above the per cpu area.
  */
-#if defined(__s390x__) && defined(MODULE)
-
-#define SHIFT_PERCPU_PTR(ptr,offset) (({                       \
-       extern int simple_identifier_##var(void);       \
-       unsigned long *__ptr;                           \
-       asm ( "larl %0, %1@GOTENT"              \
-           : "=a" (__ptr) : "X" (ptr) );               \
-       (typeof(ptr))((*__ptr) + (offset));     }))
-
-#else
-
-#define SHIFT_PERCPU_PTR(ptr, offset) (({                              \
-       extern int simple_identifier_##var(void);               \
-       unsigned long __ptr;                                    \
-       asm ( "" : "=a" (__ptr) : "0" (ptr) );                  \
-       (typeof(ptr)) (__ptr + (offset)); }))
+#define __my_cpu_offset S390_lowcore.percpu_offset
 
+/*
+ * For 64 bit module code, the module may be more than 4G above the
+ * per cpu area, use weak definitions to force the compiler to
+ * generate external references.
+ */
+#if defined(CONFIG_SMP) && defined(__s390x__) && defined(MODULE)
+#define ARCH_NEEDS_WEAK_PER_CPU
 #endif
 
-#define __my_cpu_offset S390_lowcore.percpu_offset
-
 #include <asm-generic/percpu.h>
 
 #endif /* __ARCH_S390_PERCPU__ */
index a53db23..98867df 100644 (file)
@@ -161,6 +161,7 @@ SECTIONS
        /DISCARD/ : {
                EXIT_DATA
                *(.exitcall.exit)
+               *(.discard)
        }
 
        /* Debugging sections.  */
index f53c76a..766976d 100644 (file)
@@ -171,6 +171,7 @@ SECTIONS
         */
        /DISCARD/ : {
                *(.exitcall.exit)
+               *(.discard)
        }
 
        STABS_DEBUG
index 3f8b6a9..4f6ed0f 100644 (file)
@@ -95,9 +95,6 @@ config AUDIT_ARCH
 config HAVE_SETUP_PER_CPU_AREA
        def_bool y if SPARC64
 
-config HAVE_DYNAMIC_PER_CPU_AREA
-       def_bool y if SPARC64
-
 config GENERIC_HARDIRQS_NO__DO_IRQ
        bool
        def_bool y if SPARC64
index fcbbd00..d63cf91 100644 (file)
@@ -175,6 +175,7 @@ SECTIONS
                EXIT_TEXT
                EXIT_DATA
                *(.exitcall.exit)
+               *(.discard)
        }
 
        STABS_DEBUG
index 9975e1a..2916d6e 100644 (file)
@@ -156,4 +156,6 @@ SECTIONS
   STABS_DEBUG
 
   DWARF_DEBUG
+
+  /DISCARD/    : { *(.discard) }
 }
index 11b8352..1f8a622 100644 (file)
@@ -100,4 +100,6 @@ SECTIONS
   STABS_DEBUG
 
   DWARF_DEBUG
+
+  /DISCARD/    : { *(.discard) }
 }
index c07f722..646fcab 100644 (file)
@@ -149,9 +149,6 @@ config ARCH_HAS_CACHE_LINE_SIZE
 config HAVE_SETUP_PER_CPU_AREA
        def_bool y
 
-config HAVE_DYNAMIC_PER_CPU_AREA
-       def_bool y
-
 config HAVE_CPUMASK_OF_CPU_MAP
        def_bool X86_64_SMP
 
index 6b2a52d..dca325c 100644 (file)
@@ -30,8 +30,8 @@
 #include <asm/apic.h>
 #include <asm/desc.h>
 
-static DEFINE_PER_CPU(struct cpu_cpuX_base, cpu_arr[CPU_REG_ALL_BIT]);
-static DEFINE_PER_CPU(struct cpu_private *, priv_arr[MAX_CPU_FILES]);
+static DEFINE_PER_CPU(struct cpu_cpuX_base [CPU_REG_ALL_BIT], cpu_arr);
+static DEFINE_PER_CPU(struct cpu_private * [MAX_CPU_FILES], priv_arr);
 static DEFINE_PER_CPU(int, cpu_priv_count);
 
 static DEFINE_MUTEX(cpu_debug_lock);
index af425b8..20d4983 100644 (file)
@@ -1091,7 +1091,7 @@ void mce_log_therm_throt_event(__u64 status)
  */
 static int check_interval = 5 * 60; /* 5 minutes */
 
-static DEFINE_PER_CPU(int, next_interval); /* in jiffies */
+static DEFINE_PER_CPU(int, mce_next_interval); /* in jiffies */
 static DEFINE_PER_CPU(struct timer_list, mce_timer);
 
 static void mcheck_timer(unsigned long data)
@@ -1110,7 +1110,7 @@ static void mcheck_timer(unsigned long data)
         * Alert userspace if needed.  If we logged an MCE, reduce the
         * polling interval, otherwise increase the polling interval.
         */
-       n = &__get_cpu_var(next_interval);
+       n = &__get_cpu_var(mce_next_interval);
        if (mce_notify_irq())
                *n = max(*n/2, HZ/100);
        else
@@ -1311,7 +1311,7 @@ static void mce_cpu_features(struct cpuinfo_x86 *c)
 static void mce_init_timer(void)
 {
        struct timer_list *t = &__get_cpu_var(mce_timer);
-       int *n = &__get_cpu_var(next_interval);
+       int *n = &__get_cpu_var(mce_next_interval);
 
        if (mce_ignore_ce)
                return;
@@ -1914,7 +1914,7 @@ mce_cpu_callback(struct notifier_block *nfb, unsigned long action, void *hcpu)
        case CPU_DOWN_FAILED:
        case CPU_DOWN_FAILED_FROZEN:
                t->expires = round_jiffies(jiffies +
-                                               __get_cpu_var(next_interval));
+                                          __get_cpu_var(mce_next_interval));
                add_timer_on(t, cpu);
                smp_call_function_single(cpu, mce_reenable_cpu, &action, 1);
                break;
index ddae216..bd2a2fa 100644 (file)
@@ -69,7 +69,7 @@ struct threshold_bank {
        struct threshold_block  *blocks;
        cpumask_var_t           cpus;
 };
-static DEFINE_PER_CPU(struct threshold_bank *, threshold_banks[NR_BANKS]);
+static DEFINE_PER_CPU(struct threshold_bank * [NR_BANKS], threshold_banks);
 
 #ifdef CONFIG_SMP
 static unsigned char shared_bank[NR_BANKS] = {
index d4cf4ce..13bd6d6 100644 (file)
@@ -862,7 +862,7 @@ amd_pmu_disable_counter(struct hw_perf_counter *hwc, int idx)
        x86_pmu_disable_counter(hwc, idx);
 }
 
-static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
+static DEFINE_PER_CPU(u64 [X86_PMC_IDX_MAX], pmc_prev_left);
 
 /*
  * Set the next IRQ period, based on the hwc->period_left value.
@@ -901,7 +901,7 @@ x86_perf_counter_set_period(struct perf_counter *counter,
        if (left > x86_pmu.max_period)
                left = x86_pmu.max_period;
 
-       per_cpu(prev_left[idx], smp_processor_id()) = left;
+       per_cpu(pmc_prev_left[idx], smp_processor_id()) = left;
 
        /*
         * The hw counter starts counting from this counter offset,
@@ -1086,7 +1086,7 @@ void perf_counter_print_debug(void)
                rdmsrl(x86_pmu.eventsel + idx, pmc_ctrl);
                rdmsrl(x86_pmu.perfctr  + idx, pmc_count);
 
-               prev_left = per_cpu(prev_left[idx], cpu);
+               prev_left = per_cpu(pmc_prev_left[idx], cpu);
 
                pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
                        cpu, idx, pmc_ctrl);
@@ -1559,8 +1559,8 @@ void callchain_store(struct perf_callchain_entry *entry, u64 ip)
                entry->ip[entry->nr++] = ip;
 }
 
-static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
-static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
+static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_irq_entry);
+static DEFINE_PER_CPU(struct perf_callchain_entry, pmc_nmi_entry);
 
 
 static void
@@ -1707,9 +1707,9 @@ struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
        struct perf_callchain_entry *entry;
 
        if (in_nmi())
-               entry = &__get_cpu_var(nmi_entry);
+               entry = &__get_cpu_var(pmc_nmi_entry);
        else
-               entry = &__get_cpu_var(irq_entry);
+               entry = &__get_cpu_var(pmc_irq_entry);
 
        entry->nr = 0;
 
index 41c159c..b1e2463 100644 (file)
@@ -287,6 +287,7 @@ SECTIONS
        EXIT_TEXT
        EXIT_DATA
         *(.exitcall.exit)
+       *(.discard)
   }
 
   .xt.lit : { *(.xt.lit) }
index 7a12cf6..ce8ba57 100644 (file)
@@ -146,7 +146,7 @@ enum arq_state {
 #define RQ_STATE(rq)   ((enum arq_state)(rq)->elevator_private2)
 #define RQ_SET_STATE(rq, state)        ((rq)->elevator_private2 = (void *) state)
 
-static DEFINE_PER_CPU(unsigned long, ioc_count);
+static DEFINE_PER_CPU(unsigned long, as_ioc_count);
 static struct completion *ioc_gone;
 static DEFINE_SPINLOCK(ioc_gone_lock);
 
@@ -161,7 +161,7 @@ static void as_antic_stop(struct as_data *ad);
 static void free_as_io_context(struct as_io_context *aic)
 {
        kfree(aic);
-       elv_ioc_count_dec(ioc_count);
+       elv_ioc_count_dec(as_ioc_count);
        if (ioc_gone) {
                /*
                 * AS scheduler is exiting, grab exit lock and check
@@ -169,7 +169,7 @@ static void free_as_io_context(struct as_io_context *aic)
                 * complete ioc_gone and set it back to NULL.
                 */
                spin_lock(&ioc_gone_lock);
-               if (ioc_gone && !elv_ioc_count_read(ioc_count)) {
+               if (ioc_gone && !elv_ioc_count_read(as_ioc_count)) {
                        complete(ioc_gone);
                        ioc_gone = NULL;
                }
@@ -211,7 +211,7 @@ static struct as_io_context *alloc_as_io_context(void)
                ret->seek_total = 0;
                ret->seek_samples = 0;
                ret->seek_mean = 0;
-               elv_ioc_count_inc(ioc_count);
+               elv_ioc_count_inc(as_ioc_count);
        }
 
        return ret;
@@ -1507,7 +1507,7 @@ static void __exit as_exit(void)
        ioc_gone = &all_gone;
        /* ioc_gone's update must be visible before reading ioc_count */
        smp_wmb();
-       if (elv_ioc_count_read(ioc_count))
+       if (elv_ioc_count_read(as_ioc_count))
                wait_for_completion(&all_gone);
        synchronize_rcu();
 }
index 87276eb..85208dd 100644 (file)
@@ -48,7 +48,7 @@ static int cfq_slice_idle = HZ / 125;
 static struct kmem_cache *cfq_pool;
 static struct kmem_cache *cfq_ioc_pool;
 
-static DEFINE_PER_CPU(unsigned long, ioc_count);
+static DEFINE_PER_CPU(unsigned long, cfq_ioc_count);
 static struct completion *ioc_gone;
 static DEFINE_SPINLOCK(ioc_gone_lock);
 
@@ -1427,7 +1427,7 @@ static void cfq_cic_free_rcu(struct rcu_head *head)
        cic = container_of(head, struct cfq_io_context, rcu_head);
 
        kmem_cache_free(cfq_ioc_pool, cic);
-       elv_ioc_count_dec(ioc_count);
+       elv_ioc_count_dec(cfq_ioc_count);
 
        if (ioc_gone) {
                /*
@@ -1436,7 +1436,7 @@ static void cfq_cic_free_rcu(struct rcu_head *head)
                 * complete ioc_gone and set it back to NULL
                 */
                spin_lock(&ioc_gone_lock);
-               if (ioc_gone && !elv_ioc_count_read(ioc_count)) {
+               if (ioc_gone && !elv_ioc_count_read(cfq_ioc_count)) {
                        complete(ioc_gone);
                        ioc_gone = NULL;
                }
@@ -1562,7 +1562,7 @@ cfq_alloc_io_context(struct cfq_data *cfqd, gfp_t gfp_mask)
                INIT_HLIST_NODE(&cic->cic_list);
                cic->dtor = cfq_free_io_context;
                cic->exit = cfq_exit_io_context;
-               elv_ioc_count_inc(ioc_count);
+               elv_ioc_count_inc(cfq_ioc_count);
        }
 
        return cic;
@@ -2668,7 +2668,7 @@ static void __exit cfq_exit(void)
         * this also protects us from entering cfq_slab_kill() with
         * pending RCU callbacks
         */
-       if (elv_ioc_count_read(ioc_count))
+       if (elv_ioc_count_read(cfq_ioc_count))
                wait_for_completion(&all_gone);
        cfq_slab_kill();
 }
index 7fc58af..a7ef465 100644 (file)
@@ -65,7 +65,7 @@ struct cpu_dbs_info_s {
        int cpu;
        unsigned int enable:1;
 };
-static DEFINE_PER_CPU(struct cpu_dbs_info_s, cpu_dbs_info);
+static DEFINE_PER_CPU(struct cpu_dbs_info_s, cs_cpu_dbs_info);
 
 static unsigned int dbs_enable;        /* number of CPUs using this policy */
 
@@ -138,7 +138,7 @@ dbs_cpufreq_notifier(struct notifier_block *nb, unsigned long val,
                     void *data)
 {
        struct cpufreq_freqs *freq = data;
-       struct cpu_dbs_info_s *this_dbs_info = &per_cpu(cpu_dbs_info,
+       struct cpu_dbs_info_s *this_dbs_info = &per_cpu(cs_cpu_dbs_info,
                                                        freq->cpu);
 
        struct cpufreq_policy *policy;
@@ -298,7 +298,7 @@ static ssize_t store_ignore_nice_load(struct cpufreq_policy *policy,
        /* we need to re-evaluate prev_cpu_idle */
        for_each_online_cpu(j) {
                struct cpu_dbs_info_s *dbs_info;
-               dbs_info = &per_cpu(cpu_dbs_info, j);
+               dbs_info = &per_cpu(cs_cpu_dbs_info, j);
                dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
                                                &dbs_info->prev_cpu_wall);
                if (dbs_tuners_ins.ignore_nice)
@@ -388,7 +388,7 @@ static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
                cputime64_t cur_wall_time, cur_idle_time;
                unsigned int idle_time, wall_time;
 
-               j_dbs_info = &per_cpu(cpu_dbs_info, j);
+               j_dbs_info = &per_cpu(cs_cpu_dbs_info, j);
 
                cur_idle_time = get_cpu_idle_time(j, &cur_wall_time);
 
@@ -528,7 +528,7 @@ static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
        unsigned int j;
        int rc;
 
-       this_dbs_info = &per_cpu(cpu_dbs_info, cpu);
+       this_dbs_info = &per_cpu(cs_cpu_dbs_info, cpu);
 
        switch (event) {
        case CPUFREQ_GOV_START:
@@ -548,7 +548,7 @@ static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
 
                for_each_cpu(j, policy->cpus) {
                        struct cpu_dbs_info_s *j_dbs_info;
-                       j_dbs_info = &per_cpu(cpu_dbs_info, j);
+                       j_dbs_info = &per_cpu(cs_cpu_dbs_info, j);
                        j_dbs_info->cur_policy = policy;
 
                        j_dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
index 1911d17..36f292a 100644 (file)
@@ -73,7 +73,7 @@ struct cpu_dbs_info_s {
        unsigned int enable:1,
                sample_type:1;
 };
-static DEFINE_PER_CPU(struct cpu_dbs_info_s, cpu_dbs_info);
+static DEFINE_PER_CPU(struct cpu_dbs_info_s, od_cpu_dbs_info);
 
 static unsigned int dbs_enable;        /* number of CPUs using this policy */
 
@@ -151,7 +151,8 @@ static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
        unsigned int freq_hi, freq_lo;
        unsigned int index = 0;
        unsigned int jiffies_total, jiffies_hi, jiffies_lo;
-       struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, policy->cpu);
+       struct cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info,
+                                                  policy->cpu);
 
        if (!dbs_info->freq_table) {
                dbs_info->freq_lo = 0;
@@ -196,7 +197,7 @@ static void ondemand_powersave_bias_init(void)
 {
        int i;
        for_each_online_cpu(i) {
-               struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, i);
+               struct cpu_dbs_info_s *dbs_info = &per_cpu(od_cpu_dbs_info, i);
                dbs_info->freq_table = cpufreq_frequency_get_table(i);
                dbs_info->freq_lo = 0;
        }
@@ -297,7 +298,7 @@ static ssize_t store_ignore_nice_load(struct cpufreq_policy *policy,
        /* we need to re-evaluate prev_cpu_idle */
        for_each_online_cpu(j) {
                struct cpu_dbs_info_s *dbs_info;
-               dbs_info = &per_cpu(cpu_dbs_info, j);
+               dbs_info = &per_cpu(od_cpu_dbs_info, j);
                dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
                                                &dbs_info->prev_cpu_wall);
                if (dbs_tuners_ins.ignore_nice)
@@ -391,7 +392,7 @@ static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
                unsigned int load, load_freq;
                int freq_avg;
 
-               j_dbs_info = &per_cpu(cpu_dbs_info, j);
+               j_dbs_info = &per_cpu(od_cpu_dbs_info, j);
 
                cur_idle_time = get_cpu_idle_time(j, &cur_wall_time);
 
@@ -548,7 +549,7 @@ static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
        unsigned int j;
        int rc;
 
-       this_dbs_info = &per_cpu(cpu_dbs_info, cpu);
+       this_dbs_info = &per_cpu(od_cpu_dbs_info, cpu);
 
        switch (event) {
        case CPUFREQ_GOV_START:
@@ -570,7 +571,7 @@ static int cpufreq_governor_dbs(struct cpufreq_policy *policy,
 
                for_each_cpu(j, policy->cpus) {
                        struct cpu_dbs_info_s *j_dbs_info;
-                       j_dbs_info = &per_cpu(cpu_dbs_info, j);
+                       j_dbs_info = &per_cpu(od_cpu_dbs_info, j);
                        j_dbs_info->cur_policy = policy;
 
                        j_dbs_info->prev_cpu_idle = get_cpu_idle_time(j,
index 891d2e9..7d2987e 100644 (file)
 static DEFINE_SPINLOCK(irq_mapping_update_lock);
 
 /* IRQ <-> VIRQ mapping. */
-static DEFINE_PER_CPU(int, virq_to_irq[NR_VIRQS]) = {[0 ... NR_VIRQS-1] = -1};
+static DEFINE_PER_CPU(int [NR_VIRQS], virq_to_irq) = {[0 ... NR_VIRQS-1] = -1};
 
 /* IRQ <-> IPI mapping */
-static DEFINE_PER_CPU(int, ipi_to_irq[XEN_NR_IPIS]) = {[0 ... XEN_NR_IPIS-1] = -1};
+static DEFINE_PER_CPU(int [XEN_NR_IPIS], ipi_to_irq) = {[0 ... XEN_NR_IPIS-1] = -1};
 
 /* Interrupt types. */
 enum xen_irq_type {
@@ -602,6 +602,8 @@ irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
        return IRQ_HANDLED;
 }
 
+static DEFINE_PER_CPU(unsigned, xed_nesting_count);
+
 /*
  * Search the CPUs pending events bitmasks.  For each one found, map
  * the event number to an irq, and feed it into do_IRQ() for
@@ -617,7 +619,6 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)
        struct pt_regs *old_regs = set_irq_regs(regs);
        struct shared_info *s = HYPERVISOR_shared_info;
        struct vcpu_info *vcpu_info = __get_cpu_var(xen_vcpu);
-       static DEFINE_PER_CPU(unsigned, nesting_count);
        unsigned count;
 
        exit_idle();
@@ -628,7 +629,7 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)
 
                vcpu_info->evtchn_upcall_pending = 0;
 
-               if (__get_cpu_var(nesting_count)++)
+               if (__get_cpu_var(xed_nesting_count)++)
                        goto out;
 
 #ifndef CONFIG_X86 /* No need for a barrier -- XCHG is a barrier on x86. */
@@ -653,8 +654,8 @@ void xen_evtchn_do_upcall(struct pt_regs *regs)
 
                BUG_ON(!irqs_disabled());
 
-               count = __get_cpu_var(nesting_count);
-               __get_cpu_var(nesting_count) = 0;
+               count = __get_cpu_var(xed_nesting_count);
+               __get_cpu_var(xed_nesting_count) = 0;
        } while(count != 1);
 
 out:
index dccdbed..c5c18ac 100644 (file)
 #define INIT_RAM_FS
 #endif
 
+#define DISCARDS                                                       \
+       /DISCARD/ : {                                                   \
+       EXIT_TEXT                                                       \
+       EXIT_DATA                                                       \
+       *(.exitcall.exit)                                               \
+       *(.discard)                                                     \
+       }
+
 /**
  * PERCPU_VADDR - define output section for percpu area
  * @vaddr: explicit base address (optional)
index 68438e1..aefc2f1 100644 (file)
 /*
  * Base implementations of per-CPU variable declarations and definitions, where
  * the section in which the variable is to be placed is provided by the
- * 'section' argument.  This may be used to affect the parameters governing the
+ * 'sec' argument.  This may be used to affect the parameters governing the
  * variable's storage.
  *
  * NOTE!  The sections for the DECLARE and for the DEFINE must match, lest
  * linkage errors occur due the compiler generating the wrong code to access
  * that section.
  */
-#define DECLARE_PER_CPU_SECTION(type, name, section)                   \
-       extern                                                          \
-       __attribute__((__section__(PER_CPU_BASE_SECTION section)))      \
-       PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
-
-#define DEFINE_PER_CPU_SECTION(type, name, section)                    \
-       __attribute__((__section__(PER_CPU_BASE_SECTION section)))      \
-       PER_CPU_ATTRIBUTES PER_CPU_DEF_ATTRIBUTES                       \
+#define __PCPU_ATTRS(sec)                                              \
+       __attribute__((section(PER_CPU_BASE_SECTION sec)))              \
+       PER_CPU_ATTRIBUTES
+
+#define __PCPU_DUMMY_ATTRS                                             \
+       __attribute__((section(".discard"), unused))
+
+/*
+ * s390 and alpha modules require percpu variables to be defined as
+ * weak to force the compiler to generate GOT based external
+ * references for them.  This is necessary because percpu sections
+ * will be located outside of the usually addressable area.
+ *
+ * This definition puts the following two extra restrictions when
+ * defining percpu variables.
+ *
+ * 1. The symbol must be globally unique, even the static ones.
+ * 2. Static percpu variables cannot be defined inside a function.
+ *
+ * Archs which need weak percpu definitions should define
+ * ARCH_NEEDS_WEAK_PER_CPU in asm/percpu.h when necessary.
+ *
+ * To ensure that the generic code observes the above two
+ * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak
+ * definition is used for all cases.
+ */
+#if defined(ARCH_NEEDS_WEAK_PER_CPU) || defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU)
+/*
+ * __pcpu_scope_* dummy variable is used to enforce scope.  It
+ * receives the static modifier when it's used in front of
+ * DEFINE_PER_CPU() and will trigger build failure if
+ * DECLARE_PER_CPU() is used for the same variable.
+ *
+ * __pcpu_unique_* dummy variable is used to enforce symbol uniqueness
+ * such that hidden weak symbol collision, which will cause unrelated
+ * variables to share the same address, can be detected during build.
+ */
+#define DECLARE_PER_CPU_SECTION(type, name, sec)                       \
+       extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name;             \
+       extern __PCPU_ATTRS(sec) __typeof__(type) per_cpu__##name
+
+#define DEFINE_PER_CPU_SECTION(type, name, sec)                                \
+       __PCPU_DUMMY_ATTRS char __pcpu_scope_##name;                    \
+       __PCPU_DUMMY_ATTRS char __pcpu_unique_##name;                   \
+       __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak                 \
+       __typeof__(type) per_cpu__##name
+#else
+/*
+ * Normal declaration and definition macros.
+ */
+#define DECLARE_PER_CPU_SECTION(type, name, sec)                       \
+       extern __PCPU_ATTRS(sec) __typeof__(type) per_cpu__##name
+
+#define DEFINE_PER_CPU_SECTION(type, name, sec)                                \
+       __PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES                        \
        __typeof__(type) per_cpu__##name
+#endif
 
 /*
  * Variant on the per-CPU variable declaration/definition theme used for
index 26fd9d1..e500034 100644 (file)
@@ -34,7 +34,7 @@
 
 #ifdef CONFIG_SMP
 
-#ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
+#ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA
 
 /* minimum unit size, also is the maximum supported allocation size */
 #define PCPU_MIN_UNIT_SIZE             PFN_ALIGN(64 << 10)
@@ -80,7 +80,7 @@ extern ssize_t __init pcpu_embed_first_chunk(
 
 extern void *__alloc_reserved_percpu(size_t size, size_t align);
 
-#else /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
+#else /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */
 
 struct percpu_data {
        void *ptrs[1];
@@ -99,11 +99,15 @@ struct percpu_data {
         (__typeof__(ptr))__p->ptrs[(cpu)];                             \
 })
 
-#endif /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
+#endif /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */
 
 extern void *__alloc_percpu(size_t size, size_t align);
 extern void free_percpu(void *__pdata);
 
+#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
+extern void __init setup_per_cpu_areas(void);
+#endif
+
 #else /* CONFIG_SMP */
 
 #define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
@@ -124,6 +128,8 @@ static inline void free_percpu(void *p)
        kfree(p);
 }
 
+static inline void __init setup_per_cpu_areas(void) { }
+
 #endif /* CONFIG_SMP */
 
 #define alloc_percpu(type)     (type *)__alloc_percpu(sizeof(type), \
index 2c5ade7..7183285 100644 (file)
@@ -353,7 +353,6 @@ static void __init smp_init(void)
 #define smp_init()     do { } while (0)
 #endif
 
-static inline void setup_per_cpu_areas(void) { }
 static inline void setup_nr_cpu_ids(void) { }
 static inline void smp_prepare_cpus(unsigned int maxcpus) { }
 
@@ -374,29 +373,6 @@ static void __init setup_nr_cpu_ids(void)
        nr_cpu_ids = find_last_bit(cpumask_bits(cpu_possible_mask),NR_CPUS) + 1;
 }
 
-#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
-unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
-
-EXPORT_SYMBOL(__per_cpu_offset);
-
-static void __init setup_per_cpu_areas(void)
-{
-       unsigned long size, i;
-       char *ptr;
-       unsigned long nr_possible_cpus = num_possible_cpus();
-
-       /* Copy section for each CPU (we discard the original) */
-       size = ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE);
-       ptr = alloc_bootmem_pages(size * nr_possible_cpus);
-
-       for_each_possible_cpu(i) {
-               __per_cpu_offset[i] = ptr - __per_cpu_start;
-               memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
-               ptr += size;
-       }
-}
-#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */
-
 /* Called by boot processor to activate the rest. */
 static void __init smp_init(void)
 {
index 38928fc..f593495 100644 (file)
@@ -364,7 +364,7 @@ EXPORT_SYMBOL_GPL(find_module);
 
 #ifdef CONFIG_SMP
 
-#ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
+#ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA
 
 static void *percpu_modalloc(unsigned long size, unsigned long align,
                             const char *name)
@@ -389,7 +389,7 @@ static void percpu_modfree(void *freeme)
        free_percpu(freeme);
 }
 
-#else /* ... !CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
+#else /* ... CONFIG_HAVE_LEGACY_PER_CPU_AREA */
 
 /* Number of blocks used and allocated. */
 static unsigned int pcpu_num_used, pcpu_num_allocated;
@@ -535,7 +535,7 @@ static int percpu_modinit(void)
 }
 __initcall(percpu_modinit);
 
-#endif /* CONFIG_HAVE_DYNAMIC_PER_CPU_AREA */
+#endif /* CONFIG_HAVE_LEGACY_PER_CPU_AREA */
 
 static unsigned int find_pcpusec(Elf_Ehdr *hdr,
                                 Elf_Shdr *sechdrs,
index d55a50d..fc3b974 100644 (file)
@@ -98,16 +98,16 @@ hw_perf_group_sched_in(struct perf_counter *group_leader,
 
 void __weak perf_counter_print_debug(void)     { }
 
-static DEFINE_PER_CPU(int, disable_count);
+static DEFINE_PER_CPU(int, perf_disable_count);
 
 void __perf_disable(void)
 {
-       __get_cpu_var(disable_count)++;
+       __get_cpu_var(perf_disable_count)++;
 }
 
 bool __perf_enable(void)
 {
-       return !--__get_cpu_var(disable_count);
+       return !--__get_cpu_var(perf_disable_count);
 }
 
 void perf_disable(void)
index 7c9098d..34fd81d 100644 (file)
@@ -318,12 +318,12 @@ struct task_group root_task_group;
 /* Default task group's sched entity on each cpu */
 static DEFINE_PER_CPU(struct sched_entity, init_sched_entity);
 /* Default task group's cfs_rq on each cpu */
-static DEFINE_PER_CPU(struct cfs_rq, init_cfs_rq) ____cacheline_aligned_in_smp;
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct cfs_rq, init_cfs_rq);
 #endif /* CONFIG_FAIR_GROUP_SCHED */
 
 #ifdef CONFIG_RT_GROUP_SCHED
 static DEFINE_PER_CPU(struct sched_rt_entity, init_sched_rt_entity);
-static DEFINE_PER_CPU(struct rt_rq, init_rt_rq) ____cacheline_aligned_in_smp;
+static DEFINE_PER_CPU_SHARED_ALIGNED(struct rt_rq, init_rt_rq);
 #endif /* CONFIG_RT_GROUP_SCHED */
 #else /* !CONFIG_USER_SCHED */
 #define root_task_group init_task_group
index 53c8fd3..8c193c2 100644 (file)
@@ -1334,7 +1334,7 @@ static __init void event_trace_self_tests(void)
 
 #ifdef CONFIG_FUNCTION_TRACER
 
-static DEFINE_PER_CPU(atomic_t, test_event_disable);
+static DEFINE_PER_CPU(atomic_t, ftrace_test_event_disable);
 
 static void
 function_test_events_call(unsigned long ip, unsigned long parent_ip)
@@ -1350,7 +1350,7 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip)
        pc = preempt_count();
        resched = ftrace_preempt_disable();
        cpu = raw_smp_processor_id();
-       disabled = atomic_inc_return(&per_cpu(test_event_disable, cpu));
+       disabled = atomic_inc_return(&per_cpu(ftrace_test_event_disable, cpu));
 
        if (disabled != 1)
                goto out;
@@ -1368,7 +1368,7 @@ function_test_events_call(unsigned long ip, unsigned long parent_ip)
        trace_nowake_buffer_unlock_commit(event, flags, pc);
 
  out:
-       atomic_dec(&per_cpu(test_event_disable, cpu));
+       atomic_dec(&per_cpu(ftrace_test_event_disable, cpu));
        ftrace_preempt_enable(resched);
 }
 
index 12327b2..43173c4 100644 (file)
@@ -790,6 +790,21 @@ config DEBUG_BLOCK_EXT_DEVT
 
          Say N if you are unsure.
 
+config DEBUG_FORCE_WEAK_PER_CPU
+       bool "Force weak per-cpu definitions"
+       depends on DEBUG_KERNEL
+       help
+         s390 and alpha require percpu variables in modules to be
+         defined weak to work around addressing range issue which
+         puts the following two restrictions on percpu variable
+         definitions.
+
+         1. percpu symbols must be unique whether static or not
+         2. percpu variables can't be defined inside a function
+
+         To ensure that generic code follows the above rules, this
+         option forces all percpu variables to be defined as weak.
+
 config LKDTM
        tristate "Linux Kernel Dump Test Tool Module"
        depends on DEBUG_KERNEL
index 5e0bd64..c77c648 100644 (file)
@@ -33,7 +33,7 @@ obj-$(CONFIG_FAILSLAB) += failslab.o
 obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o
 obj-$(CONFIG_FS_XIP) += filemap_xip.o
 obj-$(CONFIG_MIGRATION) += migrate.o
-ifdef CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
+ifndef CONFIG_HAVE_LEGACY_PER_CPU_AREA
 obj-$(CONFIG_SMP) += percpu.o
 else
 obj-$(CONFIG_SMP) += allocpercpu.o
index dfdee6a..df34cea 100644 (file)
@@ -5,6 +5,8 @@
  */
 #include <linux/mm.h>
 #include <linux/module.h>
+#include <linux/bootmem.h>
+#include <asm/sections.h>
 
 #ifndef cache_line_size
 #define cache_line_size()      L1_CACHE_BYTES
@@ -147,3 +149,29 @@ void free_percpu(void *__pdata)
        kfree(__percpu_disguise(__pdata));
 }
 EXPORT_SYMBOL_GPL(free_percpu);
+
+/*
+ * Generic percpu area setup.
+ */
+#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
+
+EXPORT_SYMBOL(__per_cpu_offset);
+
+void __init setup_per_cpu_areas(void)
+{
+       unsigned long size, i;
+       char *ptr;
+       unsigned long nr_possible_cpus = num_possible_cpus();
+
+       /* Copy section for each CPU (we discard the original) */
+       size = ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE);
+       ptr = alloc_bootmem_pages(size * nr_possible_cpus);
+
+       for_each_possible_cpu(i) {
+               __per_cpu_offset[i] = ptr - __per_cpu_start;
+               memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start);
+               ptr += size;
+       }
+}
+#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */
index d5292fc..177a516 100644 (file)
@@ -36,7 +36,7 @@ struct test_node {
 };
 
 static LIST_HEAD(test_list);
-static DEFINE_PER_CPU(void *, test_pointer);
+static DEFINE_PER_CPU(void *, kmemleak_test_pointer);
 
 /*
  * Some very simple testing. This function needs to be extended for
@@ -86,9 +86,9 @@ static int __init kmemleak_test_init(void)
        }
 
        for_each_possible_cpu(i) {
-               per_cpu(test_pointer, i) = kmalloc(129, GFP_KERNEL);
+               per_cpu(kmemleak_test_pointer, i) = kmalloc(129, GFP_KERNEL);
                pr_info("kmemleak: kmalloc(129) = %p\n",
-                       per_cpu(test_pointer, i));
+                       per_cpu(kmemleak_test_pointer, i));
        }
 
        return 0;
index 7687879..3c7f5e1 100644 (file)
@@ -610,6 +610,8 @@ void set_page_dirty_balance(struct page *page, int page_mkwrite)
        }
 }
 
+static DEFINE_PER_CPU(unsigned long, bdp_ratelimits) = 0;
+
 /**
  * balance_dirty_pages_ratelimited_nr - balance dirty memory state
  * @mapping: address_space which was dirtied
@@ -627,7 +629,6 @@ void set_page_dirty_balance(struct page *page, int page_mkwrite)
 void balance_dirty_pages_ratelimited_nr(struct address_space *mapping,
                                        unsigned long nr_pages_dirtied)
 {
-       static DEFINE_PER_CPU(unsigned long, ratelimits) = 0;
        unsigned long ratelimit;
        unsigned long *p;
 
@@ -640,7 +641,7 @@ void balance_dirty_pages_ratelimited_nr(struct address_space *mapping,
         * tasks in balance_dirty_pages(). Period.
         */
        preempt_disable();
-       p =  &__get_cpu_var(ratelimits);
+       p =  &__get_cpu_var(bdp_ratelimits);
        *p += nr_pages_dirtied;
        if (unlikely(*p >= ratelimit)) {
                *p = 0;
index b70f2ac..b149845 100644 (file)
@@ -43,7 +43,7 @@
  *
  * To use this allocator, arch code should do the followings.
  *
- * - define CONFIG_HAVE_DYNAMIC_PER_CPU_AREA
+ * - drop CONFIG_HAVE_LEGACY_PER_CPU_AREA
  *
  * - define __addr_to_pcpu_ptr() and __pcpu_ptr_to_addr() to translate
  *   regular address to percpu pointer and back if they need to be
@@ -1275,3 +1275,41 @@ ssize_t __init pcpu_embed_first_chunk(size_t static_size, size_t reserved_size,
                                      reserved_size, dyn_size,
                                      pcpue_unit_size, pcpue_ptr, NULL);
 }
+
+/*
+ * Generic percpu area setup.
+ *
+ * The embedding helper is used because its behavior closely resembles
+ * the original non-dynamic generic percpu area setup.  This is
+ * important because many archs have addressing restrictions and might
+ * fail if the percpu area is located far away from the previous
+ * location.  As an added bonus, in non-NUMA cases, embedding is
+ * generally a good idea TLB-wise because percpu area can piggy back
+ * on the physical linear memory mapping which uses large page
+ * mappings on applicable archs.
+ */
+#ifndef CONFIG_HAVE_SETUP_PER_CPU_AREA
+unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
+EXPORT_SYMBOL(__per_cpu_offset);
+
+void __init setup_per_cpu_areas(void)
+{
+       size_t static_size = __per_cpu_end - __per_cpu_start;
+       ssize_t unit_size;
+       unsigned long delta;
+       unsigned int cpu;
+
+       /*
+        * Always reserve area for module percpu variables.  That's
+        * what the legacy allocator did.
+        */
+       unit_size = pcpu_embed_first_chunk(static_size, PERCPU_MODULE_RESERVE,
+                                          PERCPU_DYNAMIC_RESERVE, -1);
+       if (unit_size < 0)
+               panic("Failed to initialized percpu areas.");
+
+       delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
+       for_each_possible_cpu(cpu)
+               __per_cpu_offset[cpu] = delta + cpu * unit_size;
+}
+#endif /* CONFIG_HAVE_SETUP_PER_CPU_AREA */
index e66d07d..6eedf7e 100644 (file)
@@ -19,7 +19,7 @@
 #include <linux/module.h>
 #include <linux/quicklist.h>
 
-DEFINE_PER_CPU(struct quicklist, quicklist)[CONFIG_NR_QUICK];
+DEFINE_PER_CPU(struct quicklist [CONFIG_NR_QUICK], quicklist);
 
 #define FRACTION_OF_NODE_MEM   16
 
index 819f056..ffc895c 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2092,8 +2092,8 @@ init_kmem_cache_node(struct kmem_cache_node *n, struct kmem_cache *s)
  */
 #define NR_KMEM_CACHE_CPU 100
 
-static DEFINE_PER_CPU(struct kmem_cache_cpu,
-                               kmem_cache_cpu)[NR_KMEM_CACHE_CPU];
+static DEFINE_PER_CPU(struct kmem_cache_cpu [NR_KMEM_CACHE_CPU],
+                     kmem_cache_cpu);
 
 static DEFINE_PER_CPU(struct kmem_cache_cpu *, kmem_cache_cpu_free);
 static DECLARE_BITMAP(kmem_cach_cpu_free_init_once, CONFIG_NR_CPUS);
index cd2b97f..a6e0e07 100644 (file)
@@ -37,12 +37,13 @@ __initcall(init_syncookies);
 #define COOKIEBITS 24  /* Upper bits store count */
 #define COOKIEMASK (((__u32)1 << COOKIEBITS) - 1)
 
-static DEFINE_PER_CPU(__u32, cookie_scratch)[16 + 5 + SHA_WORKSPACE_WORDS];
+static DEFINE_PER_CPU(__u32 [16 + 5 + SHA_WORKSPACE_WORDS],
+                     ipv4_cookie_scratch);
 
 static u32 cookie_hash(__be32 saddr, __be32 daddr, __be16 sport, __be16 dport,
                       u32 count, int c)
 {
-       __u32 *tmp = __get_cpu_var(cookie_scratch);
+       __u32 *tmp = __get_cpu_var(ipv4_cookie_scratch);
 
        memcpy(tmp + 4, syncookie_secret[c], sizeof(syncookie_secret[c]));
        tmp[0] = (__force u32)saddr;
index 8c25139..6b6ae91 100644 (file)
@@ -74,12 +74,13 @@ static inline struct sock *get_cookie_sock(struct sock *sk, struct sk_buff *skb,
        return child;
 }
 
-static DEFINE_PER_CPU(__u32, cookie_scratch)[16 + 5 + SHA_WORKSPACE_WORDS];
+static DEFINE_PER_CPU(__u32 [16 + 5 + SHA_WORKSPACE_WORDS],
+                     ipv6_cookie_scratch);
 
 static u32 cookie_hash(struct in6_addr *saddr, struct in6_addr *daddr,
                       __be16 sport, __be16 dport, u32 count, int c)
 {
-       __u32 *tmp = __get_cpu_var(cookie_scratch);
+       __u32 *tmp = __get_cpu_var(ipv6_cookie_scratch);
 
        /*
         * we have 320 bits of information to hash, copy in the remaining
index 02e3e3d..301ae51 100644 (file)
@@ -37,7 +37,7 @@
 #include "rds.h"
 #include "ib.h"
 
-DEFINE_PER_CPU(struct rds_ib_statistics, rds_ib_stats) ____cacheline_aligned;
+DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_ib_statistics, rds_ib_stats);
 
 static char *rds_ib_stat_names[] = {
        "ib_connect_raced",
index ccc7e8f..fafea3c 100644 (file)
@@ -37,7 +37,7 @@
 #include "rds.h"
 #include "iw.h"
 
-DEFINE_PER_CPU(struct rds_iw_statistics, rds_iw_stats) ____cacheline_aligned;
+DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_iw_statistics, rds_iw_stats);
 
 static char *rds_iw_stat_names[] = {
        "iw_connect_raced",
index c460743..de7bb84 100644 (file)
@@ -39,7 +39,7 @@ struct rds_page_remainder {
        unsigned long   r_offset;
 };
 
-DEFINE_PER_CPU(struct rds_page_remainder, rds_page_remainders) ____cacheline_aligned;
+DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_page_remainder, rds_page_remainders);
 
 /*
  * returns 0 on success or -errno on failure.
diff --git a/scripts/module-common.lds b/scripts/module-common.lds
new file mode 100644 (file)
index 0000000..47a1f9a
--- /dev/null
@@ -0,0 +1,8 @@
+/*
+ * Common module linker script, always used when linking a module.
+ * Archs are free to supply their own linker scripts.  ld will
+ * combine them automatically.
+ */
+SECTIONS {
+       /DISCARD/ : { *(.discard) }
+}