[S390] rework smp code
[linux-2.6.git] / arch / s390 / kernel / smp.c
index 2398ce6..6db8526 100644 (file)
@@ -1,23 +1,18 @@
 /*
- *  arch/s390/kernel/smp.c
+ *  SMP related functions
  *
- *    Copyright IBM Corp. 1999, 2009
- *    Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
- *              Martin Schwidefsky (schwidefsky@de.ibm.com)
- *              Heiko Carstens (heiko.carstens@de.ibm.com)
+ *    Copyright IBM Corp. 1999,2012
+ *    Author(s): Denis Joseph Barrow,
+ *              Martin Schwidefsky <schwidefsky@de.ibm.com>,
+ *              Heiko Carstens <heiko.carstens@de.ibm.com>,
  *
  *  based on other smp stuff by
  *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
  *    (c) 1998 Ingo Molnar
  *
- * We work with logical cpu numbering everywhere we can. The only
- * functions using the real cpu address (got from STAP) are the sigp
- * functions. For all other functions we use the identity mapping.
- * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
- * used e.g. to find the idle task belonging to a logical cpu. Every array
- * in the kernel is sorted by the logical cpu number and not by the physical
- * one which is causing all the confusion with __cpu_logical_map and
- * cpu_number_map in other architectures.
+ * The code outside of smp.c uses logical cpu numbers, only smp.c does
+ * the translation of logical to physical cpu ids. All new code that
+ * operates on physical cpu numbers needs to go into smp.c.
  */
 
 #define KMSG_COMPONENT "cpu"
 #include <linux/spinlock.h>
 #include <linux/kernel_stat.h>
 #include <linux/delay.h>
-#include <linux/cache.h>
 #include <linux/interrupt.h>
 #include <linux/irqflags.h>
 #include <linux/cpu.h>
-#include <linux/timex.h>
-#include <linux/bootmem.h>
 #include <linux/slab.h>
 #include <linux/crash_dump.h>
 #include <asm/asm-offsets.h>
 #include <asm/ipl.h>
 #include <asm/setup.h>
-#include <asm/sigp.h>
-#include <asm/pgalloc.h>
 #include <asm/irq.h>
-#include <asm/cpcmd.h>
 #include <asm/tlbflush.h>
 #include <asm/timer.h>
 #include <asm/lowcore.h>
 #include <asm/sclp.h>
-#include <asm/cputime.h>
 #include <asm/vdso.h>
-#include <asm/cpu.h>
 #include "entry.h"
 
-/* logical cpu to cpu address */
-unsigned short __cpu_logical_map[NR_CPUS];
+enum {
+       sigp_sense = 1,
+       sigp_external_call = 2,
+       sigp_emergency_signal = 3,
+       sigp_start = 4,
+       sigp_stop = 5,
+       sigp_restart = 6,
+       sigp_stop_and_store_status = 9,
+       sigp_initial_cpu_reset = 11,
+       sigp_cpu_reset = 12,
+       sigp_set_prefix = 13,
+       sigp_store_status_at_address = 14,
+       sigp_store_extended_status_at_address = 15,
+       sigp_set_architecture = 18,
+       sigp_conditional_emergency_signal = 19,
+       sigp_sense_running = 21,
+};
 
-static struct task_struct *current_set[NR_CPUS];
+enum {
+       sigp_order_code_accepted = 0,
+       sigp_status_stored = 1,
+       sigp_busy = 2,
+       sigp_not_operational = 3,
+};
 
-static u8 smp_cpu_type;
-static int smp_use_sigp_detection;
+enum {
+       ec_schedule = 0,
+       ec_call_function,
+       ec_call_function_single,
+       ec_stop_cpu,
+};
 
-enum s390_cpu_state {
+enum {
        CPU_STATE_STANDBY,
        CPU_STATE_CONFIGURED,
 };
 
+struct pcpu {
+       struct cpu cpu;
+       struct task_struct *idle;       /* idle process for the cpu */
+       struct _lowcore *lowcore;       /* lowcore page(s) for the cpu */
+       unsigned long async_stack;      /* async stack for the cpu */
+       unsigned long panic_stack;      /* panic stack for the cpu */
+       unsigned long ec_mask;          /* bit mask for ec_xxx functions */
+       int state;                      /* physical cpu state */
+       u32 status;                     /* last status received via sigp */
+       u16 address;                    /* physical cpu address */
+};
+
+static u8 boot_cpu_type;
+static u16 boot_cpu_address;
+static struct pcpu pcpu_devices[NR_CPUS];
+
 DEFINE_MUTEX(smp_cpu_state_mutex);
-static int smp_cpu_state[NR_CPUS];
 
-static DEFINE_PER_CPU(struct cpu, cpu_devices);
+/*
+ * Signal processor helper functions.
+ */
+static inline int __pcpu_sigp(u16 addr, u8 order, u32 parm, u32 *status)
+{
+       register unsigned int reg1 asm ("1") = parm;
+       int cc;
 
-static void smp_ext_bitcall(int, int);
+       asm volatile(
+               "       sigp    %1,%2,0(%3)\n"
+               "       ipm     %0\n"
+               "       srl     %0,28\n"
+               : "=d" (cc), "+d" (reg1) : "d" (addr), "a" (order) : "cc");
+       if (status && cc == 1)
+               *status = reg1;
+       return cc;
+}
 
-static int raw_cpu_stopped(int cpu)
+static inline int __pcpu_sigp_relax(u16 addr, u8 order, u32 parm, u32 *status)
 {
-       u32 status;
+       int cc;
 
-       switch (raw_sigp_ps(&status, 0, cpu, sigp_sense)) {
-       case sigp_status_stored:
-               /* Check for stopped and check stop state */
-               if (status & 0x50)
-                       return 1;
-               break;
-       default:
-               break;
+       while (1) {
+               cc = __pcpu_sigp(addr, order, parm, status);
+               if (cc != sigp_busy)
+                       return cc;
+               cpu_relax();
        }
-       return 0;
 }
 
-static inline int cpu_stopped(int cpu)
+static int pcpu_sigp_retry(struct pcpu *pcpu, u8 order, u32 parm)
 {
-       return raw_cpu_stopped(cpu_logical_map(cpu));
+       int cc, retry;
+
+       for (retry = 0; ; retry++) {
+               cc = __pcpu_sigp(pcpu->address, order, parm, &pcpu->status);
+               if (cc != sigp_busy)
+                       break;
+               if (retry >= 3)
+                       udelay(10);
+       }
+       return cc;
+}
+
+static inline int pcpu_stopped(struct pcpu *pcpu)
+{
+       if (__pcpu_sigp(pcpu->address, sigp_sense,
+                       0, &pcpu->status) != sigp_status_stored)
+               return 0;
+       /* Check for stopped and check stop state */
+       return !!(pcpu->status & 0x50);
+}
+
+static inline int pcpu_running(struct pcpu *pcpu)
+{
+       if (__pcpu_sigp(pcpu->address, sigp_sense_running,
+                       0, &pcpu->status) != sigp_status_stored)
+               return 1;
+       /* Check for running status */
+       return !(pcpu->status & 0x400);
 }
 
 /*
- * Ensure that PSW restart is done on an online CPU
+ * Find struct pcpu by cpu address.
  */
-void smp_restart_with_online_cpu(void)
+static struct pcpu *pcpu_find_address(const struct cpumask *mask, int address)
 {
        int cpu;
 
-       for_each_online_cpu(cpu) {
-               if (stap() == __cpu_logical_map[cpu]) {
-                       /* We are online: Enable DAT again and return */
-                       __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
-                       return;
-               }
+       for_each_cpu(cpu, mask)
+               if (pcpu_devices[cpu].address == address)
+                       return pcpu_devices + cpu;
+       return NULL;
+}
+
+static void pcpu_ec_call(struct pcpu *pcpu, int ec_bit)
+{
+       int order;
+
+       set_bit(ec_bit, &pcpu->ec_mask);
+       order = pcpu_running(pcpu) ?
+               sigp_external_call : sigp_emergency_signal;
+       pcpu_sigp_retry(pcpu, order, 0);
+}
+
+static int __cpuinit pcpu_alloc_lowcore(struct pcpu *pcpu, int cpu)
+{
+       struct _lowcore *lc;
+
+       if (pcpu != &pcpu_devices[0]) {
+               pcpu->lowcore = (struct _lowcore *)
+                       __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
+               pcpu->async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
+               pcpu->panic_stack = __get_free_page(GFP_KERNEL);
+               if (!pcpu->lowcore || !pcpu->panic_stack || !pcpu->async_stack)
+                       goto out;
        }
-       /* We are not online: Do PSW restart on an online CPU */
-       while (sigp(cpu, sigp_restart) == sigp_busy)
-               cpu_relax();
-       /* And stop ourself */
-       while (raw_sigp(stap(), sigp_stop) == sigp_busy)
-               cpu_relax();
-       for (;;);
+       lc = pcpu->lowcore;
+       memcpy(lc, &S390_lowcore, 512);
+       memset((char *) lc + 512, 0, sizeof(*lc) - 512);
+       lc->async_stack = pcpu->async_stack + ASYNC_SIZE;
+       lc->panic_stack = pcpu->panic_stack + PAGE_SIZE;
+       lc->cpu_nr = cpu;
+#ifndef CONFIG_64BIT
+       if (MACHINE_HAS_IEEE) {
+               lc->extended_save_area_addr = get_zeroed_page(GFP_KERNEL);
+               if (!lc->extended_save_area_addr)
+                       goto out;
+       }
+#else
+       if (vdso_alloc_per_cpu(lc))
+               goto out;
+#endif
+       lowcore_ptr[cpu] = lc;
+       pcpu_sigp_retry(pcpu, sigp_set_prefix, (u32)(unsigned long) lc);
+       return 0;
+out:
+       if (pcpu != &pcpu_devices[0]) {
+               free_page(pcpu->panic_stack);
+               free_pages(pcpu->async_stack, ASYNC_ORDER);
+               free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
+       }
+       return -ENOMEM;
 }
 
-void smp_switch_to_ipl_cpu(void (*func)(void *), void *data)
+static void pcpu_free_lowcore(struct pcpu *pcpu)
 {
-       struct _lowcore *lc, *current_lc;
-       struct stack_frame *sf;
-       struct pt_regs *regs;
-       unsigned long sp;
-
-       if (smp_processor_id() == 0)
-               func(data);
-       __load_psw_mask(PSW_DEFAULT_KEY | PSW_MASK_BASE |
-                       PSW_MASK_EA | PSW_MASK_BA);
-       /* Disable lowcore protection */
-       __ctl_clear_bit(0, 28);
-       current_lc = lowcore_ptr[smp_processor_id()];
-       lc = lowcore_ptr[0];
-       if (!lc)
-               lc = current_lc;
-       lc->restart_psw.mask =
-               PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
-       lc->restart_psw.addr = PSW_ADDR_AMODE | (unsigned long) smp_restart_cpu;
-       if (!cpu_online(0))
-               smp_switch_to_cpu(func, data, 0, stap(), __cpu_logical_map[0]);
-       while (sigp(0, sigp_stop_and_store_status) == sigp_busy)
-               cpu_relax();
-       sp = lc->panic_stack;
-       sp -= sizeof(struct pt_regs);
-       regs = (struct pt_regs *) sp;
-       memcpy(&regs->gprs, &current_lc->gpregs_save_area, sizeof(regs->gprs));
-       regs->psw = current_lc->psw_save_area;
-       sp -= STACK_FRAME_OVERHEAD;
-       sf = (struct stack_frame *) sp;
-       sf->back_chain = 0;
-       smp_switch_to_cpu(func, data, sp, stap(), __cpu_logical_map[0]);
+       pcpu_sigp_retry(pcpu, sigp_set_prefix, 0);
+       lowcore_ptr[pcpu - pcpu_devices] = NULL;
+#ifndef CONFIG_64BIT
+       if (MACHINE_HAS_IEEE) {
+               struct _lowcore *lc = pcpu->lowcore;
+
+               free_page((unsigned long) lc->extended_save_area_addr);
+               lc->extended_save_area_addr = 0;
+       }
+#else
+       vdso_free_per_cpu(pcpu->lowcore);
+#endif
+       if (pcpu != &pcpu_devices[0]) {
+               free_page(pcpu->panic_stack);
+               free_pages(pcpu->async_stack, ASYNC_ORDER);
+               free_pages((unsigned long) pcpu->lowcore, LC_ORDER);
+       }
+}
+
+static void pcpu_prepare_secondary(struct pcpu *pcpu, int cpu)
+{
+       struct _lowcore *lc = pcpu->lowcore;
+
+       atomic_inc(&init_mm.context.attach_count);
+       lc->cpu_nr = cpu;
+       lc->percpu_offset = __per_cpu_offset[cpu];
+       lc->kernel_asce = S390_lowcore.kernel_asce;
+       lc->machine_flags = S390_lowcore.machine_flags;
+       lc->ftrace_func = S390_lowcore.ftrace_func;
+       lc->user_timer = lc->system_timer = lc->steal_timer = 0;
+       __ctl_store(lc->cregs_save_area, 0, 15);
+       save_access_regs((unsigned int *) lc->access_regs_save_area);
+       memcpy(lc->stfle_fac_list, S390_lowcore.stfle_fac_list,
+              MAX_FACILITY_BIT/8);
+}
+
+static void pcpu_attach_task(struct pcpu *pcpu, struct task_struct *tsk)
+{
+       struct _lowcore *lc = pcpu->lowcore;
+       struct thread_info *ti = task_thread_info(tsk);
+
+       lc->kernel_stack = (unsigned long) task_stack_page(tsk) + THREAD_SIZE;
+       lc->thread_info = (unsigned long) task_thread_info(tsk);
+       lc->current_task = (unsigned long) tsk;
+       lc->user_timer = ti->user_timer;
+       lc->system_timer = ti->system_timer;
+       lc->steal_timer = 0;
+}
+
+static void pcpu_start_fn(struct pcpu *pcpu, void (*func)(void *), void *data)
+{
+       struct _lowcore *lc = pcpu->lowcore;
+
+       lc->restart_stack = lc->kernel_stack;
+       lc->restart_fn = (unsigned long) func;
+       lc->restart_data = (unsigned long) data;
+       lc->restart_source = -1UL;
+       pcpu_sigp_retry(pcpu, sigp_restart, 0);
+}
+
+/*
+ * Call function via PSW restart on pcpu and stop the current cpu.
+ */
+static void pcpu_delegate(struct pcpu *pcpu, void (*func)(void *),
+                         void *data, unsigned long stack)
+{
+       struct _lowcore *lc = pcpu->lowcore;
+       unsigned short this_cpu;
+
+       __load_psw_mask(psw_kernel_bits);
+       this_cpu = stap();
+       if (pcpu->address == this_cpu)
+               func(data);     /* should not return */
+       /* Stop target cpu (if func returns this stops the current cpu). */
+       pcpu_sigp_retry(pcpu, sigp_stop, 0);
+       /* Restart func on the target cpu and stop the current cpu. */
+       lc->restart_stack = stack;
+       lc->restart_fn = (unsigned long) func;
+       lc->restart_data = (unsigned long) data;
+       lc->restart_source = (unsigned long) this_cpu;
+       asm volatile(
+               "0:     sigp    0,%0,6  # sigp restart to target cpu\n"
+               "       brc     2,0b    # busy, try again\n"
+               "1:     sigp    0,%1,5  # sigp stop to current cpu\n"
+               "       brc     2,1b    # busy, try again\n"
+               : : "d" (pcpu->address), "d" (this_cpu) : "0", "1", "cc");
+       for (;;) ;
+}
+
+/*
+ * Call function on an online CPU.
+ */
+void smp_call_online_cpu(void (*func)(void *), void *data)
+{
+       struct pcpu *pcpu;
+
+       /* Use the current cpu if it is online. */
+       pcpu = pcpu_find_address(cpu_online_mask, stap());
+       if (!pcpu)
+               /* Use the first online cpu. */
+               pcpu = pcpu_devices + cpumask_first(cpu_online_mask);
+       pcpu_delegate(pcpu, func, data, (unsigned long) restart_stack);
+}
+
+/*
+ * Call function on the ipl CPU.
+ */
+void smp_call_ipl_cpu(void (*func)(void *), void *data)
+{
+       pcpu_delegate(&pcpu_devices[0], func, data, pcpu_devices->panic_stack);
+}
+
+int smp_find_processor_id(u16 address)
+{
+       int cpu;
+
+       for_each_present_cpu(cpu)
+               if (pcpu_devices[cpu].address == address)
+                       return cpu;
+       return -1;
 }
 
-static void smp_stop_cpu(void)
+int smp_vcpu_scheduled(int cpu)
 {
-       while (sigp(smp_processor_id(), sigp_stop) == sigp_busy)
+       return pcpu_running(pcpu_devices + cpu);
+}
+
+void smp_yield(void)
+{
+       if (MACHINE_HAS_DIAG44)
+               asm volatile("diag 0,0,0x44");
+}
+
+void smp_yield_cpu(int cpu)
+{
+       if (MACHINE_HAS_DIAG9C)
+               asm volatile("diag %0,0,0x9c"
+                            : : "d" (pcpu_devices[cpu].address));
+       else if (MACHINE_HAS_DIAG44)
+               asm volatile("diag 0,0,0x44");
+}
+
+/*
+ * Send cpus emergency shutdown signal. This gives the cpus the
+ * opportunity to complete outstanding interrupts.
+ */
+void smp_emergency_stop(cpumask_t *cpumask)
+{
+       u64 end;
+       int cpu;
+
+       end = get_clock() + (1000000UL << 12);
+       for_each_cpu(cpu, cpumask) {
+               struct pcpu *pcpu = pcpu_devices + cpu;
+               set_bit(ec_stop_cpu, &pcpu->ec_mask);
+               while (__pcpu_sigp(pcpu->address, sigp_emergency_signal,
+                                  0, NULL) == sigp_busy &&
+                      get_clock() < end)
+                       cpu_relax();
+       }
+       while (get_clock() < end) {
+               for_each_cpu(cpu, cpumask)
+                       if (pcpu_stopped(pcpu_devices + cpu))
+                               cpumask_clear_cpu(cpu, cpumask);
+               if (cpumask_empty(cpumask))
+                       break;
                cpu_relax();
+       }
 }
 
+/*
+ * Stop all cpus but the current one.
+ */
 void smp_send_stop(void)
 {
        cpumask_t cpumask;
        int cpu;
-       u64 end;
 
        /* Disable all interrupts/machine checks */
        __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
@@ -173,56 +409,46 @@ void smp_send_stop(void)
        cpumask_copy(&cpumask, cpu_online_mask);
        cpumask_clear_cpu(smp_processor_id(), &cpumask);
 
-       if (oops_in_progress) {
-               /*
-                * Give the other cpus the opportunity to complete
-                * outstanding interrupts before stopping them.
-                */
-               end = get_clock() + (1000000UL << 12);
-               for_each_cpu(cpu, &cpumask) {
-                       set_bit(ec_stop_cpu, (unsigned long *)
-                               &lowcore_ptr[cpu]->ext_call_fast);
-                       while (sigp(cpu, sigp_emergency_signal) == sigp_busy &&
-                              get_clock() < end)
-                               cpu_relax();
-               }
-               while (get_clock() < end) {
-                       for_each_cpu(cpu, &cpumask)
-                               if (cpu_stopped(cpu))
-                                       cpumask_clear_cpu(cpu, &cpumask);
-                       if (cpumask_empty(&cpumask))
-                               break;
-                       cpu_relax();
-               }
-       }
+       if (oops_in_progress)
+               smp_emergency_stop(&cpumask);
 
        /* stop all processors */
        for_each_cpu(cpu, &cpumask) {
-               while (sigp(cpu, sigp_stop) == sigp_busy)
-                       cpu_relax();
-               while (!cpu_stopped(cpu))
+               struct pcpu *pcpu = pcpu_devices + cpu;
+               pcpu_sigp_retry(pcpu, sigp_stop, 0);
+               while (!pcpu_stopped(pcpu))
                        cpu_relax();
        }
 }
 
 /*
+ * Stop the current cpu.
+ */
+void smp_stop_cpu(void)
+{
+       pcpu_sigp_retry(pcpu_devices + smp_processor_id(), sigp_stop, 0);
+       for (;;) ;
+}
+
+/*
  * This is the main routine where commands issued by other
  * cpus are handled.
  */
-
 static void do_ext_call_interrupt(unsigned int ext_int_code,
                                  unsigned int param32, unsigned long param64)
 {
        unsigned long bits;
+       int cpu;
 
+       cpu = smp_processor_id();
        if ((ext_int_code & 0xffff) == 0x1202)
-               kstat_cpu(smp_processor_id()).irqs[EXTINT_EXC]++;
+               kstat_cpu(cpu).irqs[EXTINT_EXC]++;
        else
-               kstat_cpu(smp_processor_id()).irqs[EXTINT_EMS]++;
+               kstat_cpu(cpu).irqs[EXTINT_EMS]++;
        /*
         * handle bit signal external calls
         */
-       bits = xchg(&S390_lowcore.ext_call_fast, 0);
+       bits = xchg(&pcpu_devices[cpu].ec_mask, 0);
 
        if (test_bit(ec_stop_cpu, &bits))
                smp_stop_cpu();
@@ -238,38 +464,17 @@ static void do_ext_call_interrupt(unsigned int ext_int_code,
 
 }
 
-/*
- * Send an external call sigp to another cpu and return without waiting
- * for its completion.
- */
-static void smp_ext_bitcall(int cpu, int sig)
-{
-       int order;
-
-       /*
-        * Set signaling bit in lowcore of target cpu and kick it
-        */
-       set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
-       while (1) {
-               order = smp_vcpu_scheduled(cpu) ?
-                       sigp_external_call : sigp_emergency_signal;
-               if (sigp(cpu, order) != sigp_busy)
-                       break;
-               udelay(10);
-       }
-}
-
 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 {
        int cpu;
 
        for_each_cpu(cpu, mask)
-               smp_ext_bitcall(cpu, ec_call_function);
+               pcpu_ec_call(pcpu_devices + cpu, ec_call_function);
 }
 
 void arch_send_call_function_single_ipi(int cpu)
 {
-       smp_ext_bitcall(cpu, ec_call_function_single);
+       pcpu_ec_call(pcpu_devices + cpu, ec_call_function_single);
 }
 
 #ifndef CONFIG_64BIT
@@ -295,15 +500,16 @@ EXPORT_SYMBOL(smp_ptlb_all);
  */
 void smp_send_reschedule(int cpu)
 {
-       smp_ext_bitcall(cpu, ec_schedule);
+       pcpu_ec_call(pcpu_devices + cpu, ec_schedule);
 }
 
 /*
  * parameter area for the set/clear control bit callbacks
  */
 struct ec_creg_mask_parms {
-       unsigned long orvals[16];
-       unsigned long andvals[16];
+       unsigned long orval;
+       unsigned long andval;
+       int cr;
 };
 
 /*
@@ -313,11 +519,9 @@ static void smp_ctl_bit_callback(void *info)
 {
        struct ec_creg_mask_parms *pp = info;
        unsigned long cregs[16];
-       int i;
 
        __ctl_store(cregs, 0, 15);
-       for (i = 0; i <= 15; i++)
-               cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
+       cregs[pp->cr] = (cregs[pp->cr] & pp->andval) | pp->orval;
        __ctl_load(cregs, 0, 15);
 }
 
@@ -326,11 +530,8 @@ static void smp_ctl_bit_callback(void *info)
  */
 void smp_ctl_set_bit(int cr, int bit)
 {
-       struct ec_creg_mask_parms parms;
+       struct ec_creg_mask_parms parms = { 1UL << bit, -1UL, cr };
 
-       memset(&parms.orvals, 0, sizeof(parms.orvals));
-       memset(&parms.andvals, 0xff, sizeof(parms.andvals));
-       parms.orvals[cr] = 1UL << bit;
        on_each_cpu(smp_ctl_bit_callback, &parms, 1);
 }
 EXPORT_SYMBOL(smp_ctl_set_bit);
@@ -340,216 +541,175 @@ EXPORT_SYMBOL(smp_ctl_set_bit);
  */
 void smp_ctl_clear_bit(int cr, int bit)
 {
-       struct ec_creg_mask_parms parms;
+       struct ec_creg_mask_parms parms = { 0, ~(1UL << bit), cr };
 
-       memset(&parms.orvals, 0, sizeof(parms.orvals));
-       memset(&parms.andvals, 0xff, sizeof(parms.andvals));
-       parms.andvals[cr] = ~(1UL << bit);
        on_each_cpu(smp_ctl_bit_callback, &parms, 1);
 }
 EXPORT_SYMBOL(smp_ctl_clear_bit);
 
 #if defined(CONFIG_ZFCPDUMP) || defined(CONFIG_CRASH_DUMP)
 
-static void __init smp_get_save_area(unsigned int cpu, unsigned int phy_cpu)
+struct save_area *zfcpdump_save_areas[NR_CPUS + 1];
+EXPORT_SYMBOL_GPL(zfcpdump_save_areas);
+
+static void __init smp_get_save_area(int cpu, u16 address)
 {
-       if (ipl_info.type != IPL_TYPE_FCP_DUMP && !OLDMEM_BASE)
-               return;
+       void *lc = pcpu_devices[0].lowcore;
+       struct save_area *save_area;
+
        if (is_kdump_kernel())
                return;
+       if (!OLDMEM_BASE && (address == boot_cpu_address ||
+                            ipl_info.type != IPL_TYPE_FCP_DUMP))
+               return;
        if (cpu >= NR_CPUS) {
-               pr_warning("CPU %i exceeds the maximum %i and is excluded from "
-                          "the dump\n", cpu, NR_CPUS - 1);
+               pr_warning("CPU %i exceeds the maximum %i and is excluded "
+                          "from the dump\n", cpu, NR_CPUS - 1);
                return;
        }
-       zfcpdump_save_areas[cpu] = kmalloc(sizeof(struct save_area), GFP_KERNEL);
-       while (raw_sigp(phy_cpu, sigp_stop_and_store_status) == sigp_busy)
-               cpu_relax();
-       memcpy_real(zfcpdump_save_areas[cpu],
-                   (void *)(unsigned long) store_prefix() + SAVE_AREA_BASE,
-                   sizeof(struct save_area));
+       save_area = kmalloc(sizeof(struct save_area), GFP_KERNEL);
+       if (!save_area)
+               panic("could not allocate memory for save area\n");
+       zfcpdump_save_areas[cpu] = save_area;
+#ifdef CONFIG_CRASH_DUMP
+       if (address == boot_cpu_address) {
+               /* Copy the registers of the boot cpu. */
+               copy_oldmem_page(1, (void *) save_area, sizeof(*save_area),
+                                SAVE_AREA_BASE - PAGE_SIZE, 0);
+               return;
+       }
+#endif
+       /* Get the registers of a non-boot cpu. */
+       __pcpu_sigp_relax(address, sigp_stop_and_store_status, 0, NULL);
+       memcpy_real(save_area, lc + SAVE_AREA_BASE, sizeof(*save_area));
 }
 
-struct save_area *zfcpdump_save_areas[NR_CPUS + 1];
-EXPORT_SYMBOL_GPL(zfcpdump_save_areas);
-
-#else
-
-static inline void smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) { }
-
-#endif /* CONFIG_ZFCPDUMP */
-
-static int cpu_known(int cpu_id)
+int smp_store_status(int cpu)
 {
-       int cpu;
+       struct pcpu *pcpu;
 
-       for_each_present_cpu(cpu) {
-               if (__cpu_logical_map[cpu] == cpu_id)
-                       return 1;
-       }
+       pcpu = pcpu_devices + cpu;
+       if (__pcpu_sigp_relax(pcpu->address, sigp_stop_and_store_status,
+                             0, NULL) != sigp_order_code_accepted)
+               return -EIO;
        return 0;
 }
 
-static int smp_rescan_cpus_sigp(cpumask_t avail)
-{
-       int cpu_id, logical_cpu;
+#else /* CONFIG_ZFCPDUMP || CONFIG_CRASH_DUMP */
 
-       logical_cpu = cpumask_first(&avail);
-       if (logical_cpu >= nr_cpu_ids)
-               return 0;
-       for (cpu_id = 0; cpu_id <= MAX_CPU_ADDRESS; cpu_id++) {
-               if (cpu_known(cpu_id))
-                       continue;
-               __cpu_logical_map[logical_cpu] = cpu_id;
-               cpu_set_polarization(logical_cpu, POLARIZATION_UNKNOWN);
-               if (!cpu_stopped(logical_cpu))
-                       continue;
-               set_cpu_present(logical_cpu, true);
-               smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
-               logical_cpu = cpumask_next(logical_cpu, &avail);
-               if (logical_cpu >= nr_cpu_ids)
-                       break;
-       }
-       return 0;
-}
+static inline void smp_get_save_area(int cpu, u16 address) { }
+
+#endif /* CONFIG_ZFCPDUMP || CONFIG_CRASH_DUMP */
 
-static int smp_rescan_cpus_sclp(cpumask_t avail)
+static struct sclp_cpu_info *smp_get_cpu_info(void)
 {
+       static int use_sigp_detection;
        struct sclp_cpu_info *info;
-       int cpu_id, logical_cpu, cpu;
-       int rc;
-
-       logical_cpu = cpumask_first(&avail);
-       if (logical_cpu >= nr_cpu_ids)
-               return 0;
-       info = kmalloc(sizeof(*info), GFP_KERNEL);
-       if (!info)
-               return -ENOMEM;
-       rc = sclp_get_cpu_info(info);
-       if (rc)
-               goto out;
-       for (cpu = 0; cpu < info->combined; cpu++) {
-               if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
-                       continue;
-               cpu_id = info->cpu[cpu].address;
-               if (cpu_known(cpu_id))
-                       continue;
-               __cpu_logical_map[logical_cpu] = cpu_id;
-               cpu_set_polarization(logical_cpu, POLARIZATION_UNKNOWN);
-               set_cpu_present(logical_cpu, true);
-               if (cpu >= info->configured)
-                       smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY;
-               else
-                       smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
-               logical_cpu = cpumask_next(logical_cpu, &avail);
-               if (logical_cpu >= nr_cpu_ids)
-                       break;
+       int address;
+
+       info = kzalloc(sizeof(*info), GFP_KERNEL);
+       if (info && (use_sigp_detection || sclp_get_cpu_info(info))) {
+               use_sigp_detection = 1;
+               for (address = 0; address <= MAX_CPU_ADDRESS; address++) {
+                       if (__pcpu_sigp_relax(address, sigp_sense, 0, NULL) ==
+                           sigp_not_operational)
+                               continue;
+                       info->cpu[info->configured].address = address;
+                       info->configured++;
+               }
+               info->combined = info->configured;
        }
-out:
-       kfree(info);
-       return rc;
+       return info;
 }
 
-static int __smp_rescan_cpus(void)
+static int __devinit smp_add_present_cpu(int cpu);
+
+static int __devinit __smp_rescan_cpus(struct sclp_cpu_info *info,
+                                      int sysfs_add)
 {
+       struct pcpu *pcpu;
        cpumask_t avail;
+       int cpu, nr, i;
 
+       nr = 0;
        cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
-       if (smp_use_sigp_detection)
-               return smp_rescan_cpus_sigp(avail);
-       else
-               return smp_rescan_cpus_sclp(avail);
+       cpu = cpumask_first(&avail);
+       for (i = 0; (i < info->combined) && (cpu < nr_cpu_ids); i++) {
+               if (info->has_cpu_type && info->cpu[i].type != boot_cpu_type)
+                       continue;
+               if (pcpu_find_address(cpu_present_mask, info->cpu[i].address))
+                       continue;
+               pcpu = pcpu_devices + cpu;
+               pcpu->address = info->cpu[i].address;
+               pcpu->state = (cpu >= info->configured) ?
+                       CPU_STATE_STANDBY : CPU_STATE_CONFIGURED;
+               cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
+               set_cpu_present(cpu, true);
+               if (sysfs_add && smp_add_present_cpu(cpu) != 0)
+                       set_cpu_present(cpu, false);
+               else
+                       nr++;
+               cpu = cpumask_next(cpu, &avail);
+       }
+       return nr;
 }
 
 static void __init smp_detect_cpus(void)
 {
        unsigned int cpu, c_cpus, s_cpus;
        struct sclp_cpu_info *info;
-       u16 boot_cpu_addr, cpu_addr;
 
-       c_cpus = 1;
-       s_cpus = 0;
-       boot_cpu_addr = __cpu_logical_map[0];
-       info = kmalloc(sizeof(*info), GFP_KERNEL);
+       info = smp_get_cpu_info();
        if (!info)
                panic("smp_detect_cpus failed to allocate memory\n");
-#ifdef CONFIG_CRASH_DUMP
-       if (OLDMEM_BASE && !is_kdump_kernel()) {
-               struct save_area *save_area;
-
-               save_area = kmalloc(sizeof(*save_area), GFP_KERNEL);
-               if (!save_area)
-                       panic("could not allocate memory for save area\n");
-               copy_oldmem_page(1, (void *) save_area, sizeof(*save_area),
-                                0x200, 0);
-               zfcpdump_save_areas[0] = save_area;
-       }
-#endif
-       /* Use sigp detection algorithm if sclp doesn't work. */
-       if (sclp_get_cpu_info(info)) {
-               smp_use_sigp_detection = 1;
-               for (cpu = 0; cpu <= MAX_CPU_ADDRESS; cpu++) {
-                       if (cpu == boot_cpu_addr)
-                               continue;
-                       if (!raw_cpu_stopped(cpu))
-                               continue;
-                       smp_get_save_area(c_cpus, cpu);
-                       c_cpus++;
-               }
-               goto out;
-       }
-
        if (info->has_cpu_type) {
                for (cpu = 0; cpu < info->combined; cpu++) {
-                       if (info->cpu[cpu].address == boot_cpu_addr) {
-                               smp_cpu_type = info->cpu[cpu].type;
-                               break;
-                       }
+                       if (info->cpu[cpu].address != boot_cpu_address)
+                               continue;
+                       /* The boot cpu dictates the cpu type. */
+                       boot_cpu_type = info->cpu[cpu].type;
+                       break;
                }
        }
-
+       c_cpus = s_cpus = 0;
        for (cpu = 0; cpu < info->combined; cpu++) {
-               if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
+               if (info->has_cpu_type && info->cpu[cpu].type != boot_cpu_type)
                        continue;
-               cpu_addr = info->cpu[cpu].address;
-               if (cpu_addr == boot_cpu_addr)
-                       continue;
-               if (!raw_cpu_stopped(cpu_addr)) {
+               if (cpu < info->configured) {
+                       smp_get_save_area(c_cpus, info->cpu[cpu].address);
+                       c_cpus++;
+               } else
                        s_cpus++;
-                       continue;
-               }
-               smp_get_save_area(c_cpus, cpu_addr);
-               c_cpus++;
        }
-out:
-       kfree(info);
        pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
        get_online_cpus();
-       __smp_rescan_cpus();
+       __smp_rescan_cpus(info, 0);
        put_online_cpus();
+       kfree(info);
 }
 
 /*
  *     Activate a secondary processor.
  */
-int __cpuinit start_secondary(void *cpuvoid)
+static void __cpuinit smp_start_secondary(void *cpuvoid)
 {
+       S390_lowcore.last_update_clock = get_clock();
+       S390_lowcore.restart_stack = (unsigned long) restart_stack;
+       S390_lowcore.restart_fn = (unsigned long) do_restart;
+       S390_lowcore.restart_data = 0;
+       S390_lowcore.restart_source = -1UL;
+       restore_access_regs(S390_lowcore.access_regs_save_area);
+       __ctl_load(S390_lowcore.cregs_save_area, 0, 15);
+       __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
        cpu_init();
        preempt_disable();
        init_cpu_timer();
        init_cpu_vtimer();
        pfault_init();
-
        notify_cpu_starting(smp_processor_id());
        ipi_call_lock();
        set_cpu_online(smp_processor_id(), true);
        ipi_call_unlock();
-       __ctl_clear_bit(0, 28); /* Disable lowcore protection */
-       S390_lowcore.restart_psw.mask =
-               PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
-       S390_lowcore.restart_psw.addr =
-               PSW_ADDR_AMODE | (unsigned long) psw_restart_int_handler;
-       __ctl_set_bit(0, 28); /* Enable lowcore protection */
        /*
         * Wait until the cpu which brought this one up marked it
         * active before enabling interrupts.
@@ -559,7 +719,6 @@ int __cpuinit start_secondary(void *cpuvoid)
        local_irq_enable();
        /* cpu_idle will call schedule for us */
        cpu_idle();
-       return 0;
 }
 
 struct create_idle {
@@ -578,82 +737,20 @@ static void __cpuinit smp_fork_idle(struct work_struct *work)
        complete(&c_idle->done);
 }
 
-static int __cpuinit smp_alloc_lowcore(int cpu)
-{
-       unsigned long async_stack, panic_stack;
-       struct _lowcore *lowcore;
-
-       lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
-       if (!lowcore)
-               return -ENOMEM;
-       async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
-       panic_stack = __get_free_page(GFP_KERNEL);
-       if (!panic_stack || !async_stack)
-               goto out;
-       memcpy(lowcore, &S390_lowcore, 512);
-       memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512);
-       lowcore->async_stack = async_stack + ASYNC_SIZE;
-       lowcore->panic_stack = panic_stack + PAGE_SIZE;
-       lowcore->restart_psw.mask =
-               PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
-       lowcore->restart_psw.addr =
-               PSW_ADDR_AMODE | (unsigned long) restart_int_handler;
-       if (user_mode != HOME_SPACE_MODE)
-               lowcore->restart_psw.mask |= PSW_ASC_HOME;
-#ifndef CONFIG_64BIT
-       if (MACHINE_HAS_IEEE) {
-               unsigned long save_area;
-
-               save_area = get_zeroed_page(GFP_KERNEL);
-               if (!save_area)
-                       goto out;
-               lowcore->extended_save_area_addr = (u32) save_area;
-       }
-#else
-       if (vdso_alloc_per_cpu(cpu, lowcore))
-               goto out;
-#endif
-       lowcore_ptr[cpu] = lowcore;
-       return 0;
-
-out:
-       free_page(panic_stack);
-       free_pages(async_stack, ASYNC_ORDER);
-       free_pages((unsigned long) lowcore, LC_ORDER);
-       return -ENOMEM;
-}
-
-static void smp_free_lowcore(int cpu)
-{
-       struct _lowcore *lowcore;
-
-       lowcore = lowcore_ptr[cpu];
-#ifndef CONFIG_64BIT
-       if (MACHINE_HAS_IEEE)
-               free_page((unsigned long) lowcore->extended_save_area_addr);
-#else
-       vdso_free_per_cpu(cpu, lowcore);
-#endif
-       free_page(lowcore->panic_stack - PAGE_SIZE);
-       free_pages(lowcore->async_stack - ASYNC_SIZE, ASYNC_ORDER);
-       free_pages((unsigned long) lowcore, LC_ORDER);
-       lowcore_ptr[cpu] = NULL;
-}
-
 /* Upping and downing of CPUs */
 int __cpuinit __cpu_up(unsigned int cpu)
 {
-       struct _lowcore *cpu_lowcore;
        struct create_idle c_idle;
-       struct task_struct *idle;
-       struct stack_frame *sf;
-       u32 lowcore;
-       int ccode;
+       struct pcpu *pcpu;
+       int rc;
 
-       if (smp_cpu_state[cpu] != CPU_STATE_CONFIGURED)
+       pcpu = pcpu_devices + cpu;
+       if (pcpu->state != CPU_STATE_CONFIGURED)
                return -EIO;
-       idle = current_set[cpu];
-       if (!idle) {
+       if (pcpu_sigp_retry(pcpu, sigp_initial_cpu_reset, 0) !=
+           sigp_order_code_accepted)
+               return -EIO;
+       if (!pcpu->idle) {
                c_idle.done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done);
                INIT_WORK_ONSTACK(&c_idle.work, smp_fork_idle);
                c_idle.cpu = cpu;
@@ -661,68 +758,28 @@ int __cpuinit __cpu_up(unsigned int cpu)
                wait_for_completion(&c_idle.done);
                if (IS_ERR(c_idle.idle))
                        return PTR_ERR(c_idle.idle);
-               idle = c_idle.idle;
-               current_set[cpu] = c_idle.idle;
+               pcpu->idle = c_idle.idle;
        }
-       init_idle(idle, cpu);
-       if (smp_alloc_lowcore(cpu))
-               return -ENOMEM;
-       do {
-               ccode = sigp(cpu, sigp_initial_cpu_reset);
-               if (ccode == sigp_busy)
-                       udelay(10);
-               if (ccode == sigp_not_operational)
-                       goto err_out;
-       } while (ccode == sigp_busy);
-
-       lowcore = (u32)(unsigned long)lowcore_ptr[cpu];
-       while (sigp_p(lowcore, cpu, sigp_set_prefix) == sigp_busy)
-               udelay(10);
-
-       cpu_lowcore = lowcore_ptr[cpu];
-       cpu_lowcore->kernel_stack = (unsigned long)
-               task_stack_page(idle) + THREAD_SIZE;
-       cpu_lowcore->thread_info = (unsigned long) task_thread_info(idle);
-       sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
-                                    - sizeof(struct pt_regs)
-                                    - sizeof(struct stack_frame));
-       memset(sf, 0, sizeof(struct stack_frame));
-       sf->gprs[9] = (unsigned long) sf;
-       cpu_lowcore->gpregs_save_area[15] = (unsigned long) sf;
-       __ctl_store(cpu_lowcore->cregs_save_area, 0, 15);
-       atomic_inc(&init_mm.context.attach_count);
-       asm volatile(
-               "       stam    0,15,0(%0)"
-               : : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
-       cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
-       cpu_lowcore->current_task = (unsigned long) idle;
-       cpu_lowcore->cpu_nr = cpu;
-       cpu_lowcore->kernel_asce = S390_lowcore.kernel_asce;
-       cpu_lowcore->machine_flags = S390_lowcore.machine_flags;
-       cpu_lowcore->ftrace_func = S390_lowcore.ftrace_func;
-       memcpy(cpu_lowcore->stfle_fac_list, S390_lowcore.stfle_fac_list,
-              MAX_FACILITY_BIT/8);
-       eieio();
-
-       while (sigp(cpu, sigp_restart) == sigp_busy)
-               udelay(10);
-
+       init_idle(pcpu->idle, cpu);
+       rc = pcpu_alloc_lowcore(pcpu, cpu);
+       if (rc)
+               return rc;
+       pcpu_prepare_secondary(pcpu, cpu);
+       pcpu_attach_task(pcpu, pcpu->idle);
+       pcpu_start_fn(pcpu, smp_start_secondary, NULL);
        while (!cpu_online(cpu))
                cpu_relax();
        return 0;
-
-err_out:
-       smp_free_lowcore(cpu);
-       return -EIO;
 }
 
 static int __init setup_possible_cpus(char *s)
 {
-       int pcpus, cpu;
+       int max, cpu;
 
-       pcpus = simple_strtoul(s, NULL, 0);
+       if (kstrtoint(s, 0, &max) < 0)
+               return 0;
        init_cpu_possible(cpumask_of(0));
-       for (cpu = 1; cpu < pcpus && cpu < nr_cpu_ids; cpu++)
+       for (cpu = 1; cpu < max && cpu < nr_cpu_ids; cpu++)
                set_cpu_possible(cpu, true);
        return 0;
 }
@@ -732,113 +789,67 @@ early_param("possible_cpus", setup_possible_cpus);
 
 int __cpu_disable(void)
 {
-       struct ec_creg_mask_parms cr_parms;
-       int cpu = smp_processor_id();
-
-       set_cpu_online(cpu, false);
+       unsigned long cregs[16];
 
-       /* Disable pfault pseudo page faults on this cpu. */
+       set_cpu_online(smp_processor_id(), false);
+       /* Disable pseudo page faults on this cpu. */
        pfault_fini();
-
-       memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals));
-       memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals));
-
-       /* disable all external interrupts */
-       cr_parms.orvals[0] = 0;
-       cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 11 |
-                               1 << 10 | 1 <<  9 | 1 <<  6 | 1 <<  5 |
-                               1 <<  4);
-       /* disable all I/O interrupts */
-       cr_parms.orvals[6] = 0;
-       cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 |
-                               1 << 27 | 1 << 26 | 1 << 25 | 1 << 24);
-       /* disable most machine checks */
-       cr_parms.orvals[14] = 0;
-       cr_parms.andvals[14] = ~(1 << 28 | 1 << 27 | 1 << 26 |
-                                1 << 25 | 1 << 24);
-
-       smp_ctl_bit_callback(&cr_parms);
-
+       /* Disable interrupt sources via control register. */
+       __ctl_store(cregs, 0, 15);
+       cregs[0]  &= ~0x0000ee70UL;     /* disable all external interrupts */
+       cregs[6]  &= ~0xff000000UL;     /* disable all I/O interrupts */
+       cregs[14] &= ~0x1f000000UL;     /* disable most machine checks */
+       __ctl_load(cregs, 0, 15);
        return 0;
 }
 
 void __cpu_die(unsigned int cpu)
 {
+       struct pcpu *pcpu;
+
        /* Wait until target cpu is down */
-       while (!cpu_stopped(cpu))
+       pcpu = pcpu_devices + cpu;
+       while (!pcpu_stopped(pcpu))
                cpu_relax();
-       while (sigp_p(0, cpu, sigp_set_prefix) == sigp_busy)
-               udelay(10);
-       smp_free_lowcore(cpu);
+       pcpu_free_lowcore(pcpu);
        atomic_dec(&init_mm.context.attach_count);
 }
 
 void __noreturn cpu_die(void)
 {
        idle_task_exit();
-       while (sigp(smp_processor_id(), sigp_stop) == sigp_busy)
-               cpu_relax();
-       for (;;);
+       pcpu_sigp_retry(pcpu_devices + smp_processor_id(), sigp_stop, 0);
+       for (;;) ;
 }
 
 #endif /* CONFIG_HOTPLUG_CPU */
 
 void __init smp_prepare_cpus(unsigned int max_cpus)
 {
-#ifndef CONFIG_64BIT
-       unsigned long save_area = 0;
-#endif
-       unsigned long async_stack, panic_stack;
-       struct _lowcore *lowcore;
-
-       smp_detect_cpus();
-
        /* request the 0x1201 emergency signal external interrupt */
        if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
                panic("Couldn't request external interrupt 0x1201");
        /* request the 0x1202 external call external interrupt */
        if (register_external_interrupt(0x1202, do_ext_call_interrupt) != 0)
                panic("Couldn't request external interrupt 0x1202");
-
-       /* Reallocate current lowcore, but keep its contents. */
-       lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
-       panic_stack = __get_free_page(GFP_KERNEL);
-       async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
-       BUG_ON(!lowcore || !panic_stack || !async_stack);
-#ifndef CONFIG_64BIT
-       if (MACHINE_HAS_IEEE)
-               save_area = get_zeroed_page(GFP_KERNEL);
-#endif
-       local_irq_disable();
-       local_mcck_disable();
-       lowcore_ptr[smp_processor_id()] = lowcore;
-       *lowcore = S390_lowcore;
-       lowcore->panic_stack = panic_stack + PAGE_SIZE;
-       lowcore->async_stack = async_stack + ASYNC_SIZE;
-#ifndef CONFIG_64BIT
-       if (MACHINE_HAS_IEEE)
-               lowcore->extended_save_area_addr = (u32) save_area;
-#endif
-       set_prefix((u32)(unsigned long) lowcore);
-       local_mcck_enable();
-       local_irq_enable();
-#ifdef CONFIG_64BIT
-       if (vdso_alloc_per_cpu(smp_processor_id(), &S390_lowcore))
-               BUG();
-#endif
+       smp_detect_cpus();
 }
 
 void __init smp_prepare_boot_cpu(void)
 {
-       BUG_ON(smp_processor_id() != 0);
-
-       current_thread_info()->cpu = 0;
-       set_cpu_present(0, true);
-       set_cpu_online(0, true);
+       struct pcpu *pcpu = pcpu_devices;
+
+       boot_cpu_address = stap();
+       pcpu->idle = current;
+       pcpu->state = CPU_STATE_CONFIGURED;
+       pcpu->address = boot_cpu_address;
+       pcpu->lowcore = (struct _lowcore *)(unsigned long) store_prefix();
+       pcpu->async_stack = S390_lowcore.async_stack - ASYNC_SIZE;
+       pcpu->panic_stack = S390_lowcore.panic_stack - PAGE_SIZE;
        S390_lowcore.percpu_offset = __per_cpu_offset[0];
-       current_set[0] = current;
-       smp_cpu_state[0] = CPU_STATE_CONFIGURED;
        cpu_set_polarization(0, POLARIZATION_UNKNOWN);
+       set_cpu_present(0, true);
+       set_cpu_online(0, true);
 }
 
 void __init smp_cpus_done(unsigned int max_cpus)
@@ -848,7 +859,6 @@ void __init smp_cpus_done(unsigned int max_cpus)
 void __init smp_setup_processor_id(void)
 {
        S390_lowcore.cpu_nr = 0;
-       __cpu_logical_map[0] = stap();
 }
 
 /*
@@ -864,56 +874,57 @@ int setup_profiling_timer(unsigned int multiplier)
 
 #ifdef CONFIG_HOTPLUG_CPU
 static ssize_t cpu_configure_show(struct device *dev,
-                               struct device_attribute *attr, char *buf)
+                                 struct device_attribute *attr, char *buf)
 {
        ssize_t count;
 
        mutex_lock(&smp_cpu_state_mutex);
-       count = sprintf(buf, "%d\n", smp_cpu_state[dev->id]);
+       count = sprintf(buf, "%d\n", pcpu_devices[dev->id].state);
        mutex_unlock(&smp_cpu_state_mutex);
        return count;
 }
 
 static ssize_t cpu_configure_store(struct device *dev,
-                                 struct device_attribute *attr,
-                                 const char *buf, size_t count)
+                                  struct device_attribute *attr,
+                                  const char *buf, size_t count)
 {
-       int cpu = dev->id;
-       int val, rc;
+       struct pcpu *pcpu;
+       int cpu, val, rc;
        char delim;
 
        if (sscanf(buf, "%d %c", &val, &delim) != 1)
                return -EINVAL;
        if (val != 0 && val != 1)
                return -EINVAL;
-
        get_online_cpus();
        mutex_lock(&smp_cpu_state_mutex);
        rc = -EBUSY;
        /* disallow configuration changes of online cpus and cpu 0 */
+       cpu = dev->id;
        if (cpu_online(cpu) || cpu == 0)
                goto out;
+       pcpu = pcpu_devices + cpu;
        rc = 0;
        switch (val) {
        case 0:
-               if (smp_cpu_state[cpu] == CPU_STATE_CONFIGURED) {
-                       rc = sclp_cpu_deconfigure(__cpu_logical_map[cpu]);
-                       if (!rc) {
-                               smp_cpu_state[cpu] = CPU_STATE_STANDBY;
-                               cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
-                               topology_expect_change();
-                       }
-               }
+               if (pcpu->state != CPU_STATE_CONFIGURED)
+                       break;
+               rc = sclp_cpu_deconfigure(pcpu->address);
+               if (rc)
+                       break;
+               pcpu->state = CPU_STATE_STANDBY;
+               cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
+               topology_expect_change();
                break;
        case 1:
-               if (smp_cpu_state[cpu] == CPU_STATE_STANDBY) {
-                       rc = sclp_cpu_configure(__cpu_logical_map[cpu]);
-                       if (!rc) {
-                               smp_cpu_state[cpu] = CPU_STATE_CONFIGURED;
-                               cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
-                               topology_expect_change();
-                       }
-               }
+               if (pcpu->state != CPU_STATE_STANDBY)
+                       break;
+               rc = sclp_cpu_configure(pcpu->address);
+               if (rc)
+                       break;
+               pcpu->state = CPU_STATE_CONFIGURED;
+               cpu_set_polarization(cpu, POLARIZATION_UNKNOWN);
+               topology_expect_change();
                break;
        default:
                break;
@@ -929,7 +940,7 @@ static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
 static ssize_t show_cpu_address(struct device *dev,
                                struct device_attribute *attr, char *buf)
 {
-       return sprintf(buf, "%d\n", __cpu_logical_map[dev->id]);
+       return sprintf(buf, "%d\n", pcpu_devices[dev->id].address);
 }
 static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
 
@@ -1021,7 +1032,7 @@ static int __cpuinit smp_cpu_notify(struct notifier_block *self,
                                    unsigned long action, void *hcpu)
 {
        unsigned int cpu = (unsigned int)(long)hcpu;
-       struct cpu *c = &per_cpu(cpu_devices, cpu);
+       struct cpu *c = &pcpu_devices[cpu].cpu;
        struct device *s = &c->dev;
        struct s390_idle_data *idle;
        int err = 0;
@@ -1047,7 +1058,7 @@ static struct notifier_block __cpuinitdata smp_cpu_nb = {
 
 static int __devinit smp_add_present_cpu(int cpu)
 {
-       struct cpu *c = &per_cpu(cpu_devices, cpu);
+       struct cpu *c = &pcpu_devices[cpu].cpu;
        struct device *s = &c->dev;
        int rc;
 
@@ -1085,29 +1096,21 @@ out:
 
 int __ref smp_rescan_cpus(void)
 {
-       cpumask_t newcpus;
-       int cpu;
-       int rc;
+       struct sclp_cpu_info *info;
+       int nr;
 
+       info = smp_get_cpu_info();
+       if (!info)
+               return -ENOMEM;
        get_online_cpus();
        mutex_lock(&smp_cpu_state_mutex);
-       cpumask_copy(&newcpus, cpu_present_mask);
-       rc = __smp_rescan_cpus();
-       if (rc)
-               goto out;
-       cpumask_andnot(&newcpus, cpu_present_mask, &newcpus);
-       for_each_cpu(cpu, &newcpus) {
-               rc = smp_add_present_cpu(cpu);
-               if (rc)
-                       set_cpu_present(cpu, false);
-       }
-       rc = 0;
-out:
+       nr = __smp_rescan_cpus(info, 1);
        mutex_unlock(&smp_cpu_state_mutex);
        put_online_cpus();
-       if (!cpumask_empty(&newcpus))
+       kfree(info);
+       if (nr)
                topology_schedule_update();
-       return rc;
+       return 0;
 }
 
 static ssize_t __ref rescan_store(struct device *dev,