cpu: convert 'cpu' and 'machinecheck' sysdev_class to a regular subsystem
[linux-2.6.git] / arch / s390 / kernel / smp.c
1 /*
2  *  arch/s390/kernel/smp.c
3  *
4  *    Copyright IBM Corp. 1999, 2009
5  *    Author(s): Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com),
6  *               Martin Schwidefsky (schwidefsky@de.ibm.com)
7  *               Heiko Carstens (heiko.carstens@de.ibm.com)
8  *
9  *  based on other smp stuff by
10  *    (c) 1995 Alan Cox, CymruNET Ltd  <alan@cymru.net>
11  *    (c) 1998 Ingo Molnar
12  *
13  * We work with logical cpu numbering everywhere we can. The only
14  * functions using the real cpu address (got from STAP) are the sigp
15  * functions. For all other functions we use the identity mapping.
16  * That means that cpu_number_map[i] == i for every cpu. cpu_number_map is
17  * used e.g. to find the idle task belonging to a logical cpu. Every array
18  * in the kernel is sorted by the logical cpu number and not by the physical
19  * one which is causing all the confusion with __cpu_logical_map and
20  * cpu_number_map in other architectures.
21  */
22
23 #define KMSG_COMPONENT "cpu"
24 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
25
26 #include <linux/workqueue.h>
27 #include <linux/module.h>
28 #include <linux/init.h>
29 #include <linux/mm.h>
30 #include <linux/err.h>
31 #include <linux/spinlock.h>
32 #include <linux/kernel_stat.h>
33 #include <linux/delay.h>
34 #include <linux/cache.h>
35 #include <linux/interrupt.h>
36 #include <linux/irqflags.h>
37 #include <linux/cpu.h>
38 #include <linux/timex.h>
39 #include <linux/bootmem.h>
40 #include <linux/slab.h>
41 #include <linux/crash_dump.h>
42 #include <asm/asm-offsets.h>
43 #include <asm/ipl.h>
44 #include <asm/setup.h>
45 #include <asm/sigp.h>
46 #include <asm/pgalloc.h>
47 #include <asm/irq.h>
48 #include <asm/cpcmd.h>
49 #include <asm/tlbflush.h>
50 #include <asm/timer.h>
51 #include <asm/lowcore.h>
52 #include <asm/sclp.h>
53 #include <asm/cputime.h>
54 #include <asm/vdso.h>
55 #include <asm/cpu.h>
56 #include "entry.h"
57
58 /* logical cpu to cpu address */
59 unsigned short __cpu_logical_map[NR_CPUS];
60
61 static struct task_struct *current_set[NR_CPUS];
62
63 static u8 smp_cpu_type;
64 static int smp_use_sigp_detection;
65
66 enum s390_cpu_state {
67         CPU_STATE_STANDBY,
68         CPU_STATE_CONFIGURED,
69 };
70
71 DEFINE_MUTEX(smp_cpu_state_mutex);
72 int smp_cpu_polarization[NR_CPUS];
73 static int smp_cpu_state[NR_CPUS];
74 static int cpu_management;
75
76 static DEFINE_PER_CPU(struct cpu, cpu_devices);
77
78 static void smp_ext_bitcall(int, int);
79
80 static int raw_cpu_stopped(int cpu)
81 {
82         u32 status;
83
84         switch (raw_sigp_ps(&status, 0, cpu, sigp_sense)) {
85         case sigp_status_stored:
86                 /* Check for stopped and check stop state */
87                 if (status & 0x50)
88                         return 1;
89                 break;
90         default:
91                 break;
92         }
93         return 0;
94 }
95
96 static inline int cpu_stopped(int cpu)
97 {
98         return raw_cpu_stopped(cpu_logical_map(cpu));
99 }
100
101 /*
102  * Ensure that PSW restart is done on an online CPU
103  */
104 void smp_restart_with_online_cpu(void)
105 {
106         int cpu;
107
108         for_each_online_cpu(cpu) {
109                 if (stap() == __cpu_logical_map[cpu]) {
110                         /* We are online: Enable DAT again and return */
111                         __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
112                         return;
113                 }
114         }
115         /* We are not online: Do PSW restart on an online CPU */
116         while (sigp(cpu, sigp_restart) == sigp_busy)
117                 cpu_relax();
118         /* And stop ourself */
119         while (raw_sigp(stap(), sigp_stop) == sigp_busy)
120                 cpu_relax();
121         for (;;);
122 }
123
124 void smp_switch_to_ipl_cpu(void (*func)(void *), void *data)
125 {
126         struct _lowcore *lc, *current_lc;
127         struct stack_frame *sf;
128         struct pt_regs *regs;
129         unsigned long sp;
130
131         if (smp_processor_id() == 0)
132                 func(data);
133         __load_psw_mask(PSW_DEFAULT_KEY | PSW_MASK_BASE |
134                         PSW_MASK_EA | PSW_MASK_BA);
135         /* Disable lowcore protection */
136         __ctl_clear_bit(0, 28);
137         current_lc = lowcore_ptr[smp_processor_id()];
138         lc = lowcore_ptr[0];
139         if (!lc)
140                 lc = current_lc;
141         lc->restart_psw.mask =
142                 PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
143         lc->restart_psw.addr = PSW_ADDR_AMODE | (unsigned long) smp_restart_cpu;
144         if (!cpu_online(0))
145                 smp_switch_to_cpu(func, data, 0, stap(), __cpu_logical_map[0]);
146         while (sigp(0, sigp_stop_and_store_status) == sigp_busy)
147                 cpu_relax();
148         sp = lc->panic_stack;
149         sp -= sizeof(struct pt_regs);
150         regs = (struct pt_regs *) sp;
151         memcpy(&regs->gprs, &current_lc->gpregs_save_area, sizeof(regs->gprs));
152         regs->psw = lc->psw_save_area;
153         sp -= STACK_FRAME_OVERHEAD;
154         sf = (struct stack_frame *) sp;
155         sf->back_chain = regs->gprs[15];
156         smp_switch_to_cpu(func, data, sp, stap(), __cpu_logical_map[0]);
157 }
158
159 void smp_send_stop(void)
160 {
161         int cpu, rc;
162
163         /* Disable all interrupts/machine checks */
164         __load_psw_mask(psw_kernel_bits | PSW_MASK_DAT);
165         trace_hardirqs_off();
166
167         /* stop all processors */
168         for_each_online_cpu(cpu) {
169                 if (cpu == smp_processor_id())
170                         continue;
171                 do {
172                         rc = sigp(cpu, sigp_stop);
173                 } while (rc == sigp_busy);
174
175                 while (!cpu_stopped(cpu))
176                         cpu_relax();
177         }
178 }
179
180 /*
181  * This is the main routine where commands issued by other
182  * cpus are handled.
183  */
184
185 static void do_ext_call_interrupt(unsigned int ext_int_code,
186                                   unsigned int param32, unsigned long param64)
187 {
188         unsigned long bits;
189
190         if (ext_int_code == 0x1202)
191                 kstat_cpu(smp_processor_id()).irqs[EXTINT_EXC]++;
192         else
193                 kstat_cpu(smp_processor_id()).irqs[EXTINT_EMS]++;
194         /*
195          * handle bit signal external calls
196          */
197         bits = xchg(&S390_lowcore.ext_call_fast, 0);
198
199         if (test_bit(ec_schedule, &bits))
200                 scheduler_ipi();
201
202         if (test_bit(ec_call_function, &bits))
203                 generic_smp_call_function_interrupt();
204
205         if (test_bit(ec_call_function_single, &bits))
206                 generic_smp_call_function_single_interrupt();
207 }
208
209 /*
210  * Send an external call sigp to another cpu and return without waiting
211  * for its completion.
212  */
213 static void smp_ext_bitcall(int cpu, int sig)
214 {
215         int order;
216
217         /*
218          * Set signaling bit in lowcore of target cpu and kick it
219          */
220         set_bit(sig, (unsigned long *) &lowcore_ptr[cpu]->ext_call_fast);
221         while (1) {
222                 order = smp_vcpu_scheduled(cpu) ?
223                         sigp_external_call : sigp_emergency_signal;
224                 if (sigp(cpu, order) != sigp_busy)
225                         break;
226                 udelay(10);
227         }
228 }
229
230 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
231 {
232         int cpu;
233
234         for_each_cpu(cpu, mask)
235                 smp_ext_bitcall(cpu, ec_call_function);
236 }
237
238 void arch_send_call_function_single_ipi(int cpu)
239 {
240         smp_ext_bitcall(cpu, ec_call_function_single);
241 }
242
243 #ifndef CONFIG_64BIT
244 /*
245  * this function sends a 'purge tlb' signal to another CPU.
246  */
247 static void smp_ptlb_callback(void *info)
248 {
249         __tlb_flush_local();
250 }
251
252 void smp_ptlb_all(void)
253 {
254         on_each_cpu(smp_ptlb_callback, NULL, 1);
255 }
256 EXPORT_SYMBOL(smp_ptlb_all);
257 #endif /* ! CONFIG_64BIT */
258
259 /*
260  * this function sends a 'reschedule' IPI to another CPU.
261  * it goes straight through and wastes no time serializing
262  * anything. Worst case is that we lose a reschedule ...
263  */
264 void smp_send_reschedule(int cpu)
265 {
266         smp_ext_bitcall(cpu, ec_schedule);
267 }
268
269 /*
270  * parameter area for the set/clear control bit callbacks
271  */
272 struct ec_creg_mask_parms {
273         unsigned long orvals[16];
274         unsigned long andvals[16];
275 };
276
277 /*
278  * callback for setting/clearing control bits
279  */
280 static void smp_ctl_bit_callback(void *info)
281 {
282         struct ec_creg_mask_parms *pp = info;
283         unsigned long cregs[16];
284         int i;
285
286         __ctl_store(cregs, 0, 15);
287         for (i = 0; i <= 15; i++)
288                 cregs[i] = (cregs[i] & pp->andvals[i]) | pp->orvals[i];
289         __ctl_load(cregs, 0, 15);
290 }
291
292 /*
293  * Set a bit in a control register of all cpus
294  */
295 void smp_ctl_set_bit(int cr, int bit)
296 {
297         struct ec_creg_mask_parms parms;
298
299         memset(&parms.orvals, 0, sizeof(parms.orvals));
300         memset(&parms.andvals, 0xff, sizeof(parms.andvals));
301         parms.orvals[cr] = 1UL << bit;
302         on_each_cpu(smp_ctl_bit_callback, &parms, 1);
303 }
304 EXPORT_SYMBOL(smp_ctl_set_bit);
305
306 /*
307  * Clear a bit in a control register of all cpus
308  */
309 void smp_ctl_clear_bit(int cr, int bit)
310 {
311         struct ec_creg_mask_parms parms;
312
313         memset(&parms.orvals, 0, sizeof(parms.orvals));
314         memset(&parms.andvals, 0xff, sizeof(parms.andvals));
315         parms.andvals[cr] = ~(1UL << bit);
316         on_each_cpu(smp_ctl_bit_callback, &parms, 1);
317 }
318 EXPORT_SYMBOL(smp_ctl_clear_bit);
319
320 #if defined(CONFIG_ZFCPDUMP) || defined(CONFIG_CRASH_DUMP)
321
322 static void __init smp_get_save_area(unsigned int cpu, unsigned int phy_cpu)
323 {
324         if (ipl_info.type != IPL_TYPE_FCP_DUMP && !OLDMEM_BASE)
325                 return;
326         if (is_kdump_kernel())
327                 return;
328         if (cpu >= NR_CPUS) {
329                 pr_warning("CPU %i exceeds the maximum %i and is excluded from "
330                            "the dump\n", cpu, NR_CPUS - 1);
331                 return;
332         }
333         zfcpdump_save_areas[cpu] = kmalloc(sizeof(struct save_area), GFP_KERNEL);
334         while (raw_sigp(phy_cpu, sigp_stop_and_store_status) == sigp_busy)
335                 cpu_relax();
336         memcpy_real(zfcpdump_save_areas[cpu],
337                     (void *)(unsigned long) store_prefix() + SAVE_AREA_BASE,
338                     sizeof(struct save_area));
339 }
340
341 struct save_area *zfcpdump_save_areas[NR_CPUS + 1];
342 EXPORT_SYMBOL_GPL(zfcpdump_save_areas);
343
344 #else
345
346 static inline void smp_get_save_area(unsigned int cpu, unsigned int phy_cpu) { }
347
348 #endif /* CONFIG_ZFCPDUMP */
349
350 static int cpu_known(int cpu_id)
351 {
352         int cpu;
353
354         for_each_present_cpu(cpu) {
355                 if (__cpu_logical_map[cpu] == cpu_id)
356                         return 1;
357         }
358         return 0;
359 }
360
361 static int smp_rescan_cpus_sigp(cpumask_t avail)
362 {
363         int cpu_id, logical_cpu;
364
365         logical_cpu = cpumask_first(&avail);
366         if (logical_cpu >= nr_cpu_ids)
367                 return 0;
368         for (cpu_id = 0; cpu_id <= MAX_CPU_ADDRESS; cpu_id++) {
369                 if (cpu_known(cpu_id))
370                         continue;
371                 __cpu_logical_map[logical_cpu] = cpu_id;
372                 smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
373                 if (!cpu_stopped(logical_cpu))
374                         continue;
375                 set_cpu_present(logical_cpu, true);
376                 smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
377                 logical_cpu = cpumask_next(logical_cpu, &avail);
378                 if (logical_cpu >= nr_cpu_ids)
379                         break;
380         }
381         return 0;
382 }
383
384 static int smp_rescan_cpus_sclp(cpumask_t avail)
385 {
386         struct sclp_cpu_info *info;
387         int cpu_id, logical_cpu, cpu;
388         int rc;
389
390         logical_cpu = cpumask_first(&avail);
391         if (logical_cpu >= nr_cpu_ids)
392                 return 0;
393         info = kmalloc(sizeof(*info), GFP_KERNEL);
394         if (!info)
395                 return -ENOMEM;
396         rc = sclp_get_cpu_info(info);
397         if (rc)
398                 goto out;
399         for (cpu = 0; cpu < info->combined; cpu++) {
400                 if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
401                         continue;
402                 cpu_id = info->cpu[cpu].address;
403                 if (cpu_known(cpu_id))
404                         continue;
405                 __cpu_logical_map[logical_cpu] = cpu_id;
406                 smp_cpu_polarization[logical_cpu] = POLARIZATION_UNKNWN;
407                 set_cpu_present(logical_cpu, true);
408                 if (cpu >= info->configured)
409                         smp_cpu_state[logical_cpu] = CPU_STATE_STANDBY;
410                 else
411                         smp_cpu_state[logical_cpu] = CPU_STATE_CONFIGURED;
412                 logical_cpu = cpumask_next(logical_cpu, &avail);
413                 if (logical_cpu >= nr_cpu_ids)
414                         break;
415         }
416 out:
417         kfree(info);
418         return rc;
419 }
420
421 static int __smp_rescan_cpus(void)
422 {
423         cpumask_t avail;
424
425         cpumask_xor(&avail, cpu_possible_mask, cpu_present_mask);
426         if (smp_use_sigp_detection)
427                 return smp_rescan_cpus_sigp(avail);
428         else
429                 return smp_rescan_cpus_sclp(avail);
430 }
431
432 static void __init smp_detect_cpus(void)
433 {
434         unsigned int cpu, c_cpus, s_cpus;
435         struct sclp_cpu_info *info;
436         u16 boot_cpu_addr, cpu_addr;
437
438         c_cpus = 1;
439         s_cpus = 0;
440         boot_cpu_addr = __cpu_logical_map[0];
441         info = kmalloc(sizeof(*info), GFP_KERNEL);
442         if (!info)
443                 panic("smp_detect_cpus failed to allocate memory\n");
444 #ifdef CONFIG_CRASH_DUMP
445         if (OLDMEM_BASE && !is_kdump_kernel()) {
446                 struct save_area *save_area;
447
448                 save_area = kmalloc(sizeof(*save_area), GFP_KERNEL);
449                 if (!save_area)
450                         panic("could not allocate memory for save area\n");
451                 copy_oldmem_page(1, (void *) save_area, sizeof(*save_area),
452                                  0x200, 0);
453                 zfcpdump_save_areas[0] = save_area;
454         }
455 #endif
456         /* Use sigp detection algorithm if sclp doesn't work. */
457         if (sclp_get_cpu_info(info)) {
458                 smp_use_sigp_detection = 1;
459                 for (cpu = 0; cpu <= MAX_CPU_ADDRESS; cpu++) {
460                         if (cpu == boot_cpu_addr)
461                                 continue;
462                         if (!raw_cpu_stopped(cpu))
463                                 continue;
464                         smp_get_save_area(c_cpus, cpu);
465                         c_cpus++;
466                 }
467                 goto out;
468         }
469
470         if (info->has_cpu_type) {
471                 for (cpu = 0; cpu < info->combined; cpu++) {
472                         if (info->cpu[cpu].address == boot_cpu_addr) {
473                                 smp_cpu_type = info->cpu[cpu].type;
474                                 break;
475                         }
476                 }
477         }
478
479         for (cpu = 0; cpu < info->combined; cpu++) {
480                 if (info->has_cpu_type && info->cpu[cpu].type != smp_cpu_type)
481                         continue;
482                 cpu_addr = info->cpu[cpu].address;
483                 if (cpu_addr == boot_cpu_addr)
484                         continue;
485                 if (!raw_cpu_stopped(cpu_addr)) {
486                         s_cpus++;
487                         continue;
488                 }
489                 smp_get_save_area(c_cpus, cpu_addr);
490                 c_cpus++;
491         }
492 out:
493         kfree(info);
494         pr_info("%d configured CPUs, %d standby CPUs\n", c_cpus, s_cpus);
495         get_online_cpus();
496         __smp_rescan_cpus();
497         put_online_cpus();
498 }
499
500 /*
501  *      Activate a secondary processor.
502  */
503 int __cpuinit start_secondary(void *cpuvoid)
504 {
505         cpu_init();
506         preempt_disable();
507         init_cpu_timer();
508         init_cpu_vtimer();
509         pfault_init();
510
511         notify_cpu_starting(smp_processor_id());
512         ipi_call_lock();
513         set_cpu_online(smp_processor_id(), true);
514         ipi_call_unlock();
515         __ctl_clear_bit(0, 28); /* Disable lowcore protection */
516         S390_lowcore.restart_psw.mask =
517                 PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
518         S390_lowcore.restart_psw.addr =
519                 PSW_ADDR_AMODE | (unsigned long) psw_restart_int_handler;
520         __ctl_set_bit(0, 28); /* Enable lowcore protection */
521         /*
522          * Wait until the cpu which brought this one up marked it
523          * active before enabling interrupts.
524          */
525         while (!cpumask_test_cpu(smp_processor_id(), cpu_active_mask))
526                 cpu_relax();
527         local_irq_enable();
528         /* cpu_idle will call schedule for us */
529         cpu_idle();
530         return 0;
531 }
532
533 struct create_idle {
534         struct work_struct work;
535         struct task_struct *idle;
536         struct completion done;
537         int cpu;
538 };
539
540 static void __cpuinit smp_fork_idle(struct work_struct *work)
541 {
542         struct create_idle *c_idle;
543
544         c_idle = container_of(work, struct create_idle, work);
545         c_idle->idle = fork_idle(c_idle->cpu);
546         complete(&c_idle->done);
547 }
548
549 static int __cpuinit smp_alloc_lowcore(int cpu)
550 {
551         unsigned long async_stack, panic_stack;
552         struct _lowcore *lowcore;
553
554         lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
555         if (!lowcore)
556                 return -ENOMEM;
557         async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
558         panic_stack = __get_free_page(GFP_KERNEL);
559         if (!panic_stack || !async_stack)
560                 goto out;
561         memcpy(lowcore, &S390_lowcore, 512);
562         memset((char *)lowcore + 512, 0, sizeof(*lowcore) - 512);
563         lowcore->async_stack = async_stack + ASYNC_SIZE;
564         lowcore->panic_stack = panic_stack + PAGE_SIZE;
565         lowcore->restart_psw.mask =
566                 PSW_DEFAULT_KEY | PSW_MASK_BASE | PSW_MASK_EA | PSW_MASK_BA;
567         lowcore->restart_psw.addr =
568                 PSW_ADDR_AMODE | (unsigned long) restart_int_handler;
569         if (user_mode != HOME_SPACE_MODE)
570                 lowcore->restart_psw.mask |= PSW_ASC_HOME;
571 #ifndef CONFIG_64BIT
572         if (MACHINE_HAS_IEEE) {
573                 unsigned long save_area;
574
575                 save_area = get_zeroed_page(GFP_KERNEL);
576                 if (!save_area)
577                         goto out;
578                 lowcore->extended_save_area_addr = (u32) save_area;
579         }
580 #else
581         if (vdso_alloc_per_cpu(cpu, lowcore))
582                 goto out;
583 #endif
584         lowcore_ptr[cpu] = lowcore;
585         return 0;
586
587 out:
588         free_page(panic_stack);
589         free_pages(async_stack, ASYNC_ORDER);
590         free_pages((unsigned long) lowcore, LC_ORDER);
591         return -ENOMEM;
592 }
593
594 static void smp_free_lowcore(int cpu)
595 {
596         struct _lowcore *lowcore;
597
598         lowcore = lowcore_ptr[cpu];
599 #ifndef CONFIG_64BIT
600         if (MACHINE_HAS_IEEE)
601                 free_page((unsigned long) lowcore->extended_save_area_addr);
602 #else
603         vdso_free_per_cpu(cpu, lowcore);
604 #endif
605         free_page(lowcore->panic_stack - PAGE_SIZE);
606         free_pages(lowcore->async_stack - ASYNC_SIZE, ASYNC_ORDER);
607         free_pages((unsigned long) lowcore, LC_ORDER);
608         lowcore_ptr[cpu] = NULL;
609 }
610
611 /* Upping and downing of CPUs */
612 int __cpuinit __cpu_up(unsigned int cpu)
613 {
614         struct _lowcore *cpu_lowcore;
615         struct create_idle c_idle;
616         struct task_struct *idle;
617         struct stack_frame *sf;
618         u32 lowcore;
619         int ccode;
620
621         if (smp_cpu_state[cpu] != CPU_STATE_CONFIGURED)
622                 return -EIO;
623         idle = current_set[cpu];
624         if (!idle) {
625                 c_idle.done = COMPLETION_INITIALIZER_ONSTACK(c_idle.done);
626                 INIT_WORK_ONSTACK(&c_idle.work, smp_fork_idle);
627                 c_idle.cpu = cpu;
628                 schedule_work(&c_idle.work);
629                 wait_for_completion(&c_idle.done);
630                 if (IS_ERR(c_idle.idle))
631                         return PTR_ERR(c_idle.idle);
632                 idle = c_idle.idle;
633                 current_set[cpu] = c_idle.idle;
634         }
635         init_idle(idle, cpu);
636         if (smp_alloc_lowcore(cpu))
637                 return -ENOMEM;
638         do {
639                 ccode = sigp(cpu, sigp_initial_cpu_reset);
640                 if (ccode == sigp_busy)
641                         udelay(10);
642                 if (ccode == sigp_not_operational)
643                         goto err_out;
644         } while (ccode == sigp_busy);
645
646         lowcore = (u32)(unsigned long)lowcore_ptr[cpu];
647         while (sigp_p(lowcore, cpu, sigp_set_prefix) == sigp_busy)
648                 udelay(10);
649
650         cpu_lowcore = lowcore_ptr[cpu];
651         cpu_lowcore->kernel_stack = (unsigned long)
652                 task_stack_page(idle) + THREAD_SIZE;
653         cpu_lowcore->thread_info = (unsigned long) task_thread_info(idle);
654         sf = (struct stack_frame *) (cpu_lowcore->kernel_stack
655                                      - sizeof(struct pt_regs)
656                                      - sizeof(struct stack_frame));
657         memset(sf, 0, sizeof(struct stack_frame));
658         sf->gprs[9] = (unsigned long) sf;
659         cpu_lowcore->save_area[15] = (unsigned long) sf;
660         __ctl_store(cpu_lowcore->cregs_save_area, 0, 15);
661         atomic_inc(&init_mm.context.attach_count);
662         asm volatile(
663                 "       stam    0,15,0(%0)"
664                 : : "a" (&cpu_lowcore->access_regs_save_area) : "memory");
665         cpu_lowcore->percpu_offset = __per_cpu_offset[cpu];
666         cpu_lowcore->current_task = (unsigned long) idle;
667         cpu_lowcore->cpu_nr = cpu;
668         cpu_lowcore->kernel_asce = S390_lowcore.kernel_asce;
669         cpu_lowcore->machine_flags = S390_lowcore.machine_flags;
670         cpu_lowcore->ftrace_func = S390_lowcore.ftrace_func;
671         memcpy(cpu_lowcore->stfle_fac_list, S390_lowcore.stfle_fac_list,
672                MAX_FACILITY_BIT/8);
673         eieio();
674
675         while (sigp(cpu, sigp_restart) == sigp_busy)
676                 udelay(10);
677
678         while (!cpu_online(cpu))
679                 cpu_relax();
680         return 0;
681
682 err_out:
683         smp_free_lowcore(cpu);
684         return -EIO;
685 }
686
687 static int __init setup_possible_cpus(char *s)
688 {
689         int pcpus, cpu;
690
691         pcpus = simple_strtoul(s, NULL, 0);
692         init_cpu_possible(cpumask_of(0));
693         for (cpu = 1; cpu < pcpus && cpu < nr_cpu_ids; cpu++)
694                 set_cpu_possible(cpu, true);
695         return 0;
696 }
697 early_param("possible_cpus", setup_possible_cpus);
698
699 #ifdef CONFIG_HOTPLUG_CPU
700
701 int __cpu_disable(void)
702 {
703         struct ec_creg_mask_parms cr_parms;
704         int cpu = smp_processor_id();
705
706         set_cpu_online(cpu, false);
707
708         /* Disable pfault pseudo page faults on this cpu. */
709         pfault_fini();
710
711         memset(&cr_parms.orvals, 0, sizeof(cr_parms.orvals));
712         memset(&cr_parms.andvals, 0xff, sizeof(cr_parms.andvals));
713
714         /* disable all external interrupts */
715         cr_parms.orvals[0] = 0;
716         cr_parms.andvals[0] = ~(1 << 15 | 1 << 14 | 1 << 13 | 1 << 11 |
717                                 1 << 10 | 1 <<  9 | 1 <<  6 | 1 <<  5 |
718                                 1 <<  4);
719         /* disable all I/O interrupts */
720         cr_parms.orvals[6] = 0;
721         cr_parms.andvals[6] = ~(1 << 31 | 1 << 30 | 1 << 29 | 1 << 28 |
722                                 1 << 27 | 1 << 26 | 1 << 25 | 1 << 24);
723         /* disable most machine checks */
724         cr_parms.orvals[14] = 0;
725         cr_parms.andvals[14] = ~(1 << 28 | 1 << 27 | 1 << 26 |
726                                  1 << 25 | 1 << 24);
727
728         smp_ctl_bit_callback(&cr_parms);
729
730         return 0;
731 }
732
733 void __cpu_die(unsigned int cpu)
734 {
735         /* Wait until target cpu is down */
736         while (!cpu_stopped(cpu))
737                 cpu_relax();
738         while (sigp_p(0, cpu, sigp_set_prefix) == sigp_busy)
739                 udelay(10);
740         smp_free_lowcore(cpu);
741         atomic_dec(&init_mm.context.attach_count);
742 }
743
744 void __noreturn cpu_die(void)
745 {
746         idle_task_exit();
747         while (sigp(smp_processor_id(), sigp_stop) == sigp_busy)
748                 cpu_relax();
749         for (;;);
750 }
751
752 #endif /* CONFIG_HOTPLUG_CPU */
753
754 void __init smp_prepare_cpus(unsigned int max_cpus)
755 {
756 #ifndef CONFIG_64BIT
757         unsigned long save_area = 0;
758 #endif
759         unsigned long async_stack, panic_stack;
760         struct _lowcore *lowcore;
761
762         smp_detect_cpus();
763
764         /* request the 0x1201 emergency signal external interrupt */
765         if (register_external_interrupt(0x1201, do_ext_call_interrupt) != 0)
766                 panic("Couldn't request external interrupt 0x1201");
767         /* request the 0x1202 external call external interrupt */
768         if (register_external_interrupt(0x1202, do_ext_call_interrupt) != 0)
769                 panic("Couldn't request external interrupt 0x1202");
770
771         /* Reallocate current lowcore, but keep its contents. */
772         lowcore = (void *) __get_free_pages(GFP_KERNEL | GFP_DMA, LC_ORDER);
773         panic_stack = __get_free_page(GFP_KERNEL);
774         async_stack = __get_free_pages(GFP_KERNEL, ASYNC_ORDER);
775         BUG_ON(!lowcore || !panic_stack || !async_stack);
776 #ifndef CONFIG_64BIT
777         if (MACHINE_HAS_IEEE)
778                 save_area = get_zeroed_page(GFP_KERNEL);
779 #endif
780         local_irq_disable();
781         local_mcck_disable();
782         lowcore_ptr[smp_processor_id()] = lowcore;
783         *lowcore = S390_lowcore;
784         lowcore->panic_stack = panic_stack + PAGE_SIZE;
785         lowcore->async_stack = async_stack + ASYNC_SIZE;
786 #ifndef CONFIG_64BIT
787         if (MACHINE_HAS_IEEE)
788                 lowcore->extended_save_area_addr = (u32) save_area;
789 #endif
790         set_prefix((u32)(unsigned long) lowcore);
791         local_mcck_enable();
792         local_irq_enable();
793 #ifdef CONFIG_64BIT
794         if (vdso_alloc_per_cpu(smp_processor_id(), &S390_lowcore))
795                 BUG();
796 #endif
797 }
798
799 void __init smp_prepare_boot_cpu(void)
800 {
801         BUG_ON(smp_processor_id() != 0);
802
803         current_thread_info()->cpu = 0;
804         set_cpu_present(0, true);
805         set_cpu_online(0, true);
806         S390_lowcore.percpu_offset = __per_cpu_offset[0];
807         current_set[0] = current;
808         smp_cpu_state[0] = CPU_STATE_CONFIGURED;
809         smp_cpu_polarization[0] = POLARIZATION_UNKNWN;
810 }
811
812 void __init smp_cpus_done(unsigned int max_cpus)
813 {
814 }
815
816 void __init smp_setup_processor_id(void)
817 {
818         S390_lowcore.cpu_nr = 0;
819         __cpu_logical_map[0] = stap();
820 }
821
822 /*
823  * the frequency of the profiling timer can be changed
824  * by writing a multiplier value into /proc/profile.
825  *
826  * usually you want to run this on all CPUs ;)
827  */
828 int setup_profiling_timer(unsigned int multiplier)
829 {
830         return 0;
831 }
832
833 #ifdef CONFIG_HOTPLUG_CPU
834 static ssize_t cpu_configure_show(struct device *dev,
835                                 struct device_attribute *attr, char *buf)
836 {
837         ssize_t count;
838
839         mutex_lock(&smp_cpu_state_mutex);
840         count = sprintf(buf, "%d\n", smp_cpu_state[dev->id]);
841         mutex_unlock(&smp_cpu_state_mutex);
842         return count;
843 }
844
845 static ssize_t cpu_configure_store(struct device *dev,
846                                   struct device_attribute *attr,
847                                   const char *buf, size_t count)
848 {
849         int cpu = dev->id;
850         int val, rc;
851         char delim;
852
853         if (sscanf(buf, "%d %c", &val, &delim) != 1)
854                 return -EINVAL;
855         if (val != 0 && val != 1)
856                 return -EINVAL;
857
858         get_online_cpus();
859         mutex_lock(&smp_cpu_state_mutex);
860         rc = -EBUSY;
861         /* disallow configuration changes of online cpus and cpu 0 */
862         if (cpu_online(cpu) || cpu == 0)
863                 goto out;
864         rc = 0;
865         switch (val) {
866         case 0:
867                 if (smp_cpu_state[cpu] == CPU_STATE_CONFIGURED) {
868                         rc = sclp_cpu_deconfigure(__cpu_logical_map[cpu]);
869                         if (!rc) {
870                                 smp_cpu_state[cpu] = CPU_STATE_STANDBY;
871                                 smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
872                         }
873                 }
874                 break;
875         case 1:
876                 if (smp_cpu_state[cpu] == CPU_STATE_STANDBY) {
877                         rc = sclp_cpu_configure(__cpu_logical_map[cpu]);
878                         if (!rc) {
879                                 smp_cpu_state[cpu] = CPU_STATE_CONFIGURED;
880                                 smp_cpu_polarization[cpu] = POLARIZATION_UNKNWN;
881                         }
882                 }
883                 break;
884         default:
885                 break;
886         }
887 out:
888         mutex_unlock(&smp_cpu_state_mutex);
889         put_online_cpus();
890         return rc ? rc : count;
891 }
892 static DEVICE_ATTR(configure, 0644, cpu_configure_show, cpu_configure_store);
893 #endif /* CONFIG_HOTPLUG_CPU */
894
895 static ssize_t cpu_polarization_show(struct device *dev,
896                                      struct device_attribute *attr, char *buf)
897 {
898         int cpu = dev->id;
899         ssize_t count;
900
901         mutex_lock(&smp_cpu_state_mutex);
902         switch (smp_cpu_polarization[cpu]) {
903         case POLARIZATION_HRZ:
904                 count = sprintf(buf, "horizontal\n");
905                 break;
906         case POLARIZATION_VL:
907                 count = sprintf(buf, "vertical:low\n");
908                 break;
909         case POLARIZATION_VM:
910                 count = sprintf(buf, "vertical:medium\n");
911                 break;
912         case POLARIZATION_VH:
913                 count = sprintf(buf, "vertical:high\n");
914                 break;
915         default:
916                 count = sprintf(buf, "unknown\n");
917                 break;
918         }
919         mutex_unlock(&smp_cpu_state_mutex);
920         return count;
921 }
922 static DEVICE_ATTR(polarization, 0444, cpu_polarization_show, NULL);
923
924 static ssize_t show_cpu_address(struct device *dev,
925                                 struct device_attribute *attr, char *buf)
926 {
927         return sprintf(buf, "%d\n", __cpu_logical_map[dev->id]);
928 }
929 static DEVICE_ATTR(address, 0444, show_cpu_address, NULL);
930
931
932 static struct attribute *cpu_common_attrs[] = {
933 #ifdef CONFIG_HOTPLUG_CPU
934         &dev_attr_configure.attr,
935 #endif
936         &dev_attr_address.attr,
937         &dev_attr_polarization.attr,
938         NULL,
939 };
940
941 static struct attribute_group cpu_common_attr_group = {
942         .attrs = cpu_common_attrs,
943 };
944
945 static ssize_t show_capability(struct device *dev,
946                                 struct device_attribute *attr, char *buf)
947 {
948         unsigned int capability;
949         int rc;
950
951         rc = get_cpu_capability(&capability);
952         if (rc)
953                 return rc;
954         return sprintf(buf, "%u\n", capability);
955 }
956 static DEVICE_ATTR(capability, 0444, show_capability, NULL);
957
958 static ssize_t show_idle_count(struct device *dev,
959                                 struct device_attribute *attr, char *buf)
960 {
961         struct s390_idle_data *idle;
962         unsigned long long idle_count;
963         unsigned int sequence;
964
965         idle = &per_cpu(s390_idle, dev->id);
966 repeat:
967         sequence = idle->sequence;
968         smp_rmb();
969         if (sequence & 1)
970                 goto repeat;
971         idle_count = idle->idle_count;
972         if (idle->idle_enter)
973                 idle_count++;
974         smp_rmb();
975         if (idle->sequence != sequence)
976                 goto repeat;
977         return sprintf(buf, "%llu\n", idle_count);
978 }
979 static DEVICE_ATTR(idle_count, 0444, show_idle_count, NULL);
980
981 static ssize_t show_idle_time(struct device *dev,
982                                 struct device_attribute *attr, char *buf)
983 {
984         struct s390_idle_data *idle;
985         unsigned long long now, idle_time, idle_enter;
986         unsigned int sequence;
987
988         idle = &per_cpu(s390_idle, dev->id);
989         now = get_clock();
990 repeat:
991         sequence = idle->sequence;
992         smp_rmb();
993         if (sequence & 1)
994                 goto repeat;
995         idle_time = idle->idle_time;
996         idle_enter = idle->idle_enter;
997         if (idle_enter != 0ULL && idle_enter < now)
998                 idle_time += now - idle_enter;
999         smp_rmb();
1000         if (idle->sequence != sequence)
1001                 goto repeat;
1002         return sprintf(buf, "%llu\n", idle_time >> 12);
1003 }
1004 static DEVICE_ATTR(idle_time_us, 0444, show_idle_time, NULL);
1005
1006 static struct attribute *cpu_online_attrs[] = {
1007         &dev_attr_capability.attr,
1008         &dev_attr_idle_count.attr,
1009         &dev_attr_idle_time_us.attr,
1010         NULL,
1011 };
1012
1013 static struct attribute_group cpu_online_attr_group = {
1014         .attrs = cpu_online_attrs,
1015 };
1016
1017 static int __cpuinit smp_cpu_notify(struct notifier_block *self,
1018                                     unsigned long action, void *hcpu)
1019 {
1020         unsigned int cpu = (unsigned int)(long)hcpu;
1021         struct cpu *c = &per_cpu(cpu_devices, cpu);
1022         struct device *s = &c->dev;
1023         struct s390_idle_data *idle;
1024         int err = 0;
1025
1026         switch (action) {
1027         case CPU_ONLINE:
1028         case CPU_ONLINE_FROZEN:
1029                 idle = &per_cpu(s390_idle, cpu);
1030                 memset(idle, 0, sizeof(struct s390_idle_data));
1031                 err = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
1032                 break;
1033         case CPU_DEAD:
1034         case CPU_DEAD_FROZEN:
1035                 sysfs_remove_group(&s->kobj, &cpu_online_attr_group);
1036                 break;
1037         }
1038         return notifier_from_errno(err);
1039 }
1040
1041 static struct notifier_block __cpuinitdata smp_cpu_nb = {
1042         .notifier_call = smp_cpu_notify,
1043 };
1044
1045 static int __devinit smp_add_present_cpu(int cpu)
1046 {
1047         struct cpu *c = &per_cpu(cpu_devices, cpu);
1048         struct device *s = &c->dev;
1049         int rc;
1050
1051         c->hotpluggable = 1;
1052         rc = register_cpu(c, cpu);
1053         if (rc)
1054                 goto out;
1055         rc = sysfs_create_group(&s->kobj, &cpu_common_attr_group);
1056         if (rc)
1057                 goto out_cpu;
1058         if (!cpu_online(cpu))
1059                 goto out;
1060         rc = sysfs_create_group(&s->kobj, &cpu_online_attr_group);
1061         if (!rc)
1062                 return 0;
1063         sysfs_remove_group(&s->kobj, &cpu_common_attr_group);
1064 out_cpu:
1065 #ifdef CONFIG_HOTPLUG_CPU
1066         unregister_cpu(c);
1067 #endif
1068 out:
1069         return rc;
1070 }
1071
1072 #ifdef CONFIG_HOTPLUG_CPU
1073
1074 int __ref smp_rescan_cpus(void)
1075 {
1076         cpumask_t newcpus;
1077         int cpu;
1078         int rc;
1079
1080         get_online_cpus();
1081         mutex_lock(&smp_cpu_state_mutex);
1082         cpumask_copy(&newcpus, cpu_present_mask);
1083         rc = __smp_rescan_cpus();
1084         if (rc)
1085                 goto out;
1086         cpumask_andnot(&newcpus, cpu_present_mask, &newcpus);
1087         for_each_cpu(cpu, &newcpus) {
1088                 rc = smp_add_present_cpu(cpu);
1089                 if (rc)
1090                         set_cpu_present(cpu, false);
1091         }
1092         rc = 0;
1093 out:
1094         mutex_unlock(&smp_cpu_state_mutex);
1095         put_online_cpus();
1096         if (!cpumask_empty(&newcpus))
1097                 topology_schedule_update();
1098         return rc;
1099 }
1100
1101 static ssize_t __ref rescan_store(struct device *dev,
1102                                   struct device_attribute *attr,
1103                                   const char *buf,
1104                                   size_t count)
1105 {
1106         int rc;
1107
1108         rc = smp_rescan_cpus();
1109         return rc ? rc : count;
1110 }
1111 static DEVICE_ATTR(rescan, 0200, NULL, rescan_store);
1112 #endif /* CONFIG_HOTPLUG_CPU */
1113
1114 static ssize_t dispatching_show(struct device *dev,
1115                                 struct device_attribute *attr,
1116                                 char *buf)
1117 {
1118         ssize_t count;
1119
1120         mutex_lock(&smp_cpu_state_mutex);
1121         count = sprintf(buf, "%d\n", cpu_management);
1122         mutex_unlock(&smp_cpu_state_mutex);
1123         return count;
1124 }
1125
1126 static ssize_t dispatching_store(struct device *dev,
1127                                  struct device_attribute *attr,
1128                                  const char *buf,
1129                                  size_t count)
1130 {
1131         int val, rc;
1132         char delim;
1133
1134         if (sscanf(buf, "%d %c", &val, &delim) != 1)
1135                 return -EINVAL;
1136         if (val != 0 && val != 1)
1137                 return -EINVAL;
1138         rc = 0;
1139         get_online_cpus();
1140         mutex_lock(&smp_cpu_state_mutex);
1141         if (cpu_management == val)
1142                 goto out;
1143         rc = topology_set_cpu_management(val);
1144         if (!rc)
1145                 cpu_management = val;
1146 out:
1147         mutex_unlock(&smp_cpu_state_mutex);
1148         put_online_cpus();
1149         return rc ? rc : count;
1150 }
1151 static DEVICE_ATTR(dispatching, 0644, dispatching_show,
1152                          dispatching_store);
1153
1154 static int __init topology_init(void)
1155 {
1156         int cpu;
1157         int rc;
1158
1159         register_cpu_notifier(&smp_cpu_nb);
1160
1161 #ifdef CONFIG_HOTPLUG_CPU
1162         rc = device_create_file(cpu_subsys.dev_root, &dev_attr_rescan);
1163         if (rc)
1164                 return rc;
1165 #endif
1166         rc = device_create_file(cpu_subsys.dev_root, &dev_attr_dispatching);
1167         if (rc)
1168                 return rc;
1169         for_each_present_cpu(cpu) {
1170                 rc = smp_add_present_cpu(cpu);
1171                 if (rc)
1172                         return rc;
1173         }
1174         return 0;
1175 }
1176 subsys_initcall(topology_init);