blob: a137608cd1970737a5a571b5ff2b375d3d01aaae [file] [log] [blame]
Thomas Gleixnerd2912cb2019-06-04 10:11:33 +02001// SPDX-License-Identifier: GPL-2.0-only
Linus Torvalds1da177e2005-04-16 15:20:36 -07002/*
3 * linux/arch/arm/kernel/smp.c
4 *
5 * Copyright (C) 2002 ARM Limited, All Rights Reserved.
Linus Torvalds1da177e2005-04-16 15:20:36 -07006 */
Russell Kingc97d4862006-10-25 13:59:16 +01007#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07008#include <linux/delay.h>
9#include <linux/init.h>
10#include <linux/spinlock.h>
Ingo Molnar68e21be2017-02-01 19:08:20 +010011#include <linux/sched/mm.h>
Ingo Molnaref8bd772017-02-08 18:51:36 +010012#include <linux/sched/hotplug.h>
Ingo Molnar68db0cf2017-02-08 18:51:37 +010013#include <linux/sched/task_stack.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070014#include <linux/interrupt.h>
15#include <linux/cache.h>
16#include <linux/profile.h>
17#include <linux/errno.h>
18#include <linux/mm.h>
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040019#include <linux/err.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020#include <linux/cpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021#include <linux/seq_file.h>
Russell Kingc97d4862006-10-25 13:59:16 +010022#include <linux/irq.h>
Russell King96f0e002014-09-03 23:57:13 +010023#include <linux/nmi.h>
Russell Kingbc282482009-05-17 18:58:34 +010024#include <linux/percpu.h>
25#include <linux/clockchips.h>
Russell King3c030be2010-11-30 11:07:35 +000026#include <linux/completion.h>
Richard Zhaoec971ea52012-09-05 01:08:59 +020027#include <linux/cpufreq.h>
Stephen Boydbf185252013-10-29 20:32:56 +010028#include <linux/irq_work.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
Arun Sharma600634972011-07-26 16:09:06 -070030#include <linux/atomic.h>
Russell King26602162018-05-10 13:00:43 +010031#include <asm/bugs.h>
Marc Zyngierabcee5f2011-09-08 09:06:10 +010032#include <asm/smp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070033#include <asm/cacheflush.h>
34#include <asm/cpu.h>
Russell King42578c82009-06-11 15:35:00 +010035#include <asm/cputype.h>
Jamie Iles5a567d72011-10-08 11:20:42 +010036#include <asm/exception.h>
Will Deacon89038262011-09-30 11:43:29 +010037#include <asm/idmap.h>
Vincent Guittotc9018aa2011-08-08 13:21:59 +010038#include <asm/topology.h>
Russell Kinge65f38e2005-06-18 09:33:31 +010039#include <asm/mmu_context.h>
40#include <asm/pgtable.h>
41#include <asm/pgalloc.h>
Russell King383fb3e2018-07-19 12:21:31 +010042#include <asm/procinfo.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070043#include <asm/processor.h>
Russell King37b05b62010-10-01 15:38:24 +010044#include <asm/sections.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070045#include <asm/tlbflush.h>
46#include <asm/ptrace.h>
Will Deacond6257282011-08-23 22:19:29 +010047#include <asm/smp_plat.h>
Dave Martin4588c342012-02-17 16:54:28 +000048#include <asm/virt.h>
Marc Zyngierabcee5f2011-09-08 09:06:10 +010049#include <asm/mach/arch.h>
Jonathan Austineb083752013-02-22 18:51:30 +000050#include <asm/mpu.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070051
Nicolas Pitre365ec7b2014-07-25 16:05:31 -040052#define CREATE_TRACE_POINTS
53#include <trace/events/ipi.h>
54
Linus Torvalds1da177e2005-04-16 15:20:36 -070055/*
Russell Kinge65f38e2005-06-18 09:33:31 +010056 * as from 2.5, kernels no longer have an init_tasks structure
57 * so we need some other way of telling a new secondary core
58 * where to place its SVC stack
59 */
60struct secondary_data secondary_data;
61
Linus Torvalds1da177e2005-04-16 15:20:36 -070062enum ipi_msg_type {
Stephen Boyd559a5932012-09-19 08:16:07 +010063 IPI_WAKEUP,
64 IPI_TIMER,
Linus Torvalds1da177e2005-04-16 15:20:36 -070065 IPI_RESCHEDULE,
66 IPI_CALL_FUNC,
67 IPI_CPU_STOP,
Stephen Boydbf185252013-10-29 20:32:56 +010068 IPI_IRQ_WORK,
Nicolas Pitre5135d872012-11-27 21:54:41 -050069 IPI_COMPLETION,
Arnd Bergmannbe167862019-05-14 15:41:48 -070070 /*
71 * CPU_BACKTRACE is special and not included in NR_IPI
72 * or tracable with trace_ipi_*
73 */
Marc Zyngiere7273ff2015-12-18 11:06:47 +010074 IPI_CPU_BACKTRACE,
75 /*
76 * SGI8-15 can be reserved by secure firmware, and thus may
77 * not be usable by the kernel. Please keep the above limited
78 * to at most 8 entries.
79 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070080};
81
Russell King149c2412012-01-18 15:59:45 +000082static DECLARE_COMPLETION(cpu_running);
83
Kees Cook76197512016-08-10 22:46:49 +010084static struct smp_operations smp_ops __ro_after_init;
Marc Zyngierabcee5f2011-09-08 09:06:10 +010085
Masahiro Yamada4caa9dd2015-08-26 07:49:11 +010086void __init smp_set_ops(const struct smp_operations *ops)
Marc Zyngierabcee5f2011-09-08 09:06:10 +010087{
88 if (ops)
89 smp_ops = *ops;
90};
91
Cyril Chemparathy4756dcb2012-07-21 15:55:04 -040092static unsigned long get_arch_pgd(pgd_t *pgd)
93{
Russell Kingb2c3e382015-04-04 20:09:46 +010094#ifdef CONFIG_ARM_LPAE
95 return __phys_to_pfn(virt_to_phys(pgd));
96#else
97 return virt_to_phys(pgd);
98#endif
Cyril Chemparathy4756dcb2012-07-21 15:55:04 -040099}
100
Russell King383fb3e2018-07-19 12:21:31 +0100101#if defined(CONFIG_BIG_LITTLE) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR)
102static int secondary_biglittle_prepare(unsigned int cpu)
103{
104 if (!cpu_vtable[cpu])
105 cpu_vtable[cpu] = kzalloc(sizeof(*cpu_vtable[cpu]), GFP_KERNEL);
106
107 return cpu_vtable[cpu] ? 0 : -ENOMEM;
108}
109
110static void secondary_biglittle_init(void)
111{
112 init_proc_vtable(lookup_processor(read_cpuid_id())->proc);
113}
114#else
115static int secondary_biglittle_prepare(unsigned int cpu)
116{
117 return 0;
118}
119
120static void secondary_biglittle_init(void)
121{
122}
123#endif
124
Paul Gortmaker8bd26e32013-06-17 15:43:14 -0400125int __cpu_up(unsigned int cpu, struct task_struct *idle)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700126{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127 int ret;
128
Geert Uytterhoeven084bb5b2014-08-20 20:49:54 +0100129 if (!smp_ops.smp_boot_secondary)
130 return -ENOSYS;
131
Russell King383fb3e2018-07-19 12:21:31 +0100132 ret = secondary_biglittle_prepare(cpu);
133 if (ret)
134 return ret;
135
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136 /*
Russell Kinge65f38e2005-06-18 09:33:31 +0100137 * We need to tell the secondary core where to find
138 * its stack and the page tables.
139 */
Al Viro32d39a92006-01-12 01:05:58 -0800140 secondary_data.stack = task_stack_page(idle) + THREAD_START_SP;
Jonathan Austineb083752013-02-22 18:51:30 +0000141#ifdef CONFIG_ARM_MPU
Vladimir Murzina0995c082017-10-16 12:54:05 +0100142 secondary_data.mpu_rgn_info = &mpu_rgn_info;
Jonathan Austineb083752013-02-22 18:51:30 +0000143#endif
144
Will Deaconc4a1f032012-02-28 13:02:59 +0000145#ifdef CONFIG_MMU
Russell Kingb2c3e382015-04-04 20:09:46 +0100146 secondary_data.pgdir = virt_to_phys(idmap_pgd);
Cyril Chemparathy4756dcb2012-07-21 15:55:04 -0400147 secondary_data.swapper_pg_dir = get_arch_pgd(swapper_pg_dir);
Will Deaconc4a1f032012-02-28 13:02:59 +0000148#endif
Nicolas Pitreefcfc462013-12-09 16:10:18 +0100149 sync_cache_w(&secondary_data);
Russell Kinge65f38e2005-06-18 09:33:31 +0100150
151 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152 * Now bring the CPU into our world.
153 */
Geert Uytterhoeven084bb5b2014-08-20 20:49:54 +0100154 ret = smp_ops.smp_boot_secondary(cpu, idle);
Russell Kinge65f38e2005-06-18 09:33:31 +0100155 if (ret == 0) {
Russell Kinge65f38e2005-06-18 09:33:31 +0100156 /*
157 * CPU was successfully started, wait for it
158 * to come online or time out.
159 */
Russell King149c2412012-01-18 15:59:45 +0000160 wait_for_completion_timeout(&cpu_running,
161 msecs_to_jiffies(1000));
Russell Kinge65f38e2005-06-18 09:33:31 +0100162
Russell King58613cd2010-12-18 12:34:39 +0000163 if (!cpu_online(cpu)) {
164 pr_crit("CPU%u: failed to come online\n", cpu);
Russell Kinge65f38e2005-06-18 09:33:31 +0100165 ret = -EIO;
Russell King58613cd2010-12-18 12:34:39 +0000166 }
167 } else {
168 pr_err("CPU%u: failed to boot: %d\n", cpu, ret);
Russell Kinge65f38e2005-06-18 09:33:31 +0100169 }
170
Russell Kinge65f38e2005-06-18 09:33:31 +0100171
Jonathan Austineb083752013-02-22 18:51:30 +0000172 memset(&secondary_data, 0, sizeof(secondary_data));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173 return ret;
174}
175
Marc Zyngierabcee5f2011-09-08 09:06:10 +0100176/* platform specific SMP operations */
Marc Zyngierac6c7992011-09-27 14:48:23 +0100177void __init smp_init_cpus(void)
Marc Zyngierabcee5f2011-09-08 09:06:10 +0100178{
179 if (smp_ops.smp_init_cpus)
180 smp_ops.smp_init_cpus();
181}
182
Geert Uytterhoevenfee3fd42015-04-01 13:36:57 +0100183int platform_can_secondary_boot(void)
184{
185 return !!smp_ops.smp_boot_secondary;
186}
187
Stephen Warren2103f6c2013-08-02 20:52:49 +0100188int platform_can_cpu_hotplug(void)
189{
190#ifdef CONFIG_HOTPLUG_CPU
191 if (smp_ops.cpu_kill)
192 return 1;
193#endif
194
195 return 0;
196}
197
Russell Kinga054a812005-11-02 22:24:33 +0000198#ifdef CONFIG_HOTPLUG_CPU
Marc Zyngierac6c7992011-09-27 14:48:23 +0100199static int platform_cpu_kill(unsigned int cpu)
Marc Zyngierabcee5f2011-09-08 09:06:10 +0100200{
201 if (smp_ops.cpu_kill)
202 return smp_ops.cpu_kill(cpu);
203 return 1;
204}
205
Marc Zyngierac6c7992011-09-27 14:48:23 +0100206static int platform_cpu_disable(unsigned int cpu)
Marc Zyngierabcee5f2011-09-08 09:06:10 +0100207{
208 if (smp_ops.cpu_disable)
209 return smp_ops.cpu_disable(cpu);
210
Stephen Boyd787047e2015-07-29 00:34:48 +0100211 return 0;
212}
213
214int platform_can_hotplug_cpu(unsigned int cpu)
215{
216 /* cpu_die must be specified to support hotplug */
217 if (!smp_ops.cpu_die)
218 return 0;
219
220 if (smp_ops.cpu_can_disable)
221 return smp_ops.cpu_can_disable(cpu);
222
Marc Zyngierabcee5f2011-09-08 09:06:10 +0100223 /*
224 * By default, allow disabling all CPUs except the first one,
225 * since this is special on a lot of platforms, e.g. because
226 * of clock tick interrupts.
227 */
Stephen Boyd787047e2015-07-29 00:34:48 +0100228 return cpu != 0;
Marc Zyngierabcee5f2011-09-08 09:06:10 +0100229}
Stephen Boyd787047e2015-07-29 00:34:48 +0100230
Russell Kinga054a812005-11-02 22:24:33 +0000231/*
232 * __cpu_disable runs on the processor to be shutdown.
233 */
Paul Gortmaker8bd26e32013-06-17 15:43:14 -0400234int __cpu_disable(void)
Russell Kinga054a812005-11-02 22:24:33 +0000235{
236 unsigned int cpu = smp_processor_id();
Russell Kinga054a812005-11-02 22:24:33 +0000237 int ret;
238
Russell King8e2a43f2010-05-15 10:18:05 +0100239 ret = platform_cpu_disable(cpu);
Russell Kinga054a812005-11-02 22:24:33 +0000240 if (ret)
241 return ret;
242
243 /*
244 * Take this CPU offline. Once we clear this, we can't return,
245 * and we must not schedule until we're ready to give up the cpu.
246 */
Russell Kinge03cdad2009-05-28 14:16:52 +0100247 set_cpu_online(cpu, false);
Russell Kinga054a812005-11-02 22:24:33 +0000248
249 /*
250 * OK - migrate IRQs away from this CPU
251 */
Dietmar Eggemann1b5ba352019-01-21 14:42:42 +0100252 irq_migrate_all_off_this_cpu();
Russell Kinga054a812005-11-02 22:24:33 +0000253
254 /*
255 * Flush user cache and TLB mappings, and then remove this CPU
256 * from the vm mask set of all processes.
Lorenzo Pieralisie6b866e2012-09-07 11:09:15 +0530257 *
258 * Caches are flushed to the Level of Unification Inner Shareable
259 * to write-back dirty lines to unified caches shared by all CPUs.
Russell Kinga054a812005-11-02 22:24:33 +0000260 */
Lorenzo Pieralisie6b866e2012-09-07 11:09:15 +0530261 flush_cache_louis();
Russell Kinga054a812005-11-02 22:24:33 +0000262 local_flush_tlb_all();
263
Russell Kinga054a812005-11-02 22:24:33 +0000264 return 0;
265}
266
Russell King3c030be2010-11-30 11:07:35 +0000267static DECLARE_COMPLETION(cpu_died);
268
Russell Kinga054a812005-11-02 22:24:33 +0000269/*
270 * called on the thread which is asking for a CPU to be shutdown -
271 * waits until shutdown has completed, or it is timed out.
272 */
Paul Gortmaker8bd26e32013-06-17 15:43:14 -0400273void __cpu_die(unsigned int cpu)
Russell Kinga054a812005-11-02 22:24:33 +0000274{
Russell King3c030be2010-11-30 11:07:35 +0000275 if (!wait_for_completion_timeout(&cpu_died, msecs_to_jiffies(5000))) {
276 pr_err("CPU%u: cpu didn't die\n", cpu);
277 return;
278 }
Florian Fainelli035e7872017-01-19 01:26:28 +0100279 pr_debug("CPU%u: shutdown\n", cpu);
Russell King3c030be2010-11-30 11:07:35 +0000280
Grygorii Strashko98f1b5a2018-05-08 15:19:40 +0100281 clear_tasks_mm_cpumask(cpu);
Russell King51acdfd2013-04-18 18:05:29 +0100282 /*
283 * platform_cpu_kill() is generally expected to do the powering off
284 * and/or cutting of clocks to the dying CPU. Optionally, this may
285 * be done by the CPU which is dying in preference to supporting
286 * this call, but that means there is _no_ synchronisation between
287 * the requesting CPU and the dying CPU actually losing power.
288 */
Russell Kinga054a812005-11-02 22:24:33 +0000289 if (!platform_cpu_kill(cpu))
Russell King4ed89f22014-10-28 11:26:42 +0000290 pr_err("CPU%u: unable to kill\n", cpu);
Russell Kinga054a812005-11-02 22:24:33 +0000291}
292
293/*
294 * Called from the idle thread for the CPU which has been shutdown.
295 *
296 * Note that we disable IRQs here, but do not re-enable them
297 * before returning to the caller. This is also the behaviour
298 * of the other hotplug-cpu capable cores, so presumably coming
299 * out of idle fixes this.
300 */
Stephen Boyd9205b792015-08-24 21:49:30 +0100301void arch_cpu_idle_dead(void)
Russell Kinga054a812005-11-02 22:24:33 +0000302{
303 unsigned int cpu = smp_processor_id();
304
Russell Kinga054a812005-11-02 22:24:33 +0000305 idle_task_exit();
306
Russell Kingf36d3402010-11-30 12:21:30 +0000307 local_irq_disable();
Russell Kingf36d3402010-11-30 12:21:30 +0000308
Russell King51acdfd2013-04-18 18:05:29 +0100309 /*
310 * Flush the data out of the L1 cache for this CPU. This must be
311 * before the completion to ensure that data is safely written out
312 * before platform_cpu_kill() gets called - which may disable
313 * *this* CPU and power down its cache.
314 */
315 flush_cache_louis();
316
317 /*
318 * Tell __cpu_die() that this CPU is now safe to dispose of. Once
319 * this returns, power and/or clocks can be removed at any point
320 * from this CPU and its cache by platform_cpu_kill().
321 */
Stephen Boydaa033812013-05-20 17:57:16 -0700322 complete(&cpu_died);
Russell King3c030be2010-11-30 11:07:35 +0000323
Russell Kinga054a812005-11-02 22:24:33 +0000324 /*
Russell King51acdfd2013-04-18 18:05:29 +0100325 * Ensure that the cache lines associated with that completion are
326 * written out. This covers the case where _this_ CPU is doing the
327 * powering down, to ensure that the completion is visible to the
328 * CPU waiting for this one.
329 */
330 flush_cache_louis();
331
332 /*
333 * The actual CPU shutdown procedure is at least platform (if not
334 * CPU) specific. This may remove power, or it may simply spin.
335 *
336 * Platforms are generally expected *NOT* to return from this call,
337 * although there are some which do because they have no way to
338 * power down the CPU. These platforms are the _only_ reason we
339 * have a return path which uses the fragment of assembly below.
340 *
341 * The return path should not be used for platforms which can
342 * power off the CPU.
Russell Kinga054a812005-11-02 22:24:33 +0000343 */
Russell King0a301112013-01-14 15:54:28 +0000344 if (smp_ops.cpu_die)
345 smp_ops.cpu_die(cpu);
Russell Kinga054a812005-11-02 22:24:33 +0000346
Russell King668bc382014-01-11 11:25:37 +0000347 pr_warn("CPU%u: smp_ops.cpu_die() returned, trying to resuscitate\n",
348 cpu);
349
Russell Kinga054a812005-11-02 22:24:33 +0000350 /*
351 * Do not return to the idle loop - jump back to the secondary
352 * cpu initialisation. There's some initialisation which needs
353 * to be repeated to undo the effects of taking the CPU offline.
354 */
355 __asm__("mov sp, %0\n"
Russell Kingfaabfa02010-12-20 16:58:19 +0000356 " mov fp, #0\n"
Russell Kinga054a812005-11-02 22:24:33 +0000357 " b secondary_start_kernel"
358 :
Al Viro32d39a92006-01-12 01:05:58 -0800359 : "r" (task_stack_page(current) + THREAD_SIZE - 8));
Russell Kinga054a812005-11-02 22:24:33 +0000360}
361#endif /* CONFIG_HOTPLUG_CPU */
362
Linus Torvalds1da177e2005-04-16 15:20:36 -0700363/*
Russell King05c74a62010-12-03 11:09:48 +0000364 * Called by both boot and secondaries to move global data into
365 * per-processor storage.
366 */
Paul Gortmaker8bd26e32013-06-17 15:43:14 -0400367static void smp_store_cpu_info(unsigned int cpuid)
Russell King05c74a62010-12-03 11:09:48 +0000368{
369 struct cpuinfo_arm *cpu_info = &per_cpu(cpu_data, cpuid);
370
371 cpu_info->loops_per_jiffy = loops_per_jiffy;
Lorenzo Pieralisie8d432c2012-11-06 11:57:43 +0000372 cpu_info->cpuid = read_cpuid_id();
Vincent Guittotc9018aa2011-08-08 13:21:59 +0100373
374 store_cpu_topology(cpuid);
Russell King05c74a62010-12-03 11:09:48 +0000375}
376
377/*
Russell Kinge65f38e2005-06-18 09:33:31 +0100378 * This is the secondary CPU boot entry. We're using this CPUs
379 * idle thread stack, but a set of temporary page tables.
380 */
Paul Gortmaker8bd26e32013-06-17 15:43:14 -0400381asmlinkage void secondary_start_kernel(void)
Russell Kinge65f38e2005-06-18 09:33:31 +0100382{
383 struct mm_struct *mm = &init_mm;
Will Deacon5f40b902012-10-19 17:53:01 +0100384 unsigned int cpu;
385
Russell King383fb3e2018-07-19 12:21:31 +0100386 secondary_biglittle_init();
387
Will Deacon5f40b902012-10-19 17:53:01 +0100388 /*
389 * The identity mapping is uncached (strongly ordered), so
390 * switch away from it before attempting any exclusive accesses.
391 */
392 cpu_switch_mm(mm->pgd, mm);
Will Deacon89c7e4b2013-02-28 17:48:40 +0100393 local_flush_bp_all();
Will Deacon5f40b902012-10-19 17:53:01 +0100394 enter_lazy_tlb(mm, current);
395 local_flush_tlb_all();
Russell Kinge65f38e2005-06-18 09:33:31 +0100396
Russell Kinge65f38e2005-06-18 09:33:31 +0100397 /*
398 * All kernel threads share the same mm context; grab a
399 * reference and switch to it.
400 */
Will Deacon5f40b902012-10-19 17:53:01 +0100401 cpu = smp_processor_id();
Vegard Nossumf1f10072017-02-27 14:30:07 -0800402 mmgrab(mm);
Russell Kinge65f38e2005-06-18 09:33:31 +0100403 current->active_mm = mm;
Rusty Russell56f8ba82009-09-24 09:34:49 -0600404 cpumask_set_cpu(cpu, mm_cpumask(mm));
Russell Kinge65f38e2005-06-18 09:33:31 +0100405
Rob Herring14318efb2012-11-29 20:39:54 +0100406 cpu_init();
407
Vladimir Murzin62d1c952017-12-27 10:38:55 +0100408#ifndef CONFIG_MMU
409 setup_vectors_base();
410#endif
Russell Kingc68b0272014-10-28 12:08:34 +0000411 pr_debug("CPU%u: Booted secondary processor\n", cpu);
Colin Crossfde165b2012-05-05 20:58:13 +0100412
Nick Piggin5bfb5d62005-11-08 21:39:01 -0800413 preempt_disable();
Russell King2c0136d2010-12-03 15:00:49 +0000414 trace_hardirqs_off();
Russell Kinge65f38e2005-06-18 09:33:31 +0100415
416 /*
417 * Give the platform a chance to do its own initialisation.
418 */
Russell King0a301112013-01-14 15:54:28 +0000419 if (smp_ops.smp_secondary_init)
420 smp_ops.smp_secondary_init(cpu);
Russell Kinge65f38e2005-06-18 09:33:31 +0100421
Manfred Spraule545a612008-09-07 16:57:22 +0200422 notify_cpu_starting(cpu);
Catalin Marinasa8655e82008-02-04 17:30:57 +0100423
Russell Kinge65f38e2005-06-18 09:33:31 +0100424 calibrate_delay();
425
426 smp_store_cpu_info(cpu);
427
428 /*
Russell King573619d2011-06-20 16:46:01 +0100429 * OK, now it's safe to let the boot CPU continue. Wait for
430 * the CPU migration code to notice that the CPU is online
Russell King149c2412012-01-18 15:59:45 +0000431 * before we continue - which happens after __cpu_up returns.
Russell Kinge65f38e2005-06-18 09:33:31 +0100432 */
Russell Kinge03cdad2009-05-28 14:16:52 +0100433 set_cpu_online(cpu, true);
Russell King26602162018-05-10 13:00:43 +0100434
435 check_other_bugs();
436
Russell King149c2412012-01-18 15:59:45 +0000437 complete(&cpu_running);
Thomas Gleinxereb047452011-10-14 12:44:41 +0100438
Thomas Gleinxereb047452011-10-14 12:44:41 +0100439 local_irq_enable();
440 local_fiq_enable();
Lucas Stachbbeb9202015-08-25 13:52:09 +0100441 local_abt_enable();
Thomas Gleinxereb047452011-10-14 12:44:41 +0100442
443 /*
Russell Kinge65f38e2005-06-18 09:33:31 +0100444 * OK, it's off to the idle thread for us
445 */
Thomas Gleixnerfc6d73d2016-02-26 18:43:40 +0000446 cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
Russell Kinge65f38e2005-06-18 09:33:31 +0100447}
448
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449void __init smp_cpus_done(unsigned int max_cpus)
450{
Pavel Machek4bf96362015-01-04 20:01:23 +0100451 int cpu;
452 unsigned long bogosum = 0;
453
454 for_each_online_cpu(cpu)
455 bogosum += per_cpu(cpu_data, cpu).loops_per_jiffy;
456
457 printk(KERN_INFO "SMP: Total of %d processors activated "
458 "(%lu.%02lu BogoMIPS).\n",
459 num_online_cpus(),
460 bogosum / (500000/HZ),
461 (bogosum / (5000/HZ)) % 100);
462
Dave Martin4588c342012-02-17 16:54:28 +0000463 hyp_mode_check();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700464}
465
466void __init smp_prepare_boot_cpu(void)
467{
Rob Herring14318efb2012-11-29 20:39:54 +0100468 set_my_cpu_offset(per_cpu_offset(smp_processor_id()));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700469}
470
Russell King05c74a62010-12-03 11:09:48 +0000471void __init smp_prepare_cpus(unsigned int max_cpus)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472{
Russell King05c74a62010-12-03 11:09:48 +0000473 unsigned int ncores = num_possible_cpus();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700474
Vincent Guittotc9018aa2011-08-08 13:21:59 +0100475 init_cpu_topology();
476
Russell King05c74a62010-12-03 11:09:48 +0000477 smp_store_cpu_info(smp_processor_id());
Linus Torvalds1da177e2005-04-16 15:20:36 -0700478
479 /*
Russell King05c74a62010-12-03 11:09:48 +0000480 * are we trying to boot more cores than exist?
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481 */
Russell King05c74a62010-12-03 11:09:48 +0000482 if (max_cpus > ncores)
483 max_cpus = ncores;
Stephen Boyd7fa22bd2011-07-07 01:56:51 +0100484 if (ncores > 1 && max_cpus) {
Russell King05c74a62010-12-03 11:09:48 +0000485 /*
Stephen Boyd7fa22bd2011-07-07 01:56:51 +0100486 * Initialise the present map, which describes the set of CPUs
487 * actually populated at the present time. A platform should
Russell King0a301112013-01-14 15:54:28 +0000488 * re-initialize the map in the platforms smp_prepare_cpus()
489 * if present != possible (e.g. physical hotplug).
Stephen Boyd7fa22bd2011-07-07 01:56:51 +0100490 */
Rusty Russell0b5f9c02012-03-29 15:38:30 +1030491 init_cpu_present(cpu_possible_mask);
Stephen Boyd7fa22bd2011-07-07 01:56:51 +0100492
493 /*
Russell King05c74a62010-12-03 11:09:48 +0000494 * Initialise the SCU if there are more than one CPU
495 * and let them know where to start.
496 */
Russell King0a301112013-01-14 15:54:28 +0000497 if (smp_ops.smp_prepare_cpus)
498 smp_ops.smp_prepare_cpus(max_cpus);
Russell King05c74a62010-12-03 11:09:48 +0000499 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700500}
501
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400502static void (*__smp_cross_call)(const struct cpumask *, unsigned int);
Russell King0f7b3322011-04-03 13:01:30 +0100503
504void __init set_smp_cross_call(void (*fn)(const struct cpumask *, unsigned int))
505{
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400506 if (!__smp_cross_call)
507 __smp_cross_call = fn;
Russell King0f7b3322011-04-03 13:01:30 +0100508}
509
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400510static const char *ipi_types[NR_IPI] __tracepoint_string = {
Stephen Boyd559a5932012-09-19 08:16:07 +0100511#define S(x,s) [x] = s
512 S(IPI_WAKEUP, "CPU wakeup interrupts"),
Russell King4a88abd2010-11-15 14:40:29 +0000513 S(IPI_TIMER, "Timer broadcast interrupts"),
514 S(IPI_RESCHEDULE, "Rescheduling interrupts"),
515 S(IPI_CALL_FUNC, "Function call interrupts"),
Russell King4a88abd2010-11-15 14:40:29 +0000516 S(IPI_CPU_STOP, "CPU stop interrupts"),
Stephen Boydbf185252013-10-29 20:32:56 +0100517 S(IPI_IRQ_WORK, "IRQ work interrupts"),
Nicolas Pitre5135d872012-11-27 21:54:41 -0500518 S(IPI_COMPLETION, "completion interrupts"),
Russell King4a88abd2010-11-15 14:40:29 +0000519};
520
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400521static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
522{
Paul E. McKenney7c64cc02016-04-26 10:42:25 -0700523 trace_ipi_raise_rcuidle(target, ipi_types[ipinr]);
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400524 __smp_cross_call(target, ipinr);
525}
526
Russell Kingf13cd412010-11-15 14:33:51 +0000527void show_ipi_list(struct seq_file *p, int prec)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528{
Russell King4a88abd2010-11-15 14:40:29 +0000529 unsigned int cpu, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700530
Russell King4a88abd2010-11-15 14:40:29 +0000531 for (i = 0; i < NR_IPI; i++) {
532 seq_printf(p, "%*s%u: ", prec - 1, "IPI", i);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533
Nicolas Pitre026b7c62012-12-03 21:13:03 +0100534 for_each_online_cpu(cpu)
Russell King4a88abd2010-11-15 14:40:29 +0000535 seq_printf(p, "%10u ",
536 __get_irq_stat(cpu, ipi_irqs[i]));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700537
Russell King4a88abd2010-11-15 14:40:29 +0000538 seq_printf(p, " %s\n", ipi_types[i]);
539 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540}
541
Russell Kingb54992f2010-11-15 14:46:46 +0000542u64 smp_irq_stat_cpu(unsigned int cpu)
Russell King37ee16a2005-11-08 19:08:05 +0000543{
Russell Kingb54992f2010-11-15 14:46:46 +0000544 u64 sum = 0;
545 int i;
Russell King37ee16a2005-11-08 19:08:05 +0000546
Russell Kingb54992f2010-11-15 14:46:46 +0000547 for (i = 0; i < NR_IPI; i++)
548 sum += __get_irq_stat(cpu, ipi_irqs[i]);
Russell King37ee16a2005-11-08 19:08:05 +0000549
Russell Kingb54992f2010-11-15 14:46:46 +0000550 return sum;
Russell King37ee16a2005-11-08 19:08:05 +0000551}
552
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400553void arch_send_call_function_ipi_mask(const struct cpumask *mask)
554{
555 smp_cross_call(mask, IPI_CALL_FUNC);
556}
557
558void arch_send_wakeup_ipi_mask(const struct cpumask *mask)
559{
560 smp_cross_call(mask, IPI_WAKEUP);
561}
562
563void arch_send_call_function_single_ipi(int cpu)
564{
Marc Zyngier89d798b2015-12-18 11:06:19 +0100565 smp_cross_call(cpumask_of(cpu), IPI_CALL_FUNC);
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400566}
567
568#ifdef CONFIG_IRQ_WORK
569void arch_irq_work_raise(void)
570{
Frederic Weisbecker09f6edd2014-08-16 18:47:53 +0200571 if (arch_irq_work_has_interrupt())
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400572 smp_cross_call(cpumask_of(smp_processor_id()), IPI_IRQ_WORK);
573}
574#endif
575
Russell Kingbc282482009-05-17 18:58:34 +0100576#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
Mark Rutland3d067702012-10-30 12:13:42 +0000577void tick_broadcast(const struct cpumask *mask)
Russell Kingbc282482009-05-17 18:58:34 +0100578{
Russell Kinge3fbb082010-12-20 14:47:19 +0000579 smp_cross_call(mask, IPI_TIMER);
Russell Kingbc282482009-05-17 18:58:34 +0100580}
Russell King5388a6b2010-07-26 13:19:43 +0100581#endif
Russell Kingbc282482009-05-17 18:58:34 +0100582
Thomas Gleixnerbd31b852009-07-03 08:44:46 -0500583static DEFINE_RAW_SPINLOCK(stop_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700584
585/*
586 * ipi_cpu_stop - handle IPI from smp_send_stop()
587 */
588static void ipi_cpu_stop(unsigned int cpu)
589{
Thomas Gleixner5976a662017-05-16 20:42:33 +0200590 if (system_state <= SYSTEM_RUNNING) {
Thomas Gleixnerbd31b852009-07-03 08:44:46 -0500591 raw_spin_lock(&stop_lock);
Russell King4ed89f22014-10-28 11:26:42 +0000592 pr_crit("CPU%u: stopping\n", cpu);
Russell King3d3f78d2010-07-26 13:31:27 +0100593 dump_stack();
Thomas Gleixnerbd31b852009-07-03 08:44:46 -0500594 raw_spin_unlock(&stop_lock);
Russell King3d3f78d2010-07-26 13:31:27 +0100595 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700596
Russell Kinge03cdad2009-05-28 14:16:52 +0100597 set_cpu_online(cpu, false);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700598
599 local_fiq_disable();
600 local_irq_disable();
601
Russell King5388a5b2018-04-10 11:35:36 +0100602 while (1) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700603 cpu_relax();
Russell King5388a5b2018-04-10 11:35:36 +0100604 wfe();
605 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700606}
607
Nicolas Pitre5135d872012-11-27 21:54:41 -0500608static DEFINE_PER_CPU(struct completion *, cpu_completion);
609
610int register_ipi_completion(struct completion *completion, int cpu)
611{
612 per_cpu(cpu_completion, cpu) = completion;
613 return IPI_COMPLETION;
614}
615
616static void ipi_complete(unsigned int cpu)
617{
618 complete(per_cpu(cpu_completion, cpu));
619}
620
Linus Torvalds1da177e2005-04-16 15:20:36 -0700621/*
622 * Main handler for inter-processor interrupts
Linus Torvalds1da177e2005-04-16 15:20:36 -0700623 */
Russell King40737232011-01-06 22:32:52 +0000624asmlinkage void __exception_irq_entry do_IPI(int ipinr, struct pt_regs *regs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625{
Shawn Guo0b5a1b952011-10-06 15:18:14 +0100626 handle_IPI(ipinr, regs);
627}
628
629void handle_IPI(int ipinr, struct pt_regs *regs)
630{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 unsigned int cpu = smp_processor_id();
Russell Kingc97d4862006-10-25 13:59:16 +0100632 struct pt_regs *old_regs = set_irq_regs(regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700633
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400634 if ((unsigned)ipinr < NR_IPI) {
Stephen Boyd398f7452015-06-19 21:37:56 +0100635 trace_ipi_entry_rcuidle(ipi_types[ipinr]);
Stephen Boyd559a5932012-09-19 08:16:07 +0100636 __inc_irq_stat(cpu, ipi_irqs[ipinr]);
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400637 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638
Russell King24480d92010-11-15 09:54:18 +0000639 switch (ipinr) {
Stephen Boyd559a5932012-09-19 08:16:07 +0100640 case IPI_WAKEUP:
641 break;
642
Mark Rutlande2c50112012-10-30 11:17:01 +0000643#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
Russell King24480d92010-11-15 09:54:18 +0000644 case IPI_TIMER:
Russell King7deabca2012-01-19 15:20:58 +0000645 irq_enter();
Mark Rutlande2c50112012-10-30 11:17:01 +0000646 tick_receive_broadcast();
Russell King7deabca2012-01-19 15:20:58 +0000647 irq_exit();
Russell King24480d92010-11-15 09:54:18 +0000648 break;
Mark Rutlande2c50112012-10-30 11:17:01 +0000649#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650
Russell King24480d92010-11-15 09:54:18 +0000651 case IPI_RESCHEDULE:
Peter Zijlstra184748c2011-04-05 17:23:39 +0200652 scheduler_ipi();
Russell King24480d92010-11-15 09:54:18 +0000653 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654
Russell King24480d92010-11-15 09:54:18 +0000655 case IPI_CALL_FUNC:
Russell King7deabca2012-01-19 15:20:58 +0000656 irq_enter();
Russell King24480d92010-11-15 09:54:18 +0000657 generic_smp_call_function_interrupt();
Russell King7deabca2012-01-19 15:20:58 +0000658 irq_exit();
Russell King24480d92010-11-15 09:54:18 +0000659 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660
Russell King24480d92010-11-15 09:54:18 +0000661 case IPI_CPU_STOP:
Russell King7deabca2012-01-19 15:20:58 +0000662 irq_enter();
Russell King24480d92010-11-15 09:54:18 +0000663 ipi_cpu_stop(cpu);
Russell King7deabca2012-01-19 15:20:58 +0000664 irq_exit();
Russell King24480d92010-11-15 09:54:18 +0000665 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666
Stephen Boydbf185252013-10-29 20:32:56 +0100667#ifdef CONFIG_IRQ_WORK
668 case IPI_IRQ_WORK:
669 irq_enter();
670 irq_work_run();
671 irq_exit();
672 break;
673#endif
674
Nicolas Pitre5135d872012-11-27 21:54:41 -0500675 case IPI_COMPLETION:
676 irq_enter();
677 ipi_complete(cpu);
678 irq_exit();
679 break;
680
Russell King96f0e002014-09-03 23:57:13 +0100681 case IPI_CPU_BACKTRACE:
Petr Mladek42a0bb32016-05-20 17:00:33 -0700682 printk_nmi_enter();
Russell King96f0e002014-09-03 23:57:13 +0100683 irq_enter();
684 nmi_cpu_backtrace(regs);
685 irq_exit();
Petr Mladek42a0bb32016-05-20 17:00:33 -0700686 printk_nmi_exit();
Russell King96f0e002014-09-03 23:57:13 +0100687 break;
688
Russell King24480d92010-11-15 09:54:18 +0000689 default:
Russell King4ed89f22014-10-28 11:26:42 +0000690 pr_crit("CPU%u: Unknown IPI message 0x%x\n",
691 cpu, ipinr);
Russell King24480d92010-11-15 09:54:18 +0000692 break;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700693 }
Nicolas Pitre365ec7b2014-07-25 16:05:31 -0400694
695 if ((unsigned)ipinr < NR_IPI)
Stephen Boyd398f7452015-06-19 21:37:56 +0100696 trace_ipi_exit_rcuidle(ipi_types[ipinr]);
Russell Kingc97d4862006-10-25 13:59:16 +0100697 set_irq_regs(old_regs);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698}
699
700void smp_send_reschedule(int cpu)
701{
Russell Kinge3fbb082010-12-20 14:47:19 +0000702 smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700703}
704
Linus Torvalds1da177e2005-04-16 15:20:36 -0700705void smp_send_stop(void)
706{
Russell King28e18292010-12-02 09:53:54 +0000707 unsigned long timeout;
Will Deacon6fa99b72012-04-27 12:51:43 +0100708 struct cpumask mask;
Russell King28e18292010-12-02 09:53:54 +0000709
Will Deacon6fa99b72012-04-27 12:51:43 +0100710 cpumask_copy(&mask, cpu_online_mask);
711 cpumask_clear_cpu(smp_processor_id(), &mask);
Javier Martinez Canillasc5dff4f2012-07-28 15:19:55 +0100712 if (!cpumask_empty(&mask))
713 smp_cross_call(&mask, IPI_CPU_STOP);
Russell King28e18292010-12-02 09:53:54 +0000714
715 /* Wait up to one second for other CPUs to stop */
716 timeout = USEC_PER_SEC;
717 while (num_online_cpus() > 1 && timeout--)
718 udelay(1);
719
720 if (num_online_cpus() > 1)
Joe Perches8b521cb2014-09-16 20:41:43 +0100721 pr_warn("SMP: failed to stop secondary CPUs\n");
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722}
723
Yufen Wang82c08c32018-11-02 11:51:31 +0100724/* In case panic() and panic() called at the same time on CPU1 and CPU2,
725 * and CPU 1 calls panic_smp_self_stop() before crash_smp_send_stop()
726 * CPU1 can't receive the ipi irqs from CPU2, CPU1 will be always online,
727 * kdump fails. So split out the panic_smp_self_stop() and add
728 * set_cpu_online(smp_processor_id(), false).
729 */
730void panic_smp_self_stop(void)
731{
732 pr_debug("CPU %u will stop doing anything useful since another CPU has paniced\n",
733 smp_processor_id());
734 set_cpu_online(smp_processor_id(), false);
735 while (1)
736 cpu_relax();
737}
738
Linus Torvalds1da177e2005-04-16 15:20:36 -0700739/*
740 * not supported here
741 */
Russell King5048bcb2007-07-23 12:59:46 +0100742int setup_profiling_timer(unsigned int multiplier)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700743{
744 return -EINVAL;
745}
Richard Zhaoec971ea52012-09-05 01:08:59 +0200746
747#ifdef CONFIG_CPU_FREQ
748
749static DEFINE_PER_CPU(unsigned long, l_p_j_ref);
750static DEFINE_PER_CPU(unsigned long, l_p_j_ref_freq);
751static unsigned long global_l_p_j_ref;
752static unsigned long global_l_p_j_ref_freq;
753
754static int cpufreq_callback(struct notifier_block *nb,
755 unsigned long val, void *data)
756{
757 struct cpufreq_freqs *freq = data;
Viresh Kumardf240142019-04-29 15:03:58 +0530758 struct cpumask *cpus = freq->policy->cpus;
759 int cpu, first = cpumask_first(cpus);
760 unsigned int lpj;
Richard Zhaoec971ea52012-09-05 01:08:59 +0200761
762 if (freq->flags & CPUFREQ_CONST_LOOPS)
763 return NOTIFY_OK;
764
Viresh Kumardf240142019-04-29 15:03:58 +0530765 if (!per_cpu(l_p_j_ref, first)) {
766 for_each_cpu(cpu, cpus) {
767 per_cpu(l_p_j_ref, cpu) =
768 per_cpu(cpu_data, cpu).loops_per_jiffy;
769 per_cpu(l_p_j_ref_freq, cpu) = freq->old;
770 }
771
Richard Zhaoec971ea52012-09-05 01:08:59 +0200772 if (!global_l_p_j_ref) {
773 global_l_p_j_ref = loops_per_jiffy;
774 global_l_p_j_ref_freq = freq->old;
775 }
776 }
777
778 if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) ||
Viresh Kumar0b443ea2014-03-19 11:24:58 +0530779 (val == CPUFREQ_POSTCHANGE && freq->old > freq->new)) {
Richard Zhaoec971ea52012-09-05 01:08:59 +0200780 loops_per_jiffy = cpufreq_scale(global_l_p_j_ref,
781 global_l_p_j_ref_freq,
782 freq->new);
Viresh Kumardf240142019-04-29 15:03:58 +0530783
784 lpj = cpufreq_scale(per_cpu(l_p_j_ref, first),
785 per_cpu(l_p_j_ref_freq, first), freq->new);
786 for_each_cpu(cpu, cpus)
787 per_cpu(cpu_data, cpu).loops_per_jiffy = lpj;
Richard Zhaoec971ea52012-09-05 01:08:59 +0200788 }
789 return NOTIFY_OK;
790}
791
792static struct notifier_block cpufreq_notifier = {
793 .notifier_call = cpufreq_callback,
794};
795
796static int __init register_cpufreq_notifier(void)
797{
798 return cpufreq_register_notifier(&cpufreq_notifier,
799 CPUFREQ_TRANSITION_NOTIFIER);
800}
801core_initcall(register_cpufreq_notifier);
802
803#endif
Russell King96f0e002014-09-03 23:57:13 +0100804
805static void raise_nmi(cpumask_t *mask)
806{
Arnd Bergmannbe167862019-05-14 15:41:48 -0700807 __smp_cross_call(mask, IPI_CPU_BACKTRACE);
Russell King96f0e002014-09-03 23:57:13 +0100808}
809
Chris Metcalf9a01c3e2016-10-07 17:02:45 -0700810void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self)
Russell King96f0e002014-09-03 23:57:13 +0100811{
Chris Metcalf9a01c3e2016-10-07 17:02:45 -0700812 nmi_trigger_cpumask_backtrace(mask, exclude_self, raise_nmi);
Russell King96f0e002014-09-03 23:57:13 +0100813}