blob: b139fe2d21260491d6ba310ae07746f4c2cb1e4b [file] [log] [blame]
Catalin Marinas60ffc302012-03-05 11:49:27 +00001/*
2 * Based on arch/arm/kernel/traps.c
3 *
4 * Copyright (C) 1995-2009 Russell King
5 * Copyright (C) 2012 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19
Dave P Martin9fb74102015-07-24 16:37:48 +010020#include <linux/bug.h>
Catalin Marinas60ffc302012-03-05 11:49:27 +000021#include <linux/signal.h>
22#include <linux/personality.h>
23#include <linux/kallsyms.h>
24#include <linux/spinlock.h>
25#include <linux/uaccess.h>
26#include <linux/hardirq.h>
27#include <linux/kdebug.h>
28#include <linux/module.h>
29#include <linux/kexec.h>
30#include <linux/delay.h>
31#include <linux/init.h>
Ingo Molnar3f07c012017-02-08 18:51:30 +010032#include <linux/sched/signal.h>
Ingo Molnarb17b0152017-02-08 18:51:35 +010033#include <linux/sched/debug.h>
Ingo Molnar68db0cf2017-02-08 18:51:37 +010034#include <linux/sched/task_stack.h>
Mark Rutland872d8322017-07-14 20:30:35 +010035#include <linux/sizes.h>
Catalin Marinas60ffc302012-03-05 11:49:27 +000036#include <linux/syscalls.h>
Ingo Molnar589ee622017-02-04 00:16:44 +010037#include <linux/mm_types.h>
Catalin Marinas60ffc302012-03-05 11:49:27 +000038
39#include <asm/atomic.h>
Dave P Martin9fb74102015-07-24 16:37:48 +010040#include <asm/bug.h>
James Morse0fbeb312017-11-02 12:12:34 +000041#include <asm/daifflags.h>
Will Deacon1442b6e2013-03-16 08:48:13 +000042#include <asm/debug-monitors.h>
Mark Rutland60a1f022014-11-18 12:16:30 +000043#include <asm/esr.h>
Dave P Martin9fb74102015-07-24 16:37:48 +010044#include <asm/insn.h>
Catalin Marinas60ffc302012-03-05 11:49:27 +000045#include <asm/traps.h>
Mark Rutland872d8322017-07-14 20:30:35 +010046#include <asm/smp.h>
Mark Rutlanda9ea0012016-11-03 20:23:05 +000047#include <asm/stack_pointer.h>
Catalin Marinas60ffc302012-03-05 11:49:27 +000048#include <asm/stacktrace.h>
49#include <asm/exception.h>
50#include <asm/system_misc.h>
Andre Przywara7dd01ae2016-06-28 18:07:32 +010051#include <asm/sysreg.h>
Catalin Marinas60ffc302012-03-05 11:49:27 +000052
53static const char *handler[]= {
54 "Synchronous Abort",
55 "IRQ",
56 "FIQ",
57 "Error"
58};
59
Michael Weiser5ee39a72018-02-01 23:13:38 +010060int show_unhandled_signals = 0;
Catalin Marinas60ffc302012-03-05 11:49:27 +000061
Jungseok Lee9f93f3e2015-10-17 14:28:11 +000062static void dump_backtrace_entry(unsigned long where)
Catalin Marinas60ffc302012-03-05 11:49:27 +000063{
Will Deacona25ffd32017-10-19 13:19:20 +010064 printk(" %pS\n", (void *)where);
Catalin Marinas60ffc302012-03-05 11:49:27 +000065}
66
Mark Rutlandc5cea062016-06-13 11:15:14 +010067static void __dump_instr(const char *lvl, struct pt_regs *regs)
Catalin Marinas60ffc302012-03-05 11:49:27 +000068{
69 unsigned long addr = instruction_pointer(regs);
Catalin Marinas60ffc302012-03-05 11:49:27 +000070 char str[sizeof("00000000 ") * 5 + 2 + 1], *p = str;
71 int i;
72
Catalin Marinas60ffc302012-03-05 11:49:27 +000073 for (i = -4; i < 1; i++) {
74 unsigned int val, bad;
75
Mark Rutland7a7003b2017-11-02 16:12:03 +000076 bad = get_user(val, &((u32 *)addr)[i]);
Catalin Marinas60ffc302012-03-05 11:49:27 +000077
78 if (!bad)
79 p += sprintf(p, i == 0 ? "(%08x) " : "%08x ", val);
80 else {
81 p += sprintf(p, "bad PC value");
82 break;
83 }
84 }
85 printk("%sCode: %s\n", lvl, str);
Mark Rutlandc5cea062016-06-13 11:15:14 +010086}
Catalin Marinas60ffc302012-03-05 11:49:27 +000087
Mark Rutlandc5cea062016-06-13 11:15:14 +010088static void dump_instr(const char *lvl, struct pt_regs *regs)
89{
90 if (!user_mode(regs)) {
91 mm_segment_t fs = get_fs();
92 set_fs(KERNEL_DS);
93 __dump_instr(lvl, regs);
94 set_fs(fs);
95 } else {
96 __dump_instr(lvl, regs);
97 }
Catalin Marinas60ffc302012-03-05 11:49:27 +000098}
99
Kefeng Wang1149aad2017-05-09 09:53:37 +0800100void dump_backtrace(struct pt_regs *regs, struct task_struct *tsk)
Catalin Marinas60ffc302012-03-05 11:49:27 +0000101{
102 struct stackframe frame;
AKASHI Takahiro20380bb2015-12-15 17:33:41 +0900103 int skip;
Catalin Marinas60ffc302012-03-05 11:49:27 +0000104
Mark Rutlandb5e73072016-09-23 17:55:05 +0100105 pr_debug("%s(regs = %p tsk = %p)\n", __func__, regs, tsk);
106
107 if (!tsk)
108 tsk = current;
109
Mark Rutland9bbd4c52016-11-03 20:23:08 +0000110 if (!try_get_task_stack(tsk))
111 return;
112
AKASHI Takahiro20380bb2015-12-15 17:33:41 +0900113 if (tsk == current) {
Catalin Marinas60ffc302012-03-05 11:49:27 +0000114 frame.fp = (unsigned long)__builtin_frame_address(0);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000115 frame.pc = (unsigned long)dump_backtrace;
116 } else {
117 /*
118 * task blocked in __switch_to
119 */
120 frame.fp = thread_saved_fp(tsk);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000121 frame.pc = thread_saved_pc(tsk);
122 }
AKASHI Takahiro20380bb2015-12-15 17:33:41 +0900123#ifdef CONFIG_FUNCTION_GRAPH_TRACER
124 frame.graph = tsk->curr_ret_stack;
125#endif
Catalin Marinas60ffc302012-03-05 11:49:27 +0000126
AKASHI Takahiro20380bb2015-12-15 17:33:41 +0900127 skip = !!regs;
Will Deaconc9cd0ed2015-12-21 16:44:27 +0000128 printk("Call trace:\n");
Will Deacona25ffd32017-10-19 13:19:20 +0100129 do {
AKASHI Takahiro20380bb2015-12-15 17:33:41 +0900130 /* skip until specified stack frame */
131 if (!skip) {
Ard Biesheuvel73267492017-07-22 18:45:33 +0100132 dump_backtrace_entry(frame.pc);
AKASHI Takahiro20380bb2015-12-15 17:33:41 +0900133 } else if (frame.fp == regs->regs[29]) {
134 skip = 0;
135 /*
136 * Mostly, this is the case where this function is
137 * called in panic/abort. As exception handler's
138 * stack frame does not contain the corresponding pc
139 * at which an exception has taken place, use regs->pc
140 * instead.
141 */
142 dump_backtrace_entry(regs->pc);
143 }
Will Deacona25ffd32017-10-19 13:19:20 +0100144 } while (!unwind_frame(tsk, &frame));
Mark Rutland9bbd4c52016-11-03 20:23:08 +0000145
146 put_task_stack(tsk);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000147}
148
Catalin Marinas60ffc302012-03-05 11:49:27 +0000149void show_stack(struct task_struct *tsk, unsigned long *sp)
150{
151 dump_backtrace(NULL, tsk);
152 barrier();
153}
154
155#ifdef CONFIG_PREEMPT
156#define S_PREEMPT " PREEMPT"
157#else
158#define S_PREEMPT ""
159#endif
Catalin Marinas60ffc302012-03-05 11:49:27 +0000160#define S_SMP " SMP"
Catalin Marinas60ffc302012-03-05 11:49:27 +0000161
Mark Rutland876e7a32016-11-03 20:23:06 +0000162static int __die(const char *str, int err, struct pt_regs *regs)
Catalin Marinas60ffc302012-03-05 11:49:27 +0000163{
Mark Rutland876e7a32016-11-03 20:23:06 +0000164 struct task_struct *tsk = current;
Catalin Marinas60ffc302012-03-05 11:49:27 +0000165 static int die_counter;
166 int ret;
167
168 pr_emerg("Internal error: %s: %x [#%d]" S_PREEMPT S_SMP "\n",
169 str, err, ++die_counter);
170
171 /* trap and error numbers are mostly meaningless on ARM */
172 ret = notify_die(DIE_OOPS, str, regs, err, 0, SIGSEGV);
173 if (ret == NOTIFY_STOP)
174 return ret;
175
176 print_modules();
177 __show_regs(regs);
178 pr_emerg("Process %.*s (pid: %d, stack limit = 0x%p)\n",
Mark Rutland876e7a32016-11-03 20:23:06 +0000179 TASK_COMM_LEN, tsk->comm, task_pid_nr(tsk),
180 end_of_stack(tsk));
Catalin Marinas60ffc302012-03-05 11:49:27 +0000181
Mark Rutland7ceb3a12016-06-13 11:15:15 +0100182 if (!user_mode(regs)) {
Catalin Marinas60ffc302012-03-05 11:49:27 +0000183 dump_backtrace(regs, tsk);
184 dump_instr(KERN_EMERG, regs);
185 }
186
187 return ret;
188}
189
190static DEFINE_RAW_SPINLOCK(die_lock);
191
192/*
193 * This function is protected against re-entrancy.
194 */
195void die(const char *str, struct pt_regs *regs, int err)
196{
Catalin Marinas60ffc302012-03-05 11:49:27 +0000197 int ret;
Qiao Zhou6f44a0b2017-07-07 17:29:34 +0800198 unsigned long flags;
199
200 raw_spin_lock_irqsave(&die_lock, flags);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000201
202 oops_enter();
203
Catalin Marinas60ffc302012-03-05 11:49:27 +0000204 console_verbose();
205 bust_spinlocks(1);
Mark Rutland876e7a32016-11-03 20:23:06 +0000206 ret = __die(str, err, regs);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000207
Mark Rutland876e7a32016-11-03 20:23:06 +0000208 if (regs && kexec_should_crash(current))
Catalin Marinas60ffc302012-03-05 11:49:27 +0000209 crash_kexec(regs);
210
211 bust_spinlocks(0);
Rusty Russell373d4d02013-01-21 17:17:39 +1030212 add_taint(TAINT_DIE, LOCKDEP_NOW_UNRELIABLE);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000213 oops_exit();
214
215 if (in_interrupt())
216 panic("Fatal exception in interrupt");
217 if (panic_on_oops)
218 panic("Fatal exception");
Qiao Zhou6f44a0b2017-07-07 17:29:34 +0800219
220 raw_spin_unlock_irqrestore(&die_lock, flags);
221
Catalin Marinas60ffc302012-03-05 11:49:27 +0000222 if (ret != NOTIFY_STOP)
223 do_exit(SIGSEGV);
224}
225
Will Deacona26731d2018-02-20 15:08:51 +0000226static bool show_unhandled_signals_ratelimited(void)
227{
228 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
229 DEFAULT_RATELIMIT_BURST);
230 return show_unhandled_signals && __ratelimit(&rs);
231}
232
Will Deacona1ece822018-02-20 13:46:05 +0000233void arm64_force_sig_info(struct siginfo *info, const char *str,
234 struct task_struct *tsk)
235{
236 unsigned int esr = tsk->thread.fault_code;
237 struct pt_regs *regs = task_pt_regs(tsk);
238
239 if (!unhandled_signal(tsk, info->si_signo))
240 goto send_sig;
241
242 if (!show_unhandled_signals_ratelimited())
243 goto send_sig;
244
245 pr_info("%s[%d]: unhandled exception: ", tsk->comm, task_pid_nr(tsk));
246 if (esr)
247 pr_cont("%s, ESR 0x%08x, ", esr_get_class_string(esr), esr);
248
249 pr_cont("%s", str);
250 print_vma_addr(KERN_CONT " in ", regs->pc);
251 pr_cont("\n");
252 __show_regs(regs);
253
254send_sig:
255 force_sig_info(info->si_signo, info, tsk);
256}
257
Catalin Marinas60ffc302012-03-05 11:49:27 +0000258void arm64_notify_die(const char *str, struct pt_regs *regs,
259 struct siginfo *info, int err)
260{
Catalin Marinas91413002014-04-06 23:04:12 +0100261 if (user_mode(regs)) {
Will Deacona1ece822018-02-20 13:46:05 +0000262 WARN_ON(regs != current_pt_regs());
Catalin Marinas91413002014-04-06 23:04:12 +0100263 current->thread.fault_address = 0;
264 current->thread.fault_code = err;
Will Deacona1ece822018-02-20 13:46:05 +0000265 arm64_force_sig_info(info, str, current);
Catalin Marinas91413002014-04-06 23:04:12 +0100266 } else {
Catalin Marinas60ffc302012-03-05 11:49:27 +0000267 die(str, regs, err);
Catalin Marinas91413002014-04-06 23:04:12 +0100268 }
Catalin Marinas60ffc302012-03-05 11:49:27 +0000269}
270
Julien Thierry6436bee2017-10-25 10:04:33 +0100271void arm64_skip_faulting_instruction(struct pt_regs *regs, unsigned long size)
272{
273 regs->pc += size;
274
275 /*
276 * If we were single stepping, we want to get the step exception after
277 * we return from the trap.
278 */
279 user_fastforward_single_step(current);
280}
281
Punit Agrawal9b79f522014-11-18 11:41:22 +0000282static LIST_HEAD(undef_hook);
283static DEFINE_RAW_SPINLOCK(undef_lock);
284
285void register_undef_hook(struct undef_hook *hook)
286{
287 unsigned long flags;
288
289 raw_spin_lock_irqsave(&undef_lock, flags);
290 list_add(&hook->node, &undef_hook);
291 raw_spin_unlock_irqrestore(&undef_lock, flags);
292}
293
294void unregister_undef_hook(struct undef_hook *hook)
295{
296 unsigned long flags;
297
298 raw_spin_lock_irqsave(&undef_lock, flags);
299 list_del(&hook->node);
300 raw_spin_unlock_irqrestore(&undef_lock, flags);
301}
302
303static int call_undef_hook(struct pt_regs *regs)
304{
305 struct undef_hook *hook;
306 unsigned long flags;
307 u32 instr;
308 int (*fn)(struct pt_regs *regs, u32 instr) = NULL;
309 void __user *pc = (void __user *)instruction_pointer(regs);
310
311 if (!user_mode(regs))
312 return 1;
313
314 if (compat_thumb_mode(regs)) {
315 /* 16-bit Thumb instruction */
Luc Van Oostenryck6cf5d4a2017-06-28 16:55:55 +0200316 __le16 instr_le;
317 if (get_user(instr_le, (__le16 __user *)pc))
Punit Agrawal9b79f522014-11-18 11:41:22 +0000318 goto exit;
Luc Van Oostenryck6cf5d4a2017-06-28 16:55:55 +0200319 instr = le16_to_cpu(instr_le);
Punit Agrawal9b79f522014-11-18 11:41:22 +0000320 if (aarch32_insn_is_wide(instr)) {
321 u32 instr2;
322
Luc Van Oostenryck6cf5d4a2017-06-28 16:55:55 +0200323 if (get_user(instr_le, (__le16 __user *)(pc + 2)))
Punit Agrawal9b79f522014-11-18 11:41:22 +0000324 goto exit;
Luc Van Oostenryck6cf5d4a2017-06-28 16:55:55 +0200325 instr2 = le16_to_cpu(instr_le);
Punit Agrawal9b79f522014-11-18 11:41:22 +0000326 instr = (instr << 16) | instr2;
327 }
328 } else {
329 /* 32-bit ARM instruction */
Luc Van Oostenryck6cf5d4a2017-06-28 16:55:55 +0200330 __le32 instr_le;
331 if (get_user(instr_le, (__le32 __user *)pc))
Punit Agrawal9b79f522014-11-18 11:41:22 +0000332 goto exit;
Luc Van Oostenryck6cf5d4a2017-06-28 16:55:55 +0200333 instr = le32_to_cpu(instr_le);
Punit Agrawal9b79f522014-11-18 11:41:22 +0000334 }
335
336 raw_spin_lock_irqsave(&undef_lock, flags);
337 list_for_each_entry(hook, &undef_hook, node)
338 if ((instr & hook->instr_mask) == hook->instr_val &&
339 (regs->pstate & hook->pstate_mask) == hook->pstate_val)
340 fn = hook->fn;
341
342 raw_spin_unlock_irqrestore(&undef_lock, flags);
343exit:
344 return fn ? fn(regs, instr) : 1;
345}
346
Will Deacon2c9120f32018-02-20 14:16:29 +0000347void force_signal_inject(int signal, int code, unsigned long address)
Catalin Marinas60ffc302012-03-05 11:49:27 +0000348{
349 siginfo_t info;
Andre Przywara390bf172016-06-28 18:07:31 +0100350 const char *desc;
Will Deacon2c9120f32018-02-20 14:16:29 +0000351 struct pt_regs *regs = current_pt_regs();
352
353 clear_siginfo(&info);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000354
Andre Przywara390bf172016-06-28 18:07:31 +0100355 switch (signal) {
356 case SIGILL:
357 desc = "undefined instruction";
358 break;
359 case SIGSEGV:
360 desc = "illegal memory access";
361 break;
362 default:
Dave Martinbc0ee472017-10-31 15:51:05 +0000363 desc = "unknown or unrecoverable error";
Andre Przywara390bf172016-06-28 18:07:31 +0100364 break;
365 }
366
Will Deacona7e6f1c2018-02-20 18:08:40 +0000367 /* Force signals we don't understand to SIGKILL */
368 if (WARN_ON(signal != SIGKILL ||
369 siginfo_layout(signal, code) != SIL_FAULT)) {
370 signal = SIGKILL;
371 }
372
Andre Przywara390bf172016-06-28 18:07:31 +0100373 info.si_signo = signal;
374 info.si_errno = 0;
375 info.si_code = code;
Will Deacon2c9120f32018-02-20 14:16:29 +0000376 info.si_addr = (void __user *)address;
Andre Przywara390bf172016-06-28 18:07:31 +0100377
378 arm64_notify_die(desc, regs, &info, 0);
379}
380
381/*
382 * Set up process info to signal segmentation fault - called on access error.
383 */
Will Deacon2c9120f32018-02-20 14:16:29 +0000384void arm64_notify_segfault(unsigned long addr)
Andre Przywara390bf172016-06-28 18:07:31 +0100385{
386 int code;
387
388 down_read(&current->mm->mmap_sem);
389 if (find_vma(current->mm, addr) == NULL)
390 code = SEGV_MAPERR;
391 else
392 code = SEGV_ACCERR;
393 up_read(&current->mm->mmap_sem);
394
Will Deacon2c9120f32018-02-20 14:16:29 +0000395 force_signal_inject(SIGSEGV, code, addr);
Andre Przywara390bf172016-06-28 18:07:31 +0100396}
397
398asmlinkage void __exception do_undefinstr(struct pt_regs *regs)
399{
Catalin Marinas60ffc302012-03-05 11:49:27 +0000400 /* check for AArch32 breakpoint instructions */
Will Deacon1442b6e2013-03-16 08:48:13 +0000401 if (!aarch32_break_handler(regs))
Catalin Marinas60ffc302012-03-05 11:49:27 +0000402 return;
Catalin Marinas60ffc302012-03-05 11:49:27 +0000403
Punit Agrawal9b79f522014-11-18 11:41:22 +0000404 if (call_undef_hook(regs) == 0)
405 return;
406
Will Deacon2c9120f32018-02-20 14:16:29 +0000407 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000408}
409
James Morse2a6dcb22016-10-18 11:27:46 +0100410int cpu_enable_cache_maint_trap(void *__unused)
Andre Przywara7dd01ae2016-06-28 18:07:32 +0100411{
412 config_sctlr_el1(SCTLR_EL1_UCI, 0);
James Morse2a6dcb22016-10-18 11:27:46 +0100413 return 0;
Andre Przywara7dd01ae2016-06-28 18:07:32 +0100414}
415
416#define __user_cache_maint(insn, address, res) \
Kristina Martsenko81cddd62017-05-03 16:37:45 +0100417 if (address >= user_addr_max()) { \
Andre Przywara87261d12016-10-19 14:40:54 +0100418 res = -EFAULT; \
Catalin Marinas39bc88e2016-09-02 14:54:03 +0100419 } else { \
420 uaccess_ttbr0_enable(); \
Andre Przywara87261d12016-10-19 14:40:54 +0100421 asm volatile ( \
422 "1: " insn ", %1\n" \
423 " mov %w0, #0\n" \
424 "2:\n" \
425 " .pushsection .fixup,\"ax\"\n" \
426 " .align 2\n" \
427 "3: mov %w0, %w2\n" \
428 " b 2b\n" \
429 " .popsection\n" \
430 _ASM_EXTABLE(1b, 3b) \
431 : "=r" (res) \
Catalin Marinas39bc88e2016-09-02 14:54:03 +0100432 : "r" (address), "i" (-EFAULT)); \
433 uaccess_ttbr0_disable(); \
434 }
Andre Przywara7dd01ae2016-06-28 18:07:32 +0100435
Suzuki K Poulose9dbd5bb2016-09-09 14:07:15 +0100436static void user_cache_maint_handler(unsigned int esr, struct pt_regs *regs)
Andre Przywara7dd01ae2016-06-28 18:07:32 +0100437{
438 unsigned long address;
Suzuki K Poulose9dbd5bb2016-09-09 14:07:15 +0100439 int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;
440 int crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
441 int ret = 0;
Andre Przywara7dd01ae2016-06-28 18:07:32 +0100442
Kristina Martsenko81cddd62017-05-03 16:37:45 +0100443 address = untagged_addr(pt_regs_read_reg(regs, rt));
Andre Przywara7dd01ae2016-06-28 18:07:32 +0100444
Suzuki K Poulose9dbd5bb2016-09-09 14:07:15 +0100445 switch (crm) {
446 case ESR_ELx_SYS64_ISS_CRM_DC_CVAU: /* DC CVAU, gets promoted */
447 __user_cache_maint("dc civac", address, ret);
448 break;
449 case ESR_ELx_SYS64_ISS_CRM_DC_CVAC: /* DC CVAC, gets promoted */
450 __user_cache_maint("dc civac", address, ret);
451 break;
Robin Murphye1bc5d12017-07-25 11:55:41 +0100452 case ESR_ELx_SYS64_ISS_CRM_DC_CVAP: /* DC CVAP */
453 __user_cache_maint("sys 3, c7, c12, 1", address, ret);
454 break;
Suzuki K Poulose9dbd5bb2016-09-09 14:07:15 +0100455 case ESR_ELx_SYS64_ISS_CRM_DC_CIVAC: /* DC CIVAC */
456 __user_cache_maint("dc civac", address, ret);
457 break;
458 case ESR_ELx_SYS64_ISS_CRM_IC_IVAU: /* IC IVAU */
459 __user_cache_maint("ic ivau", address, ret);
460 break;
461 default:
Will Deacon2c9120f32018-02-20 14:16:29 +0000462 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc);
Andre Przywara7dd01ae2016-06-28 18:07:32 +0100463 return;
464 }
465
466 if (ret)
Will Deacon2c9120f32018-02-20 14:16:29 +0000467 arm64_notify_segfault(address);
Andre Przywara7dd01ae2016-06-28 18:07:32 +0100468 else
Julien Thierry6436bee2017-10-25 10:04:33 +0100469 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
Andre Przywara7dd01ae2016-06-28 18:07:32 +0100470}
471
Suzuki K Poulose116c81f2016-09-09 14:07:16 +0100472static void ctr_read_handler(unsigned int esr, struct pt_regs *regs)
473{
474 int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;
Mark Rutland8b6e70f2017-02-09 15:19:19 +0000475 unsigned long val = arm64_ftr_reg_user_value(&arm64_ftr_reg_ctrel0);
Suzuki K Poulose116c81f2016-09-09 14:07:16 +0100476
Mark Rutland8b6e70f2017-02-09 15:19:19 +0000477 pt_regs_write_reg(regs, rt, val);
478
Julien Thierry6436bee2017-10-25 10:04:33 +0100479 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
Suzuki K Poulose116c81f2016-09-09 14:07:16 +0100480}
481
Marc Zyngier6126ce02017-02-01 11:48:58 +0000482static void cntvct_read_handler(unsigned int esr, struct pt_regs *regs)
483{
484 int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;
485
486 pt_regs_write_reg(regs, rt, arch_counter_get_cntvct());
Julien Thierry6436bee2017-10-25 10:04:33 +0100487 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
Marc Zyngier6126ce02017-02-01 11:48:58 +0000488}
489
Marc Zyngier98421192017-04-24 09:04:03 +0100490static void cntfrq_read_handler(unsigned int esr, struct pt_regs *regs)
491{
492 int rt = (esr & ESR_ELx_SYS64_ISS_RT_MASK) >> ESR_ELx_SYS64_ISS_RT_SHIFT;
493
Marc Zyngierc6f97ad2017-07-21 18:15:27 +0100494 pt_regs_write_reg(regs, rt, arch_timer_get_rate());
Julien Thierry6436bee2017-10-25 10:04:33 +0100495 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
Marc Zyngier98421192017-04-24 09:04:03 +0100496}
497
Suzuki K Poulose9dbd5bb2016-09-09 14:07:15 +0100498struct sys64_hook {
499 unsigned int esr_mask;
500 unsigned int esr_val;
501 void (*handler)(unsigned int esr, struct pt_regs *regs);
502};
503
504static struct sys64_hook sys64_hooks[] = {
505 {
506 .esr_mask = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_MASK,
507 .esr_val = ESR_ELx_SYS64_ISS_EL0_CACHE_OP_VAL,
508 .handler = user_cache_maint_handler,
509 },
Suzuki K Poulose116c81f2016-09-09 14:07:16 +0100510 {
511 /* Trap read access to CTR_EL0 */
512 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
513 .esr_val = ESR_ELx_SYS64_ISS_SYS_CTR_READ,
514 .handler = ctr_read_handler,
515 },
Marc Zyngier6126ce02017-02-01 11:48:58 +0000516 {
517 /* Trap read access to CNTVCT_EL0 */
518 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
519 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTVCT,
520 .handler = cntvct_read_handler,
521 },
Marc Zyngier98421192017-04-24 09:04:03 +0100522 {
523 /* Trap read access to CNTFRQ_EL0 */
524 .esr_mask = ESR_ELx_SYS64_ISS_SYS_OP_MASK,
525 .esr_val = ESR_ELx_SYS64_ISS_SYS_CNTFRQ,
526 .handler = cntfrq_read_handler,
527 },
Suzuki K Poulose9dbd5bb2016-09-09 14:07:15 +0100528 {},
529};
530
531asmlinkage void __exception do_sysinstr(unsigned int esr, struct pt_regs *regs)
532{
533 struct sys64_hook *hook;
534
535 for (hook = sys64_hooks; hook->handler; hook++)
536 if ((hook->esr_mask & esr) == hook->esr_val) {
537 hook->handler(esr, regs);
538 return;
539 }
540
Mark Rutland49f6cba2017-01-27 16:15:38 +0000541 /*
542 * New SYS instructions may previously have been undefined at EL0. Fall
543 * back to our usual undefined instruction handler so that we handle
544 * these consistently.
545 */
546 do_undefinstr(regs);
Suzuki K Poulose9dbd5bb2016-09-09 14:07:15 +0100547}
548
Catalin Marinas60ffc302012-03-05 11:49:27 +0000549long compat_arm_syscall(struct pt_regs *regs);
550
551asmlinkage long do_ni_syscall(struct pt_regs *regs)
552{
553#ifdef CONFIG_COMPAT
554 long ret;
555 if (is_compat_task()) {
556 ret = compat_arm_syscall(regs);
557 if (ret != -ENOSYS)
558 return ret;
559 }
560#endif
561
Catalin Marinas60ffc302012-03-05 11:49:27 +0000562 return sys_ni_syscall();
563}
564
Mark Rutland60a1f022014-11-18 12:16:30 +0000565static const char *esr_class_str[] = {
566 [0 ... ESR_ELx_EC_MAX] = "UNRECOGNIZED EC",
567 [ESR_ELx_EC_UNKNOWN] = "Unknown/Uncategorized",
568 [ESR_ELx_EC_WFx] = "WFI/WFE",
569 [ESR_ELx_EC_CP15_32] = "CP15 MCR/MRC",
570 [ESR_ELx_EC_CP15_64] = "CP15 MCRR/MRRC",
571 [ESR_ELx_EC_CP14_MR] = "CP14 MCR/MRC",
572 [ESR_ELx_EC_CP14_LS] = "CP14 LDC/STC",
573 [ESR_ELx_EC_FP_ASIMD] = "ASIMD",
574 [ESR_ELx_EC_CP10_ID] = "CP10 MRC/VMRS",
575 [ESR_ELx_EC_CP14_64] = "CP14 MCRR/MRRC",
576 [ESR_ELx_EC_ILL] = "PSTATE.IL",
577 [ESR_ELx_EC_SVC32] = "SVC (AArch32)",
578 [ESR_ELx_EC_HVC32] = "HVC (AArch32)",
579 [ESR_ELx_EC_SMC32] = "SMC (AArch32)",
580 [ESR_ELx_EC_SVC64] = "SVC (AArch64)",
581 [ESR_ELx_EC_HVC64] = "HVC (AArch64)",
582 [ESR_ELx_EC_SMC64] = "SMC (AArch64)",
583 [ESR_ELx_EC_SYS64] = "MSR/MRS (AArch64)",
Dave Martin67236562017-10-31 15:51:00 +0000584 [ESR_ELx_EC_SVE] = "SVE",
Mark Rutland60a1f022014-11-18 12:16:30 +0000585 [ESR_ELx_EC_IMP_DEF] = "EL3 IMP DEF",
586 [ESR_ELx_EC_IABT_LOW] = "IABT (lower EL)",
587 [ESR_ELx_EC_IABT_CUR] = "IABT (current EL)",
588 [ESR_ELx_EC_PC_ALIGN] = "PC Alignment",
589 [ESR_ELx_EC_DABT_LOW] = "DABT (lower EL)",
590 [ESR_ELx_EC_DABT_CUR] = "DABT (current EL)",
591 [ESR_ELx_EC_SP_ALIGN] = "SP Alignment",
592 [ESR_ELx_EC_FP_EXC32] = "FP (AArch32)",
593 [ESR_ELx_EC_FP_EXC64] = "FP (AArch64)",
594 [ESR_ELx_EC_SERROR] = "SError",
595 [ESR_ELx_EC_BREAKPT_LOW] = "Breakpoint (lower EL)",
596 [ESR_ELx_EC_BREAKPT_CUR] = "Breakpoint (current EL)",
597 [ESR_ELx_EC_SOFTSTP_LOW] = "Software Step (lower EL)",
598 [ESR_ELx_EC_SOFTSTP_CUR] = "Software Step (current EL)",
599 [ESR_ELx_EC_WATCHPT_LOW] = "Watchpoint (lower EL)",
600 [ESR_ELx_EC_WATCHPT_CUR] = "Watchpoint (current EL)",
601 [ESR_ELx_EC_BKPT32] = "BKPT (AArch32)",
602 [ESR_ELx_EC_VECTOR32] = "Vector catch (AArch32)",
603 [ESR_ELx_EC_BRK64] = "BRK (AArch64)",
604};
605
606const char *esr_get_class_string(u32 esr)
607{
Mark Rutland275f3442016-05-31 12:33:01 +0100608 return esr_class_str[ESR_ELx_EC(esr)];
Mark Rutland60a1f022014-11-18 12:16:30 +0000609}
610
Catalin Marinas60ffc302012-03-05 11:49:27 +0000611/*
Mark Rutland7d9e8f72017-01-18 17:23:41 +0000612 * bad_mode handles the impossible case in the exception vector. This is always
613 * fatal.
Catalin Marinas60ffc302012-03-05 11:49:27 +0000614 */
615asmlinkage void bad_mode(struct pt_regs *regs, int reason, unsigned int esr)
616{
617 console_verbose();
618
Mark Rutland8051f4d2016-05-31 12:07:47 +0100619 pr_crit("Bad mode in %s handler detected on CPU%d, code 0x%08x -- %s\n",
620 handler[reason], smp_processor_id(), esr,
621 esr_get_class_string(esr));
Mark Rutland7d9e8f72017-01-18 17:23:41 +0000622
623 die("Oops - bad mode", regs, 0);
James Morse0fbeb312017-11-02 12:12:34 +0000624 local_daif_mask();
Mark Rutland7d9e8f72017-01-18 17:23:41 +0000625 panic("bad mode");
626}
627
628/*
629 * bad_el0_sync handles unexpected, but potentially recoverable synchronous
630 * exceptions taken from EL0. Unlike bad_mode, this returns.
631 */
632asmlinkage void bad_el0_sync(struct pt_regs *regs, int reason, unsigned int esr)
633{
634 siginfo_t info;
635 void __user *pc = (void __user *)instruction_pointer(regs);
636 console_verbose();
637
638 pr_crit("Bad EL0 synchronous exception detected on CPU%d, code 0x%08x -- %s\n",
639 smp_processor_id(), esr, esr_get_class_string(esr));
Mark Rutland9955ac42013-05-28 15:54:15 +0100640 __show_regs(regs);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000641
Mark Rutland9955ac42013-05-28 15:54:15 +0100642 info.si_signo = SIGILL;
643 info.si_errno = 0;
644 info.si_code = ILL_ILLOPC;
645 info.si_addr = pc;
646
Mark Rutland7d9e8f72017-01-18 17:23:41 +0000647 current->thread.fault_address = 0;
648 current->thread.fault_code = 0;
649
650 force_sig_info(info.si_signo, &info, current);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000651}
652
Mark Rutland872d8322017-07-14 20:30:35 +0100653#ifdef CONFIG_VMAP_STACK
654
655DEFINE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack)
656 __aligned(16);
657
658asmlinkage void handle_bad_stack(struct pt_regs *regs)
659{
660 unsigned long tsk_stk = (unsigned long)current->stack;
661 unsigned long irq_stk = (unsigned long)this_cpu_read(irq_stack_ptr);
662 unsigned long ovf_stk = (unsigned long)this_cpu_ptr(overflow_stack);
663 unsigned int esr = read_sysreg(esr_el1);
664 unsigned long far = read_sysreg(far_el1);
665
666 console_verbose();
667 pr_emerg("Insufficient stack space to handle exception!");
668
669 pr_emerg("ESR: 0x%08x -- %s\n", esr, esr_get_class_string(esr));
670 pr_emerg("FAR: 0x%016lx\n", far);
671
672 pr_emerg("Task stack: [0x%016lx..0x%016lx]\n",
673 tsk_stk, tsk_stk + THREAD_SIZE);
674 pr_emerg("IRQ stack: [0x%016lx..0x%016lx]\n",
675 irq_stk, irq_stk + THREAD_SIZE);
676 pr_emerg("Overflow stack: [0x%016lx..0x%016lx]\n",
677 ovf_stk, ovf_stk + OVERFLOW_STACK_SIZE);
678
679 __show_regs(regs);
680
681 /*
682 * We use nmi_panic to limit the potential for recusive overflows, and
683 * to get a better stack trace.
684 */
685 nmi_panic(NULL, "kernel stack overflow");
686 cpu_park_loop();
687}
688#endif
689
James Morse6bf0dcf2018-01-15 19:38:57 +0000690void __noreturn arm64_serror_panic(struct pt_regs *regs, u32 esr)
Xie XiuQia92d4d12017-11-02 12:12:42 +0000691{
Xie XiuQia92d4d12017-11-02 12:12:42 +0000692 console_verbose();
693
694 pr_crit("SError Interrupt on CPU%d, code 0x%08x -- %s\n",
695 smp_processor_id(), esr, esr_get_class_string(esr));
James Morse6bf0dcf2018-01-15 19:38:57 +0000696 if (regs)
697 __show_regs(regs);
Xie XiuQia92d4d12017-11-02 12:12:42 +0000698
James Morse6bf0dcf2018-01-15 19:38:57 +0000699 nmi_panic(regs, "Asynchronous SError Interrupt");
700
701 cpu_park_loop();
702 unreachable();
703}
704
705bool arm64_is_fatal_ras_serror(struct pt_regs *regs, unsigned int esr)
706{
707 u32 aet = arm64_ras_serror_get_severity(esr);
708
709 switch (aet) {
710 case ESR_ELx_AET_CE: /* corrected error */
711 case ESR_ELx_AET_UEO: /* restartable, not yet consumed */
712 /*
713 * The CPU can make progress. We may take UEO again as
714 * a more severe error.
715 */
716 return false;
717
718 case ESR_ELx_AET_UEU: /* Uncorrected Unrecoverable */
719 case ESR_ELx_AET_UER: /* Uncorrected Recoverable */
720 /*
721 * The CPU can't make progress. The exception may have
722 * been imprecise.
723 */
724 return true;
725
726 case ESR_ELx_AET_UC: /* Uncontainable or Uncategorized error */
727 default:
728 /* Error has been silently propagated */
729 arm64_serror_panic(regs, esr);
730 }
731}
732
733asmlinkage void do_serror(struct pt_regs *regs, unsigned int esr)
734{
735 nmi_enter();
736
737 /* non-RAS errors are not containable */
738 if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(regs, esr))
739 arm64_serror_panic(regs, esr);
740
741 nmi_exit();
Xie XiuQia92d4d12017-11-02 12:12:42 +0000742}
743
Catalin Marinas60ffc302012-03-05 11:49:27 +0000744void __pte_error(const char *file, int line, unsigned long val)
745{
Will Deaconc9cd0ed2015-12-21 16:44:27 +0000746 pr_err("%s:%d: bad pte %016lx.\n", file, line, val);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000747}
748
749void __pmd_error(const char *file, int line, unsigned long val)
750{
Will Deaconc9cd0ed2015-12-21 16:44:27 +0000751 pr_err("%s:%d: bad pmd %016lx.\n", file, line, val);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000752}
753
Jungseok Leec79b954b2014-05-12 18:40:51 +0900754void __pud_error(const char *file, int line, unsigned long val)
755{
Will Deaconc9cd0ed2015-12-21 16:44:27 +0000756 pr_err("%s:%d: bad pud %016lx.\n", file, line, val);
Jungseok Leec79b954b2014-05-12 18:40:51 +0900757}
758
Catalin Marinas60ffc302012-03-05 11:49:27 +0000759void __pgd_error(const char *file, int line, unsigned long val)
760{
Will Deaconc9cd0ed2015-12-21 16:44:27 +0000761 pr_err("%s:%d: bad pgd %016lx.\n", file, line, val);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000762}
763
Dave P Martin9fb74102015-07-24 16:37:48 +0100764/* GENERIC_BUG traps */
765
766int is_valid_bugaddr(unsigned long addr)
767{
768 /*
769 * bug_handler() only called for BRK #BUG_BRK_IMM.
770 * So the answer is trivial -- any spurious instances with no
771 * bug table entry will be rejected by report_bug() and passed
772 * back to the debug-monitors code and handled as a fatal
773 * unexpected debug exception.
774 */
775 return 1;
776}
777
778static int bug_handler(struct pt_regs *regs, unsigned int esr)
779{
780 if (user_mode(regs))
781 return DBG_HOOK_ERROR;
782
783 switch (report_bug(regs->pc, regs)) {
784 case BUG_TRAP_TYPE_BUG:
785 die("Oops - BUG", regs, 0);
786 break;
787
788 case BUG_TRAP_TYPE_WARN:
789 break;
790
791 default:
792 /* unknown/unrecognised bug trap type */
793 return DBG_HOOK_ERROR;
794 }
795
796 /* If thread survives, skip over the BUG instruction and continue: */
Julien Thierry6436bee2017-10-25 10:04:33 +0100797 arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
Dave P Martin9fb74102015-07-24 16:37:48 +0100798 return DBG_HOOK_HANDLED;
799}
800
801static struct break_hook bug_break_hook = {
802 .esr_val = 0xf2000000 | BUG_BRK_IMM,
803 .esr_mask = 0xffffffff,
804 .fn = bug_handler,
805};
806
807/*
808 * Initial handler for AArch64 BRK exceptions
809 * This handler only used until debug_traps_init().
810 */
811int __init early_brk64(unsigned long addr, unsigned int esr,
812 struct pt_regs *regs)
813{
814 return bug_handler(regs, esr) != DBG_HOOK_HANDLED;
815}
816
817/* This registration must happen early, before debug_traps_init(). */
Catalin Marinas60ffc302012-03-05 11:49:27 +0000818void __init trap_init(void)
819{
Dave P Martin9fb74102015-07-24 16:37:48 +0100820 register_break_hook(&bug_break_hook);
Catalin Marinas60ffc302012-03-05 11:49:27 +0000821}