Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/cmarinas...
[linux-2.6.git] / arch / mips / kernel / ftrace.c
1 /*
2  * Code for replacing ftrace calls with jumps.
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2009, 2010 DSLab, Lanzhou University, China
6  * Author: Wu Zhangjin <wuzhangjin@gmail.com>
7  *
8  * Thanks goes to Steven Rostedt for writing the original x86 version.
9  */
10
11 #include <linux/uaccess.h>
12 #include <linux/init.h>
13 #include <linux/ftrace.h>
14
15 #include <asm/asm.h>
16 #include <asm/asm-offsets.h>
17 #include <asm/cacheflush.h>
18 #include <asm/uasm.h>
19
20 #include <asm-generic/sections.h>
21
22 #ifdef CONFIG_DYNAMIC_FTRACE
23
24 #define JAL 0x0c000000          /* jump & link: ip --> ra, jump to target */
25 #define ADDR_MASK 0x03ffffff    /*  op_code|addr : 31...26|25 ....0 */
26 #define JUMP_RANGE_MASK ((1UL << 28) - 1)
27
28 #define INSN_NOP 0x00000000     /* nop */
29 #define INSN_JAL(addr)  \
30         ((unsigned int)(JAL | (((addr) >> 2) & ADDR_MASK)))
31
32 static unsigned int insn_jal_ftrace_caller __read_mostly;
33 static unsigned int insn_lui_v1_hi16_mcount __read_mostly;
34 static unsigned int insn_j_ftrace_graph_caller __maybe_unused __read_mostly;
35
36 static inline void ftrace_dyn_arch_init_insns(void)
37 {
38         u32 *buf;
39         unsigned int v1;
40
41         /* lui v1, hi16_mcount */
42         v1 = 3;
43         buf = (u32 *)&insn_lui_v1_hi16_mcount;
44         UASM_i_LA_mostly(&buf, v1, MCOUNT_ADDR);
45
46         /* jal (ftrace_caller + 8), jump over the first two instruction */
47         buf = (u32 *)&insn_jal_ftrace_caller;
48         uasm_i_jal(&buf, (FTRACE_ADDR + 8) & JUMP_RANGE_MASK);
49
50 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
51         /* j ftrace_graph_caller */
52         buf = (u32 *)&insn_j_ftrace_graph_caller;
53         uasm_i_j(&buf, (unsigned long)ftrace_graph_caller & JUMP_RANGE_MASK);
54 #endif
55 }
56
57 /*
58  * Check if the address is in kernel space
59  *
60  * Clone core_kernel_text() from kernel/extable.c, but doesn't call
61  * init_kernel_text() for Ftrace doesn't trace functions in init sections.
62  */
63 static inline int in_kernel_space(unsigned long ip)
64 {
65         if (ip >= (unsigned long)_stext &&
66             ip <= (unsigned long)_etext)
67                 return 1;
68         return 0;
69 }
70
71 static int ftrace_modify_code(unsigned long ip, unsigned int new_code)
72 {
73         int faulted;
74
75         /* *(unsigned int *)ip = new_code; */
76         safe_store_code(new_code, ip, faulted);
77
78         if (unlikely(faulted))
79                 return -EFAULT;
80
81         flush_icache_range(ip, ip + 8);
82
83         return 0;
84 }
85
86 /*
87  * The details about the calling site of mcount on MIPS
88  *
89  * 1. For kernel:
90  *
91  * move at, ra
92  * jal _mcount          --> nop
93  *
94  * 2. For modules:
95  *
96  * 2.1 For KBUILD_MCOUNT_RA_ADDRESS and CONFIG_32BIT
97  *
98  * lui v1, hi_16bit_of_mcount        --> b 1f (0x10000005)
99  * addiu v1, v1, low_16bit_of_mcount
100  * move at, ra
101  * move $12, ra_address
102  * jalr v1
103  *  sub sp, sp, 8
104  *                                  1: offset = 5 instructions
105  * 2.2 For the Other situations
106  *
107  * lui v1, hi_16bit_of_mcount        --> b 1f (0x10000004)
108  * addiu v1, v1, low_16bit_of_mcount
109  * move at, ra
110  * jalr v1
111  *  nop | move $12, ra_address | sub sp, sp, 8
112  *                                  1: offset = 4 instructions
113  */
114
115 #if defined(KBUILD_MCOUNT_RA_ADDRESS) && defined(CONFIG_32BIT)
116 #define MCOUNT_OFFSET_INSNS 5
117 #else
118 #define MCOUNT_OFFSET_INSNS 4
119 #endif
120 #define INSN_B_1F (0x10000000 | MCOUNT_OFFSET_INSNS)
121
122 int ftrace_make_nop(struct module *mod,
123                     struct dyn_ftrace *rec, unsigned long addr)
124 {
125         unsigned int new;
126         unsigned long ip = rec->ip;
127
128         /*
129          * If ip is in kernel space, no long call, otherwise, long call is
130          * needed.
131          */
132         new = in_kernel_space(ip) ? INSN_NOP : INSN_B_1F;
133
134         return ftrace_modify_code(ip, new);
135 }
136
137 int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
138 {
139         unsigned int new;
140         unsigned long ip = rec->ip;
141
142         new = in_kernel_space(ip) ? insn_jal_ftrace_caller :
143                 insn_lui_v1_hi16_mcount;
144
145         return ftrace_modify_code(ip, new);
146 }
147
148 #define FTRACE_CALL_IP ((unsigned long)(&ftrace_call))
149
150 int ftrace_update_ftrace_func(ftrace_func_t func)
151 {
152         unsigned int new;
153
154         new = INSN_JAL((unsigned long)func);
155
156         return ftrace_modify_code(FTRACE_CALL_IP, new);
157 }
158
159 int __init ftrace_dyn_arch_init(void *data)
160 {
161         /* Encode the instructions when booting */
162         ftrace_dyn_arch_init_insns();
163
164         /* Remove "b ftrace_stub" to ensure ftrace_caller() is executed */
165         ftrace_modify_code(MCOUNT_ADDR, INSN_NOP);
166
167         /* The return code is retured via data */
168         *(unsigned long *)data = 0;
169
170         return 0;
171 }
172 #endif  /* CONFIG_DYNAMIC_FTRACE */
173
174 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
175
176 #ifdef CONFIG_DYNAMIC_FTRACE
177
178 extern void ftrace_graph_call(void);
179 #define FTRACE_GRAPH_CALL_IP    ((unsigned long)(&ftrace_graph_call))
180
181 int ftrace_enable_ftrace_graph_caller(void)
182 {
183         return ftrace_modify_code(FTRACE_GRAPH_CALL_IP,
184                         insn_j_ftrace_graph_caller);
185 }
186
187 int ftrace_disable_ftrace_graph_caller(void)
188 {
189         return ftrace_modify_code(FTRACE_GRAPH_CALL_IP, INSN_NOP);
190 }
191
192 #endif  /* CONFIG_DYNAMIC_FTRACE */
193
194 #ifndef KBUILD_MCOUNT_RA_ADDRESS
195
196 #define S_RA_SP (0xafbf << 16)  /* s{d,w} ra, offset(sp) */
197 #define S_R_SP  (0xafb0 << 16)  /* s{d,w} R, offset(sp) */
198 #define OFFSET_MASK     0xffff  /* stack offset range: 0 ~ PT_SIZE */
199
200 unsigned long ftrace_get_parent_ra_addr(unsigned long self_ra, unsigned long
201                 old_parent_ra, unsigned long parent_ra_addr, unsigned long fp)
202 {
203         unsigned long sp, ip, tmp;
204         unsigned int code;
205         int faulted;
206
207         /*
208          * For module, move the ip from the return address after the
209          * instruction "lui v1, hi_16bit_of_mcount"(offset is 24), but for
210          * kernel, move after the instruction "move ra, at"(offset is 16)
211          */
212         ip = self_ra - (in_kernel_space(self_ra) ? 16 : 24);
213
214         /*
215          * search the text until finding the non-store instruction or "s{d,w}
216          * ra, offset(sp)" instruction
217          */
218         do {
219                 /* get the code at "ip": code = *(unsigned int *)ip; */
220                 safe_load_code(code, ip, faulted);
221
222                 if (unlikely(faulted))
223                         return 0;
224                 /*
225                  * If we hit the non-store instruction before finding where the
226                  * ra is stored, then this is a leaf function and it does not
227                  * store the ra on the stack
228                  */
229                 if ((code & S_R_SP) != S_R_SP)
230                         return parent_ra_addr;
231
232                 /* Move to the next instruction */
233                 ip -= 4;
234         } while ((code & S_RA_SP) != S_RA_SP);
235
236         sp = fp + (code & OFFSET_MASK);
237
238         /* tmp = *(unsigned long *)sp; */
239         safe_load_stack(tmp, sp, faulted);
240         if (unlikely(faulted))
241                 return 0;
242
243         if (tmp == old_parent_ra)
244                 return sp;
245         return 0;
246 }
247
248 #endif  /* !KBUILD_MCOUNT_RA_ADDRESS */
249
250 /*
251  * Hook the return address and push it in the stack of return addrs
252  * in current thread info.
253  */
254 void prepare_ftrace_return(unsigned long *parent_ra_addr, unsigned long self_ra,
255                            unsigned long fp)
256 {
257         unsigned long old_parent_ra;
258         struct ftrace_graph_ent trace;
259         unsigned long return_hooker = (unsigned long)
260             &return_to_handler;
261         int faulted, insns;
262
263         if (unlikely(atomic_read(&current->tracing_graph_pause)))
264                 return;
265
266         /*
267          * "parent_ra_addr" is the stack address saved the return address of
268          * the caller of _mcount.
269          *
270          * if the gcc < 4.5, a leaf function does not save the return address
271          * in the stack address, so, we "emulate" one in _mcount's stack space,
272          * and hijack it directly, but for a non-leaf function, it save the
273          * return address to the its own stack space, we can not hijack it
274          * directly, but need to find the real stack address,
275          * ftrace_get_parent_addr() does it!
276          *
277          * if gcc>= 4.5, with the new -mmcount-ra-address option, for a
278          * non-leaf function, the location of the return address will be saved
279          * to $12 for us, and for a leaf function, only put a zero into $12. we
280          * do it in ftrace_graph_caller of mcount.S.
281          */
282
283         /* old_parent_ra = *parent_ra_addr; */
284         safe_load_stack(old_parent_ra, parent_ra_addr, faulted);
285         if (unlikely(faulted))
286                 goto out;
287 #ifndef KBUILD_MCOUNT_RA_ADDRESS
288         parent_ra_addr = (unsigned long *)ftrace_get_parent_ra_addr(self_ra,
289                         old_parent_ra, (unsigned long)parent_ra_addr, fp);
290         /*
291          * If fails when getting the stack address of the non-leaf function's
292          * ra, stop function graph tracer and return
293          */
294         if (parent_ra_addr == 0)
295                 goto out;
296 #endif
297         /* *parent_ra_addr = return_hooker; */
298         safe_store_stack(return_hooker, parent_ra_addr, faulted);
299         if (unlikely(faulted))
300                 goto out;
301
302         if (ftrace_push_return_trace(old_parent_ra, self_ra, &trace.depth, fp)
303             == -EBUSY) {
304                 *parent_ra_addr = old_parent_ra;
305                 return;
306         }
307
308         /*
309          * Get the recorded ip of the current mcount calling site in the
310          * __mcount_loc section, which will be used to filter the function
311          * entries configured through the tracing/set_graph_function interface.
312          */
313
314         insns = in_kernel_space(self_ra) ? 2 : MCOUNT_OFFSET_INSNS + 1;
315         trace.func = self_ra - (MCOUNT_INSN_SIZE * insns);
316
317         /* Only trace if the calling function expects to */
318         if (!ftrace_graph_entry(&trace)) {
319                 current->curr_ret_stack--;
320                 *parent_ra_addr = old_parent_ra;
321         }
322         return;
323 out:
324         ftrace_graph_stop();
325         WARN_ON(1);
326 }
327 #endif  /* CONFIG_FUNCTION_GRAPH_TRACER */