tracing: Enable records during the module load
[linux-2.6.git] / kernel / trace / ftrace.c
1 /*
2  * Infrastructure for profiling code inserted by 'gcc -pg'.
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally ported from the -rt patch by:
8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code in the latency_tracer, that is:
11  *
12  *  Copyright (C) 2004-2006 Ingo Molnar
13  *  Copyright (C) 2004 William Lee Irwin III
14  */
15
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/suspend.h>
21 #include <linux/debugfs.h>
22 #include <linux/hardirq.h>
23 #include <linux/kthread.h>
24 #include <linux/uaccess.h>
25 #include <linux/kprobes.h>
26 #include <linux/ftrace.h>
27 #include <linux/sysctl.h>
28 #include <linux/ctype.h>
29 #include <linux/list.h>
30 #include <linux/hash.h>
31
32 #include <trace/events/sched.h>
33
34 #include <asm/ftrace.h>
35 #include <asm/setup.h>
36
37 #include "trace_output.h"
38 #include "trace_stat.h"
39
40 #define FTRACE_WARN_ON(cond)                    \
41         do {                                    \
42                 if (WARN_ON(cond))              \
43                         ftrace_kill();          \
44         } while (0)
45
46 #define FTRACE_WARN_ON_ONCE(cond)               \
47         do {                                    \
48                 if (WARN_ON_ONCE(cond))         \
49                         ftrace_kill();          \
50         } while (0)
51
52 /* hash bits for specific function selection */
53 #define FTRACE_HASH_BITS 7
54 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
55
56 /* ftrace_enabled is a method to turn ftrace on or off */
57 int ftrace_enabled __read_mostly;
58 static int last_ftrace_enabled;
59
60 /* Quick disabling of function tracer. */
61 int function_trace_stop;
62
63 /* List for set_ftrace_pid's pids. */
64 LIST_HEAD(ftrace_pids);
65 struct ftrace_pid {
66         struct list_head list;
67         struct pid *pid;
68 };
69
70 /*
71  * ftrace_disabled is set when an anomaly is discovered.
72  * ftrace_disabled is much stronger than ftrace_enabled.
73  */
74 static int ftrace_disabled __read_mostly;
75
76 static DEFINE_MUTEX(ftrace_lock);
77
78 static struct ftrace_ops ftrace_list_end __read_mostly =
79 {
80         .func           = ftrace_stub,
81 };
82
83 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
84 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
85 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
86 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
87
88 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
89 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
90 #endif
91
92 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
93 {
94         struct ftrace_ops *op = ftrace_list;
95
96         /* in case someone actually ports this to alpha! */
97         read_barrier_depends();
98
99         while (op != &ftrace_list_end) {
100                 /* silly alpha */
101                 read_barrier_depends();
102                 op->func(ip, parent_ip);
103                 op = op->next;
104         };
105 }
106
107 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
108 {
109         if (!test_tsk_trace_trace(current))
110                 return;
111
112         ftrace_pid_function(ip, parent_ip);
113 }
114
115 static void set_ftrace_pid_function(ftrace_func_t func)
116 {
117         /* do not set ftrace_pid_function to itself! */
118         if (func != ftrace_pid_func)
119                 ftrace_pid_function = func;
120 }
121
122 /**
123  * clear_ftrace_function - reset the ftrace function
124  *
125  * This NULLs the ftrace function and in essence stops
126  * tracing.  There may be lag
127  */
128 void clear_ftrace_function(void)
129 {
130         ftrace_trace_function = ftrace_stub;
131         __ftrace_trace_function = ftrace_stub;
132         ftrace_pid_function = ftrace_stub;
133 }
134
135 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
136 /*
137  * For those archs that do not test ftrace_trace_stop in their
138  * mcount call site, we need to do it from C.
139  */
140 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
141 {
142         if (function_trace_stop)
143                 return;
144
145         __ftrace_trace_function(ip, parent_ip);
146 }
147 #endif
148
149 static int __register_ftrace_function(struct ftrace_ops *ops)
150 {
151         ops->next = ftrace_list;
152         /*
153          * We are entering ops into the ftrace_list but another
154          * CPU might be walking that list. We need to make sure
155          * the ops->next pointer is valid before another CPU sees
156          * the ops pointer included into the ftrace_list.
157          */
158         smp_wmb();
159         ftrace_list = ops;
160
161         if (ftrace_enabled) {
162                 ftrace_func_t func;
163
164                 if (ops->next == &ftrace_list_end)
165                         func = ops->func;
166                 else
167                         func = ftrace_list_func;
168
169                 if (!list_empty(&ftrace_pids)) {
170                         set_ftrace_pid_function(func);
171                         func = ftrace_pid_func;
172                 }
173
174                 /*
175                  * For one func, simply call it directly.
176                  * For more than one func, call the chain.
177                  */
178 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
179                 ftrace_trace_function = func;
180 #else
181                 __ftrace_trace_function = func;
182                 ftrace_trace_function = ftrace_test_stop_func;
183 #endif
184         }
185
186         return 0;
187 }
188
189 static int __unregister_ftrace_function(struct ftrace_ops *ops)
190 {
191         struct ftrace_ops **p;
192
193         /*
194          * If we are removing the last function, then simply point
195          * to the ftrace_stub.
196          */
197         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
198                 ftrace_trace_function = ftrace_stub;
199                 ftrace_list = &ftrace_list_end;
200                 return 0;
201         }
202
203         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
204                 if (*p == ops)
205                         break;
206
207         if (*p != ops)
208                 return -1;
209
210         *p = (*p)->next;
211
212         if (ftrace_enabled) {
213                 /* If we only have one func left, then call that directly */
214                 if (ftrace_list->next == &ftrace_list_end) {
215                         ftrace_func_t func = ftrace_list->func;
216
217                         if (!list_empty(&ftrace_pids)) {
218                                 set_ftrace_pid_function(func);
219                                 func = ftrace_pid_func;
220                         }
221 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
222                         ftrace_trace_function = func;
223 #else
224                         __ftrace_trace_function = func;
225 #endif
226                 }
227         }
228
229         return 0;
230 }
231
232 static void ftrace_update_pid_func(void)
233 {
234         ftrace_func_t func;
235
236         if (ftrace_trace_function == ftrace_stub)
237                 return;
238
239 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
240         func = ftrace_trace_function;
241 #else
242         func = __ftrace_trace_function;
243 #endif
244
245         if (!list_empty(&ftrace_pids)) {
246                 set_ftrace_pid_function(func);
247                 func = ftrace_pid_func;
248         } else {
249                 if (func == ftrace_pid_func)
250                         func = ftrace_pid_function;
251         }
252
253 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
254         ftrace_trace_function = func;
255 #else
256         __ftrace_trace_function = func;
257 #endif
258 }
259
260 #ifdef CONFIG_FUNCTION_PROFILER
261 struct ftrace_profile {
262         struct hlist_node               node;
263         unsigned long                   ip;
264         unsigned long                   counter;
265 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
266         unsigned long long              time;
267 #endif
268 };
269
270 struct ftrace_profile_page {
271         struct ftrace_profile_page      *next;
272         unsigned long                   index;
273         struct ftrace_profile           records[];
274 };
275
276 struct ftrace_profile_stat {
277         atomic_t                        disabled;
278         struct hlist_head               *hash;
279         struct ftrace_profile_page      *pages;
280         struct ftrace_profile_page      *start;
281         struct tracer_stat              stat;
282 };
283
284 #define PROFILE_RECORDS_SIZE                                            \
285         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
286
287 #define PROFILES_PER_PAGE                                       \
288         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
289
290 static int ftrace_profile_bits __read_mostly;
291 static int ftrace_profile_enabled __read_mostly;
292
293 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
294 static DEFINE_MUTEX(ftrace_profile_lock);
295
296 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
297
298 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
299
300 static void *
301 function_stat_next(void *v, int idx)
302 {
303         struct ftrace_profile *rec = v;
304         struct ftrace_profile_page *pg;
305
306         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
307
308  again:
309         if (idx != 0)
310                 rec++;
311
312         if ((void *)rec >= (void *)&pg->records[pg->index]) {
313                 pg = pg->next;
314                 if (!pg)
315                         return NULL;
316                 rec = &pg->records[0];
317                 if (!rec->counter)
318                         goto again;
319         }
320
321         return rec;
322 }
323
324 static void *function_stat_start(struct tracer_stat *trace)
325 {
326         struct ftrace_profile_stat *stat =
327                 container_of(trace, struct ftrace_profile_stat, stat);
328
329         if (!stat || !stat->start)
330                 return NULL;
331
332         return function_stat_next(&stat->start->records[0], 0);
333 }
334
335 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
336 /* function graph compares on total time */
337 static int function_stat_cmp(void *p1, void *p2)
338 {
339         struct ftrace_profile *a = p1;
340         struct ftrace_profile *b = p2;
341
342         if (a->time < b->time)
343                 return -1;
344         if (a->time > b->time)
345                 return 1;
346         else
347                 return 0;
348 }
349 #else
350 /* not function graph compares against hits */
351 static int function_stat_cmp(void *p1, void *p2)
352 {
353         struct ftrace_profile *a = p1;
354         struct ftrace_profile *b = p2;
355
356         if (a->counter < b->counter)
357                 return -1;
358         if (a->counter > b->counter)
359                 return 1;
360         else
361                 return 0;
362 }
363 #endif
364
365 static int function_stat_headers(struct seq_file *m)
366 {
367 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
368         seq_printf(m, "  Function                               "
369                    "Hit    Time            Avg\n"
370                       "  --------                               "
371                    "---    ----            ---\n");
372 #else
373         seq_printf(m, "  Function                               Hit\n"
374                       "  --------                               ---\n");
375 #endif
376         return 0;
377 }
378
379 static int function_stat_show(struct seq_file *m, void *v)
380 {
381         struct ftrace_profile *rec = v;
382         char str[KSYM_SYMBOL_LEN];
383 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
384         static DEFINE_MUTEX(mutex);
385         static struct trace_seq s;
386         unsigned long long avg;
387 #endif
388
389         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
390         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
391
392 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
393         seq_printf(m, "    ");
394         avg = rec->time;
395         do_div(avg, rec->counter);
396
397         mutex_lock(&mutex);
398         trace_seq_init(&s);
399         trace_print_graph_duration(rec->time, &s);
400         trace_seq_puts(&s, "    ");
401         trace_print_graph_duration(avg, &s);
402         trace_print_seq(m, &s);
403         mutex_unlock(&mutex);
404 #endif
405         seq_putc(m, '\n');
406
407         return 0;
408 }
409
410 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
411 {
412         struct ftrace_profile_page *pg;
413
414         pg = stat->pages = stat->start;
415
416         while (pg) {
417                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
418                 pg->index = 0;
419                 pg = pg->next;
420         }
421
422         memset(stat->hash, 0,
423                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
424 }
425
426 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
427 {
428         struct ftrace_profile_page *pg;
429         int functions;
430         int pages;
431         int i;
432
433         /* If we already allocated, do nothing */
434         if (stat->pages)
435                 return 0;
436
437         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
438         if (!stat->pages)
439                 return -ENOMEM;
440
441 #ifdef CONFIG_DYNAMIC_FTRACE
442         functions = ftrace_update_tot_cnt;
443 #else
444         /*
445          * We do not know the number of functions that exist because
446          * dynamic tracing is what counts them. With past experience
447          * we have around 20K functions. That should be more than enough.
448          * It is highly unlikely we will execute every function in
449          * the kernel.
450          */
451         functions = 20000;
452 #endif
453
454         pg = stat->start = stat->pages;
455
456         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
457
458         for (i = 0; i < pages; i++) {
459                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
460                 if (!pg->next)
461                         goto out_free;
462                 pg = pg->next;
463         }
464
465         return 0;
466
467  out_free:
468         pg = stat->start;
469         while (pg) {
470                 unsigned long tmp = (unsigned long)pg;
471
472                 pg = pg->next;
473                 free_page(tmp);
474         }
475
476         free_page((unsigned long)stat->pages);
477         stat->pages = NULL;
478         stat->start = NULL;
479
480         return -ENOMEM;
481 }
482
483 static int ftrace_profile_init_cpu(int cpu)
484 {
485         struct ftrace_profile_stat *stat;
486         int size;
487
488         stat = &per_cpu(ftrace_profile_stats, cpu);
489
490         if (stat->hash) {
491                 /* If the profile is already created, simply reset it */
492                 ftrace_profile_reset(stat);
493                 return 0;
494         }
495
496         /*
497          * We are profiling all functions, but usually only a few thousand
498          * functions are hit. We'll make a hash of 1024 items.
499          */
500         size = FTRACE_PROFILE_HASH_SIZE;
501
502         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
503
504         if (!stat->hash)
505                 return -ENOMEM;
506
507         if (!ftrace_profile_bits) {
508                 size--;
509
510                 for (; size; size >>= 1)
511                         ftrace_profile_bits++;
512         }
513
514         /* Preallocate the function profiling pages */
515         if (ftrace_profile_pages_init(stat) < 0) {
516                 kfree(stat->hash);
517                 stat->hash = NULL;
518                 return -ENOMEM;
519         }
520
521         return 0;
522 }
523
524 static int ftrace_profile_init(void)
525 {
526         int cpu;
527         int ret = 0;
528
529         for_each_online_cpu(cpu) {
530                 ret = ftrace_profile_init_cpu(cpu);
531                 if (ret)
532                         break;
533         }
534
535         return ret;
536 }
537
538 /* interrupts must be disabled */
539 static struct ftrace_profile *
540 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
541 {
542         struct ftrace_profile *rec;
543         struct hlist_head *hhd;
544         struct hlist_node *n;
545         unsigned long key;
546
547         key = hash_long(ip, ftrace_profile_bits);
548         hhd = &stat->hash[key];
549
550         if (hlist_empty(hhd))
551                 return NULL;
552
553         hlist_for_each_entry_rcu(rec, n, hhd, node) {
554                 if (rec->ip == ip)
555                         return rec;
556         }
557
558         return NULL;
559 }
560
561 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
562                                struct ftrace_profile *rec)
563 {
564         unsigned long key;
565
566         key = hash_long(rec->ip, ftrace_profile_bits);
567         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
568 }
569
570 /*
571  * The memory is already allocated, this simply finds a new record to use.
572  */
573 static struct ftrace_profile *
574 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
575 {
576         struct ftrace_profile *rec = NULL;
577
578         /* prevent recursion (from NMIs) */
579         if (atomic_inc_return(&stat->disabled) != 1)
580                 goto out;
581
582         /*
583          * Try to find the function again since an NMI
584          * could have added it
585          */
586         rec = ftrace_find_profiled_func(stat, ip);
587         if (rec)
588                 goto out;
589
590         if (stat->pages->index == PROFILES_PER_PAGE) {
591                 if (!stat->pages->next)
592                         goto out;
593                 stat->pages = stat->pages->next;
594         }
595
596         rec = &stat->pages->records[stat->pages->index++];
597         rec->ip = ip;
598         ftrace_add_profile(stat, rec);
599
600  out:
601         atomic_dec(&stat->disabled);
602
603         return rec;
604 }
605
606 static void
607 function_profile_call(unsigned long ip, unsigned long parent_ip)
608 {
609         struct ftrace_profile_stat *stat;
610         struct ftrace_profile *rec;
611         unsigned long flags;
612
613         if (!ftrace_profile_enabled)
614                 return;
615
616         local_irq_save(flags);
617
618         stat = &__get_cpu_var(ftrace_profile_stats);
619         if (!stat->hash || !ftrace_profile_enabled)
620                 goto out;
621
622         rec = ftrace_find_profiled_func(stat, ip);
623         if (!rec) {
624                 rec = ftrace_profile_alloc(stat, ip);
625                 if (!rec)
626                         goto out;
627         }
628
629         rec->counter++;
630  out:
631         local_irq_restore(flags);
632 }
633
634 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
635 static int profile_graph_entry(struct ftrace_graph_ent *trace)
636 {
637         function_profile_call(trace->func, 0);
638         return 1;
639 }
640
641 static void profile_graph_return(struct ftrace_graph_ret *trace)
642 {
643         struct ftrace_profile_stat *stat;
644         unsigned long long calltime;
645         struct ftrace_profile *rec;
646         unsigned long flags;
647
648         local_irq_save(flags);
649         stat = &__get_cpu_var(ftrace_profile_stats);
650         if (!stat->hash || !ftrace_profile_enabled)
651                 goto out;
652
653         calltime = trace->rettime - trace->calltime;
654
655         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
656                 int index;
657
658                 index = trace->depth;
659
660                 /* Append this call time to the parent time to subtract */
661                 if (index)
662                         current->ret_stack[index - 1].subtime += calltime;
663
664                 if (current->ret_stack[index].subtime < calltime)
665                         calltime -= current->ret_stack[index].subtime;
666                 else
667                         calltime = 0;
668         }
669
670         rec = ftrace_find_profiled_func(stat, trace->func);
671         if (rec)
672                 rec->time += calltime;
673
674  out:
675         local_irq_restore(flags);
676 }
677
678 static int register_ftrace_profiler(void)
679 {
680         return register_ftrace_graph(&profile_graph_return,
681                                      &profile_graph_entry);
682 }
683
684 static void unregister_ftrace_profiler(void)
685 {
686         unregister_ftrace_graph();
687 }
688 #else
689 static struct ftrace_ops ftrace_profile_ops __read_mostly =
690 {
691         .func           = function_profile_call,
692 };
693
694 static int register_ftrace_profiler(void)
695 {
696         return register_ftrace_function(&ftrace_profile_ops);
697 }
698
699 static void unregister_ftrace_profiler(void)
700 {
701         unregister_ftrace_function(&ftrace_profile_ops);
702 }
703 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
704
705 static ssize_t
706 ftrace_profile_write(struct file *filp, const char __user *ubuf,
707                      size_t cnt, loff_t *ppos)
708 {
709         unsigned long val;
710         char buf[64];           /* big enough to hold a number */
711         int ret;
712
713         if (cnt >= sizeof(buf))
714                 return -EINVAL;
715
716         if (copy_from_user(&buf, ubuf, cnt))
717                 return -EFAULT;
718
719         buf[cnt] = 0;
720
721         ret = strict_strtoul(buf, 10, &val);
722         if (ret < 0)
723                 return ret;
724
725         val = !!val;
726
727         mutex_lock(&ftrace_profile_lock);
728         if (ftrace_profile_enabled ^ val) {
729                 if (val) {
730                         ret = ftrace_profile_init();
731                         if (ret < 0) {
732                                 cnt = ret;
733                                 goto out;
734                         }
735
736                         ret = register_ftrace_profiler();
737                         if (ret < 0) {
738                                 cnt = ret;
739                                 goto out;
740                         }
741                         ftrace_profile_enabled = 1;
742                 } else {
743                         ftrace_profile_enabled = 0;
744                         /*
745                          * unregister_ftrace_profiler calls stop_machine
746                          * so this acts like an synchronize_sched.
747                          */
748                         unregister_ftrace_profiler();
749                 }
750         }
751  out:
752         mutex_unlock(&ftrace_profile_lock);
753
754         filp->f_pos += cnt;
755
756         return cnt;
757 }
758
759 static ssize_t
760 ftrace_profile_read(struct file *filp, char __user *ubuf,
761                      size_t cnt, loff_t *ppos)
762 {
763         char buf[64];           /* big enough to hold a number */
764         int r;
765
766         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
767         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
768 }
769
770 static const struct file_operations ftrace_profile_fops = {
771         .open           = tracing_open_generic,
772         .read           = ftrace_profile_read,
773         .write          = ftrace_profile_write,
774 };
775
776 /* used to initialize the real stat files */
777 static struct tracer_stat function_stats __initdata = {
778         .name           = "functions",
779         .stat_start     = function_stat_start,
780         .stat_next      = function_stat_next,
781         .stat_cmp       = function_stat_cmp,
782         .stat_headers   = function_stat_headers,
783         .stat_show      = function_stat_show
784 };
785
786 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
787 {
788         struct ftrace_profile_stat *stat;
789         struct dentry *entry;
790         char *name;
791         int ret;
792         int cpu;
793
794         for_each_possible_cpu(cpu) {
795                 stat = &per_cpu(ftrace_profile_stats, cpu);
796
797                 /* allocate enough for function name + cpu number */
798                 name = kmalloc(32, GFP_KERNEL);
799                 if (!name) {
800                         /*
801                          * The files created are permanent, if something happens
802                          * we still do not free memory.
803                          */
804                         WARN(1,
805                              "Could not allocate stat file for cpu %d\n",
806                              cpu);
807                         return;
808                 }
809                 stat->stat = function_stats;
810                 snprintf(name, 32, "function%d", cpu);
811                 stat->stat.name = name;
812                 ret = register_stat_tracer(&stat->stat);
813                 if (ret) {
814                         WARN(1,
815                              "Could not register function stat for cpu %d\n",
816                              cpu);
817                         kfree(name);
818                         return;
819                 }
820         }
821
822         entry = debugfs_create_file("function_profile_enabled", 0644,
823                                     d_tracer, NULL, &ftrace_profile_fops);
824         if (!entry)
825                 pr_warning("Could not create debugfs "
826                            "'function_profile_enabled' entry\n");
827 }
828
829 #else /* CONFIG_FUNCTION_PROFILER */
830 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
831 {
832 }
833 #endif /* CONFIG_FUNCTION_PROFILER */
834
835 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
836
837 #ifdef CONFIG_DYNAMIC_FTRACE
838
839 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
840 # error Dynamic ftrace depends on MCOUNT_RECORD
841 #endif
842
843 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
844
845 struct ftrace_func_probe {
846         struct hlist_node       node;
847         struct ftrace_probe_ops *ops;
848         unsigned long           flags;
849         unsigned long           ip;
850         void                    *data;
851         struct rcu_head         rcu;
852 };
853
854 enum {
855         FTRACE_ENABLE_CALLS             = (1 << 0),
856         FTRACE_DISABLE_CALLS            = (1 << 1),
857         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
858         FTRACE_ENABLE_MCOUNT            = (1 << 3),
859         FTRACE_DISABLE_MCOUNT           = (1 << 4),
860         FTRACE_START_FUNC_RET           = (1 << 5),
861         FTRACE_STOP_FUNC_RET            = (1 << 6),
862 };
863
864 static int ftrace_filtered;
865
866 static struct dyn_ftrace *ftrace_new_addrs;
867
868 static DEFINE_MUTEX(ftrace_regex_lock);
869
870 struct ftrace_page {
871         struct ftrace_page      *next;
872         int                     index;
873         struct dyn_ftrace       records[];
874 };
875
876 #define ENTRIES_PER_PAGE \
877   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
878
879 /* estimate from running different kernels */
880 #define NR_TO_INIT              10000
881
882 static struct ftrace_page       *ftrace_pages_start;
883 static struct ftrace_page       *ftrace_pages;
884
885 static struct dyn_ftrace *ftrace_free_records;
886
887 /*
888  * This is a double for. Do not use 'break' to break out of the loop,
889  * you must use a goto.
890  */
891 #define do_for_each_ftrace_rec(pg, rec)                                 \
892         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
893                 int _____i;                                             \
894                 for (_____i = 0; _____i < pg->index; _____i++) {        \
895                         rec = &pg->records[_____i];
896
897 #define while_for_each_ftrace_rec()             \
898                 }                               \
899         }
900
901 #ifdef CONFIG_KPROBES
902
903 static int frozen_record_count;
904
905 static inline void freeze_record(struct dyn_ftrace *rec)
906 {
907         if (!(rec->flags & FTRACE_FL_FROZEN)) {
908                 rec->flags |= FTRACE_FL_FROZEN;
909                 frozen_record_count++;
910         }
911 }
912
913 static inline void unfreeze_record(struct dyn_ftrace *rec)
914 {
915         if (rec->flags & FTRACE_FL_FROZEN) {
916                 rec->flags &= ~FTRACE_FL_FROZEN;
917                 frozen_record_count--;
918         }
919 }
920
921 static inline int record_frozen(struct dyn_ftrace *rec)
922 {
923         return rec->flags & FTRACE_FL_FROZEN;
924 }
925 #else
926 # define freeze_record(rec)                     ({ 0; })
927 # define unfreeze_record(rec)                   ({ 0; })
928 # define record_frozen(rec)                     ({ 0; })
929 #endif /* CONFIG_KPROBES */
930
931 static void ftrace_free_rec(struct dyn_ftrace *rec)
932 {
933         rec->freelist = ftrace_free_records;
934         ftrace_free_records = rec;
935         rec->flags |= FTRACE_FL_FREE;
936 }
937
938 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
939 {
940         struct dyn_ftrace *rec;
941
942         /* First check for freed records */
943         if (ftrace_free_records) {
944                 rec = ftrace_free_records;
945
946                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
947                         FTRACE_WARN_ON_ONCE(1);
948                         ftrace_free_records = NULL;
949                         return NULL;
950                 }
951
952                 ftrace_free_records = rec->freelist;
953                 memset(rec, 0, sizeof(*rec));
954                 return rec;
955         }
956
957         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
958                 if (!ftrace_pages->next) {
959                         /* allocate another page */
960                         ftrace_pages->next =
961                                 (void *)get_zeroed_page(GFP_KERNEL);
962                         if (!ftrace_pages->next)
963                                 return NULL;
964                 }
965                 ftrace_pages = ftrace_pages->next;
966         }
967
968         return &ftrace_pages->records[ftrace_pages->index++];
969 }
970
971 static struct dyn_ftrace *
972 ftrace_record_ip(unsigned long ip)
973 {
974         struct dyn_ftrace *rec;
975
976         if (ftrace_disabled)
977                 return NULL;
978
979         rec = ftrace_alloc_dyn_node(ip);
980         if (!rec)
981                 return NULL;
982
983         rec->ip = ip;
984         rec->newlist = ftrace_new_addrs;
985         ftrace_new_addrs = rec;
986
987         return rec;
988 }
989
990 static void print_ip_ins(const char *fmt, unsigned char *p)
991 {
992         int i;
993
994         printk(KERN_CONT "%s", fmt);
995
996         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
997                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
998 }
999
1000 static void ftrace_bug(int failed, unsigned long ip)
1001 {
1002         switch (failed) {
1003         case -EFAULT:
1004                 FTRACE_WARN_ON_ONCE(1);
1005                 pr_info("ftrace faulted on modifying ");
1006                 print_ip_sym(ip);
1007                 break;
1008         case -EINVAL:
1009                 FTRACE_WARN_ON_ONCE(1);
1010                 pr_info("ftrace failed to modify ");
1011                 print_ip_sym(ip);
1012                 print_ip_ins(" actual: ", (unsigned char *)ip);
1013                 printk(KERN_CONT "\n");
1014                 break;
1015         case -EPERM:
1016                 FTRACE_WARN_ON_ONCE(1);
1017                 pr_info("ftrace faulted on writing ");
1018                 print_ip_sym(ip);
1019                 break;
1020         default:
1021                 FTRACE_WARN_ON_ONCE(1);
1022                 pr_info("ftrace faulted on unknown error ");
1023                 print_ip_sym(ip);
1024         }
1025 }
1026
1027
1028 static int
1029 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1030 {
1031         unsigned long ftrace_addr;
1032         unsigned long flag = 0UL;
1033
1034         ftrace_addr = (unsigned long)FTRACE_ADDR;
1035
1036         /*
1037          * If this record is not to be traced or we want to disable it,
1038          * then disable it.
1039          *
1040          * If we want to enable it and filtering is off, then enable it.
1041          *
1042          * If we want to enable it and filtering is on, enable it only if
1043          * it's filtered
1044          */
1045         if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) {
1046                 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
1047                         flag = FTRACE_FL_ENABLED;
1048         }
1049
1050         /* If the state of this record hasn't changed, then do nothing */
1051         if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1052                 return 0;
1053
1054         if (flag) {
1055                 rec->flags |= FTRACE_FL_ENABLED;
1056                 return ftrace_make_call(rec, ftrace_addr);
1057         }
1058
1059         rec->flags &= ~FTRACE_FL_ENABLED;
1060         return ftrace_make_nop(NULL, rec, ftrace_addr);
1061 }
1062
1063 static void ftrace_replace_code(int enable)
1064 {
1065         struct dyn_ftrace *rec;
1066         struct ftrace_page *pg;
1067         int failed;
1068
1069         do_for_each_ftrace_rec(pg, rec) {
1070                 /*
1071                  * Skip over free records, records that have
1072                  * failed and not converted.
1073                  */
1074                 if (rec->flags & FTRACE_FL_FREE ||
1075                     rec->flags & FTRACE_FL_FAILED ||
1076                     !(rec->flags & FTRACE_FL_CONVERTED))
1077                         continue;
1078
1079                 /* ignore updates to this record's mcount site */
1080                 if (get_kprobe((void *)rec->ip)) {
1081                         freeze_record(rec);
1082                         continue;
1083                 } else {
1084                         unfreeze_record(rec);
1085                 }
1086
1087                 failed = __ftrace_replace_code(rec, enable);
1088                 if (failed) {
1089                         rec->flags |= FTRACE_FL_FAILED;
1090                         ftrace_bug(failed, rec->ip);
1091                         /* Stop processing */
1092                         return;
1093                 }
1094         } while_for_each_ftrace_rec();
1095 }
1096
1097 static int
1098 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1099 {
1100         unsigned long ip;
1101         int ret;
1102
1103         ip = rec->ip;
1104
1105         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1106         if (ret) {
1107                 ftrace_bug(ret, ip);
1108                 rec->flags |= FTRACE_FL_FAILED;
1109                 return 0;
1110         }
1111         return 1;
1112 }
1113
1114 /*
1115  * archs can override this function if they must do something
1116  * before the modifying code is performed.
1117  */
1118 int __weak ftrace_arch_code_modify_prepare(void)
1119 {
1120         return 0;
1121 }
1122
1123 /*
1124  * archs can override this function if they must do something
1125  * after the modifying code is performed.
1126  */
1127 int __weak ftrace_arch_code_modify_post_process(void)
1128 {
1129         return 0;
1130 }
1131
1132 static int __ftrace_modify_code(void *data)
1133 {
1134         int *command = data;
1135
1136         if (*command & FTRACE_ENABLE_CALLS)
1137                 ftrace_replace_code(1);
1138         else if (*command & FTRACE_DISABLE_CALLS)
1139                 ftrace_replace_code(0);
1140
1141         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1142                 ftrace_update_ftrace_func(ftrace_trace_function);
1143
1144         if (*command & FTRACE_START_FUNC_RET)
1145                 ftrace_enable_ftrace_graph_caller();
1146         else if (*command & FTRACE_STOP_FUNC_RET)
1147                 ftrace_disable_ftrace_graph_caller();
1148
1149         return 0;
1150 }
1151
1152 static void ftrace_run_update_code(int command)
1153 {
1154         int ret;
1155
1156         ret = ftrace_arch_code_modify_prepare();
1157         FTRACE_WARN_ON(ret);
1158         if (ret)
1159                 return;
1160
1161         stop_machine(__ftrace_modify_code, &command, NULL);
1162
1163         ret = ftrace_arch_code_modify_post_process();
1164         FTRACE_WARN_ON(ret);
1165 }
1166
1167 static ftrace_func_t saved_ftrace_func;
1168 static int ftrace_start_up;
1169
1170 static void ftrace_startup_enable(int command)
1171 {
1172         if (saved_ftrace_func != ftrace_trace_function) {
1173                 saved_ftrace_func = ftrace_trace_function;
1174                 command |= FTRACE_UPDATE_TRACE_FUNC;
1175         }
1176
1177         if (!command || !ftrace_enabled)
1178                 return;
1179
1180         ftrace_run_update_code(command);
1181 }
1182
1183 static void ftrace_startup(int command)
1184 {
1185         if (unlikely(ftrace_disabled))
1186                 return;
1187
1188         ftrace_start_up++;
1189         command |= FTRACE_ENABLE_CALLS;
1190
1191         ftrace_startup_enable(command);
1192 }
1193
1194 static void ftrace_shutdown(int command)
1195 {
1196         if (unlikely(ftrace_disabled))
1197                 return;
1198
1199         ftrace_start_up--;
1200         /*
1201          * Just warn in case of unbalance, no need to kill ftrace, it's not
1202          * critical but the ftrace_call callers may be never nopped again after
1203          * further ftrace uses.
1204          */
1205         WARN_ON_ONCE(ftrace_start_up < 0);
1206
1207         if (!ftrace_start_up)
1208                 command |= FTRACE_DISABLE_CALLS;
1209
1210         if (saved_ftrace_func != ftrace_trace_function) {
1211                 saved_ftrace_func = ftrace_trace_function;
1212                 command |= FTRACE_UPDATE_TRACE_FUNC;
1213         }
1214
1215         if (!command || !ftrace_enabled)
1216                 return;
1217
1218         ftrace_run_update_code(command);
1219 }
1220
1221 static void ftrace_startup_sysctl(void)
1222 {
1223         int command = FTRACE_ENABLE_MCOUNT;
1224
1225         if (unlikely(ftrace_disabled))
1226                 return;
1227
1228         /* Force update next time */
1229         saved_ftrace_func = NULL;
1230         /* ftrace_start_up is true if we want ftrace running */
1231         if (ftrace_start_up)
1232                 command |= FTRACE_ENABLE_CALLS;
1233
1234         ftrace_run_update_code(command);
1235 }
1236
1237 static void ftrace_shutdown_sysctl(void)
1238 {
1239         int command = FTRACE_DISABLE_MCOUNT;
1240
1241         if (unlikely(ftrace_disabled))
1242                 return;
1243
1244         /* ftrace_start_up is true if ftrace is running */
1245         if (ftrace_start_up)
1246                 command |= FTRACE_DISABLE_CALLS;
1247
1248         ftrace_run_update_code(command);
1249 }
1250
1251 static cycle_t          ftrace_update_time;
1252 static unsigned long    ftrace_update_cnt;
1253 unsigned long           ftrace_update_tot_cnt;
1254
1255 static int ftrace_update_code(struct module *mod)
1256 {
1257         struct dyn_ftrace *p;
1258         cycle_t start, stop;
1259
1260         start = ftrace_now(raw_smp_processor_id());
1261         ftrace_update_cnt = 0;
1262
1263         while (ftrace_new_addrs) {
1264
1265                 /* If something went wrong, bail without enabling anything */
1266                 if (unlikely(ftrace_disabled))
1267                         return -1;
1268
1269                 p = ftrace_new_addrs;
1270                 ftrace_new_addrs = p->newlist;
1271                 p->flags = 0L;
1272
1273                 /*
1274                  * Do the initial record convertion from mcount jump
1275                  * to the NOP instructions.
1276                  */
1277                 if (!ftrace_code_disable(mod, p)) {
1278                         ftrace_free_rec(p);
1279                         continue;
1280                 }
1281
1282                 p->flags |= FTRACE_FL_CONVERTED;
1283                 ftrace_update_cnt++;
1284
1285                 /*
1286                  * If the tracing is enabled, go ahead and enable the record.
1287                  *
1288                  * The reason not to enable the record immediatelly is the
1289                  * inherent check of ftrace_make_nop/ftrace_make_call for
1290                  * correct previous instructions.  Making first the NOP
1291                  * conversion puts the module to the correct state, thus
1292                  * passing the ftrace_make_call check.
1293                  */
1294                 if (ftrace_start_up) {
1295                         int failed = __ftrace_replace_code(p, 1);
1296                         if (failed) {
1297                                 ftrace_bug(failed, p->ip);
1298                                 ftrace_free_rec(p);
1299                         }
1300                 }
1301         }
1302
1303         stop = ftrace_now(raw_smp_processor_id());
1304         ftrace_update_time = stop - start;
1305         ftrace_update_tot_cnt += ftrace_update_cnt;
1306
1307         return 0;
1308 }
1309
1310 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1311 {
1312         struct ftrace_page *pg;
1313         int cnt;
1314         int i;
1315
1316         /* allocate a few pages */
1317         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1318         if (!ftrace_pages_start)
1319                 return -1;
1320
1321         /*
1322          * Allocate a few more pages.
1323          *
1324          * TODO: have some parser search vmlinux before
1325          *   final linking to find all calls to ftrace.
1326          *   Then we can:
1327          *    a) know how many pages to allocate.
1328          *     and/or
1329          *    b) set up the table then.
1330          *
1331          *  The dynamic code is still necessary for
1332          *  modules.
1333          */
1334
1335         pg = ftrace_pages = ftrace_pages_start;
1336
1337         cnt = num_to_init / ENTRIES_PER_PAGE;
1338         pr_info("ftrace: allocating %ld entries in %d pages\n",
1339                 num_to_init, cnt + 1);
1340
1341         for (i = 0; i < cnt; i++) {
1342                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1343
1344                 /* If we fail, we'll try later anyway */
1345                 if (!pg->next)
1346                         break;
1347
1348                 pg = pg->next;
1349         }
1350
1351         return 0;
1352 }
1353
1354 enum {
1355         FTRACE_ITER_FILTER      = (1 << 0),
1356         FTRACE_ITER_NOTRACE     = (1 << 1),
1357         FTRACE_ITER_FAILURES    = (1 << 2),
1358         FTRACE_ITER_PRINTALL    = (1 << 3),
1359         FTRACE_ITER_HASH        = (1 << 4),
1360 };
1361
1362 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1363
1364 struct ftrace_iterator {
1365         struct ftrace_page      *pg;
1366         int                     hidx;
1367         int                     idx;
1368         unsigned                flags;
1369         struct trace_parser     parser;
1370 };
1371
1372 static void *
1373 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
1374 {
1375         struct ftrace_iterator *iter = m->private;
1376         struct hlist_node *hnd = v;
1377         struct hlist_head *hhd;
1378
1379         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
1380
1381         (*pos)++;
1382
1383  retry:
1384         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1385                 return NULL;
1386
1387         hhd = &ftrace_func_hash[iter->hidx];
1388
1389         if (hlist_empty(hhd)) {
1390                 iter->hidx++;
1391                 hnd = NULL;
1392                 goto retry;
1393         }
1394
1395         if (!hnd)
1396                 hnd = hhd->first;
1397         else {
1398                 hnd = hnd->next;
1399                 if (!hnd) {
1400                         iter->hidx++;
1401                         goto retry;
1402                 }
1403         }
1404
1405         return hnd;
1406 }
1407
1408 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1409 {
1410         struct ftrace_iterator *iter = m->private;
1411         void *p = NULL;
1412         loff_t l;
1413
1414         if (!(iter->flags & FTRACE_ITER_HASH))
1415                 *pos = 0;
1416
1417         iter->flags |= FTRACE_ITER_HASH;
1418
1419         iter->hidx = 0;
1420         for (l = 0; l <= *pos; ) {
1421                 p = t_hash_next(m, p, &l);
1422                 if (!p)
1423                         break;
1424         }
1425         return p;
1426 }
1427
1428 static int t_hash_show(struct seq_file *m, void *v)
1429 {
1430         struct ftrace_func_probe *rec;
1431         struct hlist_node *hnd = v;
1432
1433         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
1434
1435         if (rec->ops->print)
1436                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1437
1438         seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1439
1440         if (rec->data)
1441                 seq_printf(m, ":%p", rec->data);
1442         seq_putc(m, '\n');
1443
1444         return 0;
1445 }
1446
1447 static void *
1448 t_next(struct seq_file *m, void *v, loff_t *pos)
1449 {
1450         struct ftrace_iterator *iter = m->private;
1451         struct dyn_ftrace *rec = NULL;
1452
1453         if (iter->flags & FTRACE_ITER_HASH)
1454                 return t_hash_next(m, v, pos);
1455
1456         (*pos)++;
1457
1458         if (iter->flags & FTRACE_ITER_PRINTALL)
1459                 return NULL;
1460
1461  retry:
1462         if (iter->idx >= iter->pg->index) {
1463                 if (iter->pg->next) {
1464                         iter->pg = iter->pg->next;
1465                         iter->idx = 0;
1466                         goto retry;
1467                 }
1468         } else {
1469                 rec = &iter->pg->records[iter->idx++];
1470                 if ((rec->flags & FTRACE_FL_FREE) ||
1471
1472                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
1473                      (rec->flags & FTRACE_FL_FAILED)) ||
1474
1475                     ((iter->flags & FTRACE_ITER_FAILURES) &&
1476                      !(rec->flags & FTRACE_FL_FAILED)) ||
1477
1478                     ((iter->flags & FTRACE_ITER_FILTER) &&
1479                      !(rec->flags & FTRACE_FL_FILTER)) ||
1480
1481                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
1482                      !(rec->flags & FTRACE_FL_NOTRACE))) {
1483                         rec = NULL;
1484                         goto retry;
1485                 }
1486         }
1487
1488         return rec;
1489 }
1490
1491 static void *t_start(struct seq_file *m, loff_t *pos)
1492 {
1493         struct ftrace_iterator *iter = m->private;
1494         void *p = NULL;
1495         loff_t l;
1496
1497         mutex_lock(&ftrace_lock);
1498         /*
1499          * For set_ftrace_filter reading, if we have the filter
1500          * off, we can short cut and just print out that all
1501          * functions are enabled.
1502          */
1503         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
1504                 if (*pos > 0)
1505                         return t_hash_start(m, pos);
1506                 iter->flags |= FTRACE_ITER_PRINTALL;
1507                 return iter;
1508         }
1509
1510         if (iter->flags & FTRACE_ITER_HASH)
1511                 return t_hash_start(m, pos);
1512
1513         iter->pg = ftrace_pages_start;
1514         iter->idx = 0;
1515         for (l = 0; l <= *pos; ) {
1516                 p = t_next(m, p, &l);
1517                 if (!p)
1518                         break;
1519         }
1520
1521         if (!p && iter->flags & FTRACE_ITER_FILTER)
1522                 return t_hash_start(m, pos);
1523
1524         return p;
1525 }
1526
1527 static void t_stop(struct seq_file *m, void *p)
1528 {
1529         mutex_unlock(&ftrace_lock);
1530 }
1531
1532 static int t_show(struct seq_file *m, void *v)
1533 {
1534         struct ftrace_iterator *iter = m->private;
1535         struct dyn_ftrace *rec = v;
1536
1537         if (iter->flags & FTRACE_ITER_HASH)
1538                 return t_hash_show(m, v);
1539
1540         if (iter->flags & FTRACE_ITER_PRINTALL) {
1541                 seq_printf(m, "#### all functions enabled ####\n");
1542                 return 0;
1543         }
1544
1545         if (!rec)
1546                 return 0;
1547
1548         seq_printf(m, "%ps\n", (void *)rec->ip);
1549
1550         return 0;
1551 }
1552
1553 static const struct seq_operations show_ftrace_seq_ops = {
1554         .start = t_start,
1555         .next = t_next,
1556         .stop = t_stop,
1557         .show = t_show,
1558 };
1559
1560 static int
1561 ftrace_avail_open(struct inode *inode, struct file *file)
1562 {
1563         struct ftrace_iterator *iter;
1564         int ret;
1565
1566         if (unlikely(ftrace_disabled))
1567                 return -ENODEV;
1568
1569         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1570         if (!iter)
1571                 return -ENOMEM;
1572
1573         iter->pg = ftrace_pages_start;
1574
1575         ret = seq_open(file, &show_ftrace_seq_ops);
1576         if (!ret) {
1577                 struct seq_file *m = file->private_data;
1578
1579                 m->private = iter;
1580         } else {
1581                 kfree(iter);
1582         }
1583
1584         return ret;
1585 }
1586
1587 static int
1588 ftrace_failures_open(struct inode *inode, struct file *file)
1589 {
1590         int ret;
1591         struct seq_file *m;
1592         struct ftrace_iterator *iter;
1593
1594         ret = ftrace_avail_open(inode, file);
1595         if (!ret) {
1596                 m = (struct seq_file *)file->private_data;
1597                 iter = (struct ftrace_iterator *)m->private;
1598                 iter->flags = FTRACE_ITER_FAILURES;
1599         }
1600
1601         return ret;
1602 }
1603
1604
1605 static void ftrace_filter_reset(int enable)
1606 {
1607         struct ftrace_page *pg;
1608         struct dyn_ftrace *rec;
1609         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1610
1611         mutex_lock(&ftrace_lock);
1612         if (enable)
1613                 ftrace_filtered = 0;
1614         do_for_each_ftrace_rec(pg, rec) {
1615                 if (rec->flags & FTRACE_FL_FAILED)
1616                         continue;
1617                 rec->flags &= ~type;
1618         } while_for_each_ftrace_rec();
1619         mutex_unlock(&ftrace_lock);
1620 }
1621
1622 static int
1623 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1624 {
1625         struct ftrace_iterator *iter;
1626         int ret = 0;
1627
1628         if (unlikely(ftrace_disabled))
1629                 return -ENODEV;
1630
1631         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1632         if (!iter)
1633                 return -ENOMEM;
1634
1635         if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
1636                 kfree(iter);
1637                 return -ENOMEM;
1638         }
1639
1640         mutex_lock(&ftrace_regex_lock);
1641         if ((file->f_mode & FMODE_WRITE) &&
1642             (file->f_flags & O_TRUNC))
1643                 ftrace_filter_reset(enable);
1644
1645         if (file->f_mode & FMODE_READ) {
1646                 iter->pg = ftrace_pages_start;
1647                 iter->flags = enable ? FTRACE_ITER_FILTER :
1648                         FTRACE_ITER_NOTRACE;
1649
1650                 ret = seq_open(file, &show_ftrace_seq_ops);
1651                 if (!ret) {
1652                         struct seq_file *m = file->private_data;
1653                         m->private = iter;
1654                 } else {
1655                         trace_parser_put(&iter->parser);
1656                         kfree(iter);
1657                 }
1658         } else
1659                 file->private_data = iter;
1660         mutex_unlock(&ftrace_regex_lock);
1661
1662         return ret;
1663 }
1664
1665 static int
1666 ftrace_filter_open(struct inode *inode, struct file *file)
1667 {
1668         return ftrace_regex_open(inode, file, 1);
1669 }
1670
1671 static int
1672 ftrace_notrace_open(struct inode *inode, struct file *file)
1673 {
1674         return ftrace_regex_open(inode, file, 0);
1675 }
1676
1677 static loff_t
1678 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1679 {
1680         loff_t ret;
1681
1682         if (file->f_mode & FMODE_READ)
1683                 ret = seq_lseek(file, offset, origin);
1684         else
1685                 file->f_pos = ret = 1;
1686
1687         return ret;
1688 }
1689
1690 static int ftrace_match(char *str, char *regex, int len, int type)
1691 {
1692         int matched = 0;
1693         char *ptr;
1694
1695         switch (type) {
1696         case MATCH_FULL:
1697                 if (strcmp(str, regex) == 0)
1698                         matched = 1;
1699                 break;
1700         case MATCH_FRONT_ONLY:
1701                 if (strncmp(str, regex, len) == 0)
1702                         matched = 1;
1703                 break;
1704         case MATCH_MIDDLE_ONLY:
1705                 if (strstr(str, regex))
1706                         matched = 1;
1707                 break;
1708         case MATCH_END_ONLY:
1709                 ptr = strstr(str, regex);
1710                 if (ptr && (ptr[len] == 0))
1711                         matched = 1;
1712                 break;
1713         }
1714
1715         return matched;
1716 }
1717
1718 static int
1719 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1720 {
1721         char str[KSYM_SYMBOL_LEN];
1722
1723         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1724         return ftrace_match(str, regex, len, type);
1725 }
1726
1727 static void ftrace_match_records(char *buff, int len, int enable)
1728 {
1729         unsigned int search_len;
1730         struct ftrace_page *pg;
1731         struct dyn_ftrace *rec;
1732         unsigned long flag;
1733         char *search;
1734         int type;
1735         int not;
1736
1737         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1738         type = filter_parse_regex(buff, len, &search, &not);
1739
1740         search_len = strlen(search);
1741
1742         mutex_lock(&ftrace_lock);
1743         do_for_each_ftrace_rec(pg, rec) {
1744
1745                 if (rec->flags & FTRACE_FL_FAILED)
1746                         continue;
1747
1748                 if (ftrace_match_record(rec, search, search_len, type)) {
1749                         if (not)
1750                                 rec->flags &= ~flag;
1751                         else
1752                                 rec->flags |= flag;
1753                 }
1754                 /*
1755                  * Only enable filtering if we have a function that
1756                  * is filtered on.
1757                  */
1758                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1759                         ftrace_filtered = 1;
1760         } while_for_each_ftrace_rec();
1761         mutex_unlock(&ftrace_lock);
1762 }
1763
1764 static int
1765 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1766                            char *regex, int len, int type)
1767 {
1768         char str[KSYM_SYMBOL_LEN];
1769         char *modname;
1770
1771         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1772
1773         if (!modname || strcmp(modname, mod))
1774                 return 0;
1775
1776         /* blank search means to match all funcs in the mod */
1777         if (len)
1778                 return ftrace_match(str, regex, len, type);
1779         else
1780                 return 1;
1781 }
1782
1783 static void ftrace_match_module_records(char *buff, char *mod, int enable)
1784 {
1785         unsigned search_len = 0;
1786         struct ftrace_page *pg;
1787         struct dyn_ftrace *rec;
1788         int type = MATCH_FULL;
1789         char *search = buff;
1790         unsigned long flag;
1791         int not = 0;
1792
1793         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1794
1795         /* blank or '*' mean the same */
1796         if (strcmp(buff, "*") == 0)
1797                 buff[0] = 0;
1798
1799         /* handle the case of 'dont filter this module' */
1800         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1801                 buff[0] = 0;
1802                 not = 1;
1803         }
1804
1805         if (strlen(buff)) {
1806                 type = filter_parse_regex(buff, strlen(buff), &search, &not);
1807                 search_len = strlen(search);
1808         }
1809
1810         mutex_lock(&ftrace_lock);
1811         do_for_each_ftrace_rec(pg, rec) {
1812
1813                 if (rec->flags & FTRACE_FL_FAILED)
1814                         continue;
1815
1816                 if (ftrace_match_module_record(rec, mod,
1817                                                search, search_len, type)) {
1818                         if (not)
1819                                 rec->flags &= ~flag;
1820                         else
1821                                 rec->flags |= flag;
1822                 }
1823                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1824                         ftrace_filtered = 1;
1825
1826         } while_for_each_ftrace_rec();
1827         mutex_unlock(&ftrace_lock);
1828 }
1829
1830 /*
1831  * We register the module command as a template to show others how
1832  * to register the a command as well.
1833  */
1834
1835 static int
1836 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1837 {
1838         char *mod;
1839
1840         /*
1841          * cmd == 'mod' because we only registered this func
1842          * for the 'mod' ftrace_func_command.
1843          * But if you register one func with multiple commands,
1844          * you can tell which command was used by the cmd
1845          * parameter.
1846          */
1847
1848         /* we must have a module name */
1849         if (!param)
1850                 return -EINVAL;
1851
1852         mod = strsep(&param, ":");
1853         if (!strlen(mod))
1854                 return -EINVAL;
1855
1856         ftrace_match_module_records(func, mod, enable);
1857         return 0;
1858 }
1859
1860 static struct ftrace_func_command ftrace_mod_cmd = {
1861         .name                   = "mod",
1862         .func                   = ftrace_mod_callback,
1863 };
1864
1865 static int __init ftrace_mod_cmd_init(void)
1866 {
1867         return register_ftrace_command(&ftrace_mod_cmd);
1868 }
1869 device_initcall(ftrace_mod_cmd_init);
1870
1871 static void
1872 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1873 {
1874         struct ftrace_func_probe *entry;
1875         struct hlist_head *hhd;
1876         struct hlist_node *n;
1877         unsigned long key;
1878         int resched;
1879
1880         key = hash_long(ip, FTRACE_HASH_BITS);
1881
1882         hhd = &ftrace_func_hash[key];
1883
1884         if (hlist_empty(hhd))
1885                 return;
1886
1887         /*
1888          * Disable preemption for these calls to prevent a RCU grace
1889          * period. This syncs the hash iteration and freeing of items
1890          * on the hash. rcu_read_lock is too dangerous here.
1891          */
1892         resched = ftrace_preempt_disable();
1893         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1894                 if (entry->ip == ip)
1895                         entry->ops->func(ip, parent_ip, &entry->data);
1896         }
1897         ftrace_preempt_enable(resched);
1898 }
1899
1900 static struct ftrace_ops trace_probe_ops __read_mostly =
1901 {
1902         .func           = function_trace_probe_call,
1903 };
1904
1905 static int ftrace_probe_registered;
1906
1907 static void __enable_ftrace_function_probe(void)
1908 {
1909         int i;
1910
1911         if (ftrace_probe_registered)
1912                 return;
1913
1914         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1915                 struct hlist_head *hhd = &ftrace_func_hash[i];
1916                 if (hhd->first)
1917                         break;
1918         }
1919         /* Nothing registered? */
1920         if (i == FTRACE_FUNC_HASHSIZE)
1921                 return;
1922
1923         __register_ftrace_function(&trace_probe_ops);
1924         ftrace_startup(0);
1925         ftrace_probe_registered = 1;
1926 }
1927
1928 static void __disable_ftrace_function_probe(void)
1929 {
1930         int i;
1931
1932         if (!ftrace_probe_registered)
1933                 return;
1934
1935         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1936                 struct hlist_head *hhd = &ftrace_func_hash[i];
1937                 if (hhd->first)
1938                         return;
1939         }
1940
1941         /* no more funcs left */
1942         __unregister_ftrace_function(&trace_probe_ops);
1943         ftrace_shutdown(0);
1944         ftrace_probe_registered = 0;
1945 }
1946
1947
1948 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1949 {
1950         struct ftrace_func_probe *entry =
1951                 container_of(rhp, struct ftrace_func_probe, rcu);
1952
1953         if (entry->ops->free)
1954                 entry->ops->free(&entry->data);
1955         kfree(entry);
1956 }
1957
1958
1959 int
1960 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1961                               void *data)
1962 {
1963         struct ftrace_func_probe *entry;
1964         struct ftrace_page *pg;
1965         struct dyn_ftrace *rec;
1966         int type, len, not;
1967         unsigned long key;
1968         int count = 0;
1969         char *search;
1970
1971         type = filter_parse_regex(glob, strlen(glob), &search, &not);
1972         len = strlen(search);
1973
1974         /* we do not support '!' for function probes */
1975         if (WARN_ON(not))
1976                 return -EINVAL;
1977
1978         mutex_lock(&ftrace_lock);
1979         do_for_each_ftrace_rec(pg, rec) {
1980
1981                 if (rec->flags & FTRACE_FL_FAILED)
1982                         continue;
1983
1984                 if (!ftrace_match_record(rec, search, len, type))
1985                         continue;
1986
1987                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1988                 if (!entry) {
1989                         /* If we did not process any, then return error */
1990                         if (!count)
1991                                 count = -ENOMEM;
1992                         goto out_unlock;
1993                 }
1994
1995                 count++;
1996
1997                 entry->data = data;
1998
1999                 /*
2000                  * The caller might want to do something special
2001                  * for each function we find. We call the callback
2002                  * to give the caller an opportunity to do so.
2003                  */
2004                 if (ops->callback) {
2005                         if (ops->callback(rec->ip, &entry->data) < 0) {
2006                                 /* caller does not like this func */
2007                                 kfree(entry);
2008                                 continue;
2009                         }
2010                 }
2011
2012                 entry->ops = ops;
2013                 entry->ip = rec->ip;
2014
2015                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2016                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2017
2018         } while_for_each_ftrace_rec();
2019         __enable_ftrace_function_probe();
2020
2021  out_unlock:
2022         mutex_unlock(&ftrace_lock);
2023
2024         return count;
2025 }
2026
2027 enum {
2028         PROBE_TEST_FUNC         = 1,
2029         PROBE_TEST_DATA         = 2
2030 };
2031
2032 static void
2033 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2034                                   void *data, int flags)
2035 {
2036         struct ftrace_func_probe *entry;
2037         struct hlist_node *n, *tmp;
2038         char str[KSYM_SYMBOL_LEN];
2039         int type = MATCH_FULL;
2040         int i, len = 0;
2041         char *search;
2042
2043         if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2044                 glob = NULL;
2045         else if (glob) {
2046                 int not;
2047
2048                 type = filter_parse_regex(glob, strlen(glob), &search, &not);
2049                 len = strlen(search);
2050
2051                 /* we do not support '!' for function probes */
2052                 if (WARN_ON(not))
2053                         return;
2054         }
2055
2056         mutex_lock(&ftrace_lock);
2057         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2058                 struct hlist_head *hhd = &ftrace_func_hash[i];
2059
2060                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2061
2062                         /* break up if statements for readability */
2063                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2064                                 continue;
2065
2066                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
2067                                 continue;
2068
2069                         /* do this last, since it is the most expensive */
2070                         if (glob) {
2071                                 kallsyms_lookup(entry->ip, NULL, NULL,
2072                                                 NULL, str);
2073                                 if (!ftrace_match(str, glob, len, type))
2074                                         continue;
2075                         }
2076
2077                         hlist_del(&entry->node);
2078                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2079                 }
2080         }
2081         __disable_ftrace_function_probe();
2082         mutex_unlock(&ftrace_lock);
2083 }
2084
2085 void
2086 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2087                                 void *data)
2088 {
2089         __unregister_ftrace_function_probe(glob, ops, data,
2090                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
2091 }
2092
2093 void
2094 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2095 {
2096         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2097 }
2098
2099 void unregister_ftrace_function_probe_all(char *glob)
2100 {
2101         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2102 }
2103
2104 static LIST_HEAD(ftrace_commands);
2105 static DEFINE_MUTEX(ftrace_cmd_mutex);
2106
2107 int register_ftrace_command(struct ftrace_func_command *cmd)
2108 {
2109         struct ftrace_func_command *p;
2110         int ret = 0;
2111
2112         mutex_lock(&ftrace_cmd_mutex);
2113         list_for_each_entry(p, &ftrace_commands, list) {
2114                 if (strcmp(cmd->name, p->name) == 0) {
2115                         ret = -EBUSY;
2116                         goto out_unlock;
2117                 }
2118         }
2119         list_add(&cmd->list, &ftrace_commands);
2120  out_unlock:
2121         mutex_unlock(&ftrace_cmd_mutex);
2122
2123         return ret;
2124 }
2125
2126 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2127 {
2128         struct ftrace_func_command *p, *n;
2129         int ret = -ENODEV;
2130
2131         mutex_lock(&ftrace_cmd_mutex);
2132         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2133                 if (strcmp(cmd->name, p->name) == 0) {
2134                         ret = 0;
2135                         list_del_init(&p->list);
2136                         goto out_unlock;
2137                 }
2138         }
2139  out_unlock:
2140         mutex_unlock(&ftrace_cmd_mutex);
2141
2142         return ret;
2143 }
2144
2145 static int ftrace_process_regex(char *buff, int len, int enable)
2146 {
2147         char *func, *command, *next = buff;
2148         struct ftrace_func_command *p;
2149         int ret = -EINVAL;
2150
2151         func = strsep(&next, ":");
2152
2153         if (!next) {
2154                 ftrace_match_records(func, len, enable);
2155                 return 0;
2156         }
2157
2158         /* command found */
2159
2160         command = strsep(&next, ":");
2161
2162         mutex_lock(&ftrace_cmd_mutex);
2163         list_for_each_entry(p, &ftrace_commands, list) {
2164                 if (strcmp(p->name, command) == 0) {
2165                         ret = p->func(func, command, next, enable);
2166                         goto out_unlock;
2167                 }
2168         }
2169  out_unlock:
2170         mutex_unlock(&ftrace_cmd_mutex);
2171
2172         return ret;
2173 }
2174
2175 static ssize_t
2176 ftrace_regex_write(struct file *file, const char __user *ubuf,
2177                    size_t cnt, loff_t *ppos, int enable)
2178 {
2179         struct ftrace_iterator *iter;
2180         struct trace_parser *parser;
2181         ssize_t ret, read;
2182
2183         if (!cnt)
2184                 return 0;
2185
2186         mutex_lock(&ftrace_regex_lock);
2187
2188         if (file->f_mode & FMODE_READ) {
2189                 struct seq_file *m = file->private_data;
2190                 iter = m->private;
2191         } else
2192                 iter = file->private_data;
2193
2194         parser = &iter->parser;
2195         read = trace_get_user(parser, ubuf, cnt, ppos);
2196
2197         if (read >= 0 && trace_parser_loaded(parser) &&
2198             !trace_parser_cont(parser)) {
2199                 ret = ftrace_process_regex(parser->buffer,
2200                                            parser->idx, enable);
2201                 if (ret)
2202                         goto out;
2203
2204                 trace_parser_clear(parser);
2205         }
2206
2207         ret = read;
2208
2209         mutex_unlock(&ftrace_regex_lock);
2210 out:
2211         return ret;
2212 }
2213
2214 static ssize_t
2215 ftrace_filter_write(struct file *file, const char __user *ubuf,
2216                     size_t cnt, loff_t *ppos)
2217 {
2218         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2219 }
2220
2221 static ssize_t
2222 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2223                      size_t cnt, loff_t *ppos)
2224 {
2225         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2226 }
2227
2228 static void
2229 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2230 {
2231         if (unlikely(ftrace_disabled))
2232                 return;
2233
2234         mutex_lock(&ftrace_regex_lock);
2235         if (reset)
2236                 ftrace_filter_reset(enable);
2237         if (buf)
2238                 ftrace_match_records(buf, len, enable);
2239         mutex_unlock(&ftrace_regex_lock);
2240 }
2241
2242 /**
2243  * ftrace_set_filter - set a function to filter on in ftrace
2244  * @buf - the string that holds the function filter text.
2245  * @len - the length of the string.
2246  * @reset - non zero to reset all filters before applying this filter.
2247  *
2248  * Filters denote which functions should be enabled when tracing is enabled.
2249  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2250  */
2251 void ftrace_set_filter(unsigned char *buf, int len, int reset)
2252 {
2253         ftrace_set_regex(buf, len, reset, 1);
2254 }
2255
2256 /**
2257  * ftrace_set_notrace - set a function to not trace in ftrace
2258  * @buf - the string that holds the function notrace text.
2259  * @len - the length of the string.
2260  * @reset - non zero to reset all filters before applying this filter.
2261  *
2262  * Notrace Filters denote which functions should not be enabled when tracing
2263  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2264  * for tracing.
2265  */
2266 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
2267 {
2268         ftrace_set_regex(buf, len, reset, 0);
2269 }
2270
2271 /*
2272  * command line interface to allow users to set filters on boot up.
2273  */
2274 #define FTRACE_FILTER_SIZE              COMMAND_LINE_SIZE
2275 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
2276 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
2277 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
2278
2279 static int __init set_ftrace_notrace(char *str)
2280 {
2281         strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
2282         return 1;
2283 }
2284 __setup("ftrace_notrace=", set_ftrace_notrace);
2285
2286 static int __init set_ftrace_filter(char *str)
2287 {
2288         strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
2289         return 1;
2290 }
2291 __setup("ftrace_filter=", set_ftrace_filter);
2292
2293 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2294 static int __init set_graph_function(char *str)
2295 {
2296         strncpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
2297         return 1;
2298 }
2299 __setup("ftrace_graph_filter=", set_graph_function);
2300
2301 static void __init set_ftrace_early_graph(char *buf)
2302 {
2303         int ret;
2304         char *func;
2305
2306         while (buf) {
2307                 func = strsep(&buf, ",");
2308                 /* we allow only one expression at a time */
2309                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2310                                       func);
2311                 if (ret)
2312                         printk(KERN_DEBUG "ftrace: function %s not "
2313                                           "traceable\n", func);
2314         }
2315 }
2316 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2317
2318 static void __init set_ftrace_early_filter(char *buf, int enable)
2319 {
2320         char *func;
2321
2322         while (buf) {
2323                 func = strsep(&buf, ",");
2324                 ftrace_set_regex(func, strlen(func), 0, enable);
2325         }
2326 }
2327
2328 static void __init set_ftrace_early_filters(void)
2329 {
2330         if (ftrace_filter_buf[0])
2331                 set_ftrace_early_filter(ftrace_filter_buf, 1);
2332         if (ftrace_notrace_buf[0])
2333                 set_ftrace_early_filter(ftrace_notrace_buf, 0);
2334 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2335         if (ftrace_graph_buf[0])
2336                 set_ftrace_early_graph(ftrace_graph_buf);
2337 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2338 }
2339
2340 static int
2341 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2342 {
2343         struct seq_file *m = (struct seq_file *)file->private_data;
2344         struct ftrace_iterator *iter;
2345         struct trace_parser *parser;
2346
2347         mutex_lock(&ftrace_regex_lock);
2348         if (file->f_mode & FMODE_READ) {
2349                 iter = m->private;
2350
2351                 seq_release(inode, file);
2352         } else
2353                 iter = file->private_data;
2354
2355         parser = &iter->parser;
2356         if (trace_parser_loaded(parser)) {
2357                 parser->buffer[parser->idx] = 0;
2358                 ftrace_match_records(parser->buffer, parser->idx, enable);
2359         }
2360
2361         mutex_lock(&ftrace_lock);
2362         if (ftrace_start_up && ftrace_enabled)
2363                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2364         mutex_unlock(&ftrace_lock);
2365
2366         trace_parser_put(parser);
2367         kfree(iter);
2368
2369         mutex_unlock(&ftrace_regex_lock);
2370         return 0;
2371 }
2372
2373 static int
2374 ftrace_filter_release(struct inode *inode, struct file *file)
2375 {
2376         return ftrace_regex_release(inode, file, 1);
2377 }
2378
2379 static int
2380 ftrace_notrace_release(struct inode *inode, struct file *file)
2381 {
2382         return ftrace_regex_release(inode, file, 0);
2383 }
2384
2385 static const struct file_operations ftrace_avail_fops = {
2386         .open = ftrace_avail_open,
2387         .read = seq_read,
2388         .llseek = seq_lseek,
2389         .release = seq_release_private,
2390 };
2391
2392 static const struct file_operations ftrace_failures_fops = {
2393         .open = ftrace_failures_open,
2394         .read = seq_read,
2395         .llseek = seq_lseek,
2396         .release = seq_release_private,
2397 };
2398
2399 static const struct file_operations ftrace_filter_fops = {
2400         .open = ftrace_filter_open,
2401         .read = seq_read,
2402         .write = ftrace_filter_write,
2403         .llseek = ftrace_regex_lseek,
2404         .release = ftrace_filter_release,
2405 };
2406
2407 static const struct file_operations ftrace_notrace_fops = {
2408         .open = ftrace_notrace_open,
2409         .read = seq_read,
2410         .write = ftrace_notrace_write,
2411         .llseek = ftrace_regex_lseek,
2412         .release = ftrace_notrace_release,
2413 };
2414
2415 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2416
2417 static DEFINE_MUTEX(graph_lock);
2418
2419 int ftrace_graph_count;
2420 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
2421
2422 static void *
2423 __g_next(struct seq_file *m, loff_t *pos)
2424 {
2425         if (*pos >= ftrace_graph_count)
2426                 return NULL;
2427         return &ftrace_graph_funcs[*pos];
2428 }
2429
2430 static void *
2431 g_next(struct seq_file *m, void *v, loff_t *pos)
2432 {
2433         (*pos)++;
2434         return __g_next(m, pos);
2435 }
2436
2437 static void *g_start(struct seq_file *m, loff_t *pos)
2438 {
2439         mutex_lock(&graph_lock);
2440
2441         /* Nothing, tell g_show to print all functions are enabled */
2442         if (!ftrace_graph_count && !*pos)
2443                 return (void *)1;
2444
2445         return __g_next(m, pos);
2446 }
2447
2448 static void g_stop(struct seq_file *m, void *p)
2449 {
2450         mutex_unlock(&graph_lock);
2451 }
2452
2453 static int g_show(struct seq_file *m, void *v)
2454 {
2455         unsigned long *ptr = v;
2456
2457         if (!ptr)
2458                 return 0;
2459
2460         if (ptr == (unsigned long *)1) {
2461                 seq_printf(m, "#### all functions enabled ####\n");
2462                 return 0;
2463         }
2464
2465         seq_printf(m, "%ps\n", (void *)*ptr);
2466
2467         return 0;
2468 }
2469
2470 static const struct seq_operations ftrace_graph_seq_ops = {
2471         .start = g_start,
2472         .next = g_next,
2473         .stop = g_stop,
2474         .show = g_show,
2475 };
2476
2477 static int
2478 ftrace_graph_open(struct inode *inode, struct file *file)
2479 {
2480         int ret = 0;
2481
2482         if (unlikely(ftrace_disabled))
2483                 return -ENODEV;
2484
2485         mutex_lock(&graph_lock);
2486         if ((file->f_mode & FMODE_WRITE) &&
2487             (file->f_flags & O_TRUNC)) {
2488                 ftrace_graph_count = 0;
2489                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2490         }
2491         mutex_unlock(&graph_lock);
2492
2493         if (file->f_mode & FMODE_READ)
2494                 ret = seq_open(file, &ftrace_graph_seq_ops);
2495
2496         return ret;
2497 }
2498
2499 static int
2500 ftrace_graph_release(struct inode *inode, struct file *file)
2501 {
2502         if (file->f_mode & FMODE_READ)
2503                 seq_release(inode, file);
2504         return 0;
2505 }
2506
2507 static int
2508 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2509 {
2510         struct dyn_ftrace *rec;
2511         struct ftrace_page *pg;
2512         int search_len;
2513         int found = 0;
2514         int type, not;
2515         char *search;
2516         bool exists;
2517         int i;
2518
2519         if (ftrace_disabled)
2520                 return -ENODEV;
2521
2522         /* decode regex */
2523         type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
2524         if (not)
2525                 return -EINVAL;
2526
2527         search_len = strlen(search);
2528
2529         mutex_lock(&ftrace_lock);
2530         do_for_each_ftrace_rec(pg, rec) {
2531
2532                 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2533                         break;
2534
2535                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2536                         continue;
2537
2538                 if (ftrace_match_record(rec, search, search_len, type)) {
2539                         /* ensure it is not already in the array */
2540                         exists = false;
2541                         for (i = 0; i < *idx; i++)
2542                                 if (array[i] == rec->ip) {
2543                                         exists = true;
2544                                         break;
2545                                 }
2546                         if (!exists) {
2547                                 array[(*idx)++] = rec->ip;
2548                                 found = 1;
2549                         }
2550                 }
2551         } while_for_each_ftrace_rec();
2552
2553         mutex_unlock(&ftrace_lock);
2554
2555         return found ? 0 : -EINVAL;
2556 }
2557
2558 static ssize_t
2559 ftrace_graph_write(struct file *file, const char __user *ubuf,
2560                    size_t cnt, loff_t *ppos)
2561 {
2562         struct trace_parser parser;
2563         ssize_t read, ret;
2564
2565         if (!cnt || cnt < 0)
2566                 return 0;
2567
2568         mutex_lock(&graph_lock);
2569
2570         if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2571                 ret = -EBUSY;
2572                 goto out_unlock;
2573         }
2574
2575         if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
2576                 ret = -ENOMEM;
2577                 goto out_unlock;
2578         }
2579
2580         read = trace_get_user(&parser, ubuf, cnt, ppos);
2581
2582         if (read >= 0 && trace_parser_loaded((&parser))) {
2583                 parser.buffer[parser.idx] = 0;
2584
2585                 /* we allow only one expression at a time */
2586                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2587                                         parser.buffer);
2588                 if (ret)
2589                         goto out_free;
2590         }
2591
2592         ret = read;
2593
2594 out_free:
2595         trace_parser_put(&parser);
2596 out_unlock:
2597         mutex_unlock(&graph_lock);
2598
2599         return ret;
2600 }
2601
2602 static const struct file_operations ftrace_graph_fops = {
2603         .open           = ftrace_graph_open,
2604         .read           = seq_read,
2605         .write          = ftrace_graph_write,
2606         .release        = ftrace_graph_release,
2607 };
2608 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2609
2610 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2611 {
2612
2613         trace_create_file("available_filter_functions", 0444,
2614                         d_tracer, NULL, &ftrace_avail_fops);
2615
2616         trace_create_file("failures", 0444,
2617                         d_tracer, NULL, &ftrace_failures_fops);
2618
2619         trace_create_file("set_ftrace_filter", 0644, d_tracer,
2620                         NULL, &ftrace_filter_fops);
2621
2622         trace_create_file("set_ftrace_notrace", 0644, d_tracer,
2623                                     NULL, &ftrace_notrace_fops);
2624
2625 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2626         trace_create_file("set_graph_function", 0444, d_tracer,
2627                                     NULL,
2628                                     &ftrace_graph_fops);
2629 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2630
2631         return 0;
2632 }
2633
2634 static int ftrace_process_locs(struct module *mod,
2635                                unsigned long *start,
2636                                unsigned long *end)
2637 {
2638         unsigned long *p;
2639         unsigned long addr;
2640         unsigned long flags;
2641
2642         mutex_lock(&ftrace_lock);
2643         p = start;
2644         while (p < end) {
2645                 addr = ftrace_call_adjust(*p++);
2646                 /*
2647                  * Some architecture linkers will pad between
2648                  * the different mcount_loc sections of different
2649                  * object files to satisfy alignments.
2650                  * Skip any NULL pointers.
2651                  */
2652                 if (!addr)
2653                         continue;
2654                 ftrace_record_ip(addr);
2655         }
2656
2657         /* disable interrupts to prevent kstop machine */
2658         local_irq_save(flags);
2659         ftrace_update_code(mod);
2660         local_irq_restore(flags);
2661         mutex_unlock(&ftrace_lock);
2662
2663         return 0;
2664 }
2665
2666 #ifdef CONFIG_MODULES
2667 void ftrace_release_mod(struct module *mod)
2668 {
2669         struct dyn_ftrace *rec;
2670         struct ftrace_page *pg;
2671
2672         if (ftrace_disabled)
2673                 return;
2674
2675         mutex_lock(&ftrace_lock);
2676         do_for_each_ftrace_rec(pg, rec) {
2677                 if (within_module_core(rec->ip, mod)) {
2678                         /*
2679                          * rec->ip is changed in ftrace_free_rec()
2680                          * It should not between s and e if record was freed.
2681                          */
2682                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
2683                         ftrace_free_rec(rec);
2684                 }
2685         } while_for_each_ftrace_rec();
2686         mutex_unlock(&ftrace_lock);
2687 }
2688
2689 static void ftrace_init_module(struct module *mod,
2690                                unsigned long *start, unsigned long *end)
2691 {
2692         if (ftrace_disabled || start == end)
2693                 return;
2694         ftrace_process_locs(mod, start, end);
2695 }
2696
2697 static int ftrace_module_notify(struct notifier_block *self,
2698                                 unsigned long val, void *data)
2699 {
2700         struct module *mod = data;
2701
2702         switch (val) {
2703         case MODULE_STATE_COMING:
2704                 ftrace_init_module(mod, mod->ftrace_callsites,
2705                                    mod->ftrace_callsites +
2706                                    mod->num_ftrace_callsites);
2707                 break;
2708         case MODULE_STATE_GOING:
2709                 ftrace_release_mod(mod);
2710                 break;
2711         }
2712
2713         return 0;
2714 }
2715 #else
2716 static int ftrace_module_notify(struct notifier_block *self,
2717                                 unsigned long val, void *data)
2718 {
2719         return 0;
2720 }
2721 #endif /* CONFIG_MODULES */
2722
2723 struct notifier_block ftrace_module_nb = {
2724         .notifier_call = ftrace_module_notify,
2725         .priority = 0,
2726 };
2727
2728 extern unsigned long __start_mcount_loc[];
2729 extern unsigned long __stop_mcount_loc[];
2730
2731 void __init ftrace_init(void)
2732 {
2733         unsigned long count, addr, flags;
2734         int ret;
2735
2736         /* Keep the ftrace pointer to the stub */
2737         addr = (unsigned long)ftrace_stub;
2738
2739         local_irq_save(flags);
2740         ftrace_dyn_arch_init(&addr);
2741         local_irq_restore(flags);
2742
2743         /* ftrace_dyn_arch_init places the return code in addr */
2744         if (addr)
2745                 goto failed;
2746
2747         count = __stop_mcount_loc - __start_mcount_loc;
2748
2749         ret = ftrace_dyn_table_alloc(count);
2750         if (ret)
2751                 goto failed;
2752
2753         last_ftrace_enabled = ftrace_enabled = 1;
2754
2755         ret = ftrace_process_locs(NULL,
2756                                   __start_mcount_loc,
2757                                   __stop_mcount_loc);
2758
2759         ret = register_module_notifier(&ftrace_module_nb);
2760         if (ret)
2761                 pr_warning("Failed to register trace ftrace module notifier\n");
2762
2763         set_ftrace_early_filters();
2764
2765         return;
2766  failed:
2767         ftrace_disabled = 1;
2768 }
2769
2770 #else
2771
2772 static int __init ftrace_nodyn_init(void)
2773 {
2774         ftrace_enabled = 1;
2775         return 0;
2776 }
2777 device_initcall(ftrace_nodyn_init);
2778
2779 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2780 static inline void ftrace_startup_enable(int command) { }
2781 /* Keep as macros so we do not need to define the commands */
2782 # define ftrace_startup(command)        do { } while (0)
2783 # define ftrace_shutdown(command)       do { } while (0)
2784 # define ftrace_startup_sysctl()        do { } while (0)
2785 # define ftrace_shutdown_sysctl()       do { } while (0)
2786 #endif /* CONFIG_DYNAMIC_FTRACE */
2787
2788 static void clear_ftrace_swapper(void)
2789 {
2790         struct task_struct *p;
2791         int cpu;
2792
2793         get_online_cpus();
2794         for_each_online_cpu(cpu) {
2795                 p = idle_task(cpu);
2796                 clear_tsk_trace_trace(p);
2797         }
2798         put_online_cpus();
2799 }
2800
2801 static void set_ftrace_swapper(void)
2802 {
2803         struct task_struct *p;
2804         int cpu;
2805
2806         get_online_cpus();
2807         for_each_online_cpu(cpu) {
2808                 p = idle_task(cpu);
2809                 set_tsk_trace_trace(p);
2810         }
2811         put_online_cpus();
2812 }
2813
2814 static void clear_ftrace_pid(struct pid *pid)
2815 {
2816         struct task_struct *p;
2817
2818         rcu_read_lock();
2819         do_each_pid_task(pid, PIDTYPE_PID, p) {
2820                 clear_tsk_trace_trace(p);
2821         } while_each_pid_task(pid, PIDTYPE_PID, p);
2822         rcu_read_unlock();
2823
2824         put_pid(pid);
2825 }
2826
2827 static void set_ftrace_pid(struct pid *pid)
2828 {
2829         struct task_struct *p;
2830
2831         rcu_read_lock();
2832         do_each_pid_task(pid, PIDTYPE_PID, p) {
2833                 set_tsk_trace_trace(p);
2834         } while_each_pid_task(pid, PIDTYPE_PID, p);
2835         rcu_read_unlock();
2836 }
2837
2838 static void clear_ftrace_pid_task(struct pid *pid)
2839 {
2840         if (pid == ftrace_swapper_pid)
2841                 clear_ftrace_swapper();
2842         else
2843                 clear_ftrace_pid(pid);
2844 }
2845
2846 static void set_ftrace_pid_task(struct pid *pid)
2847 {
2848         if (pid == ftrace_swapper_pid)
2849                 set_ftrace_swapper();
2850         else
2851                 set_ftrace_pid(pid);
2852 }
2853
2854 static int ftrace_pid_add(int p)
2855 {
2856         struct pid *pid;
2857         struct ftrace_pid *fpid;
2858         int ret = -EINVAL;
2859
2860         mutex_lock(&ftrace_lock);
2861
2862         if (!p)
2863                 pid = ftrace_swapper_pid;
2864         else
2865                 pid = find_get_pid(p);
2866
2867         if (!pid)
2868                 goto out;
2869
2870         ret = 0;
2871
2872         list_for_each_entry(fpid, &ftrace_pids, list)
2873                 if (fpid->pid == pid)
2874                         goto out_put;
2875
2876         ret = -ENOMEM;
2877
2878         fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
2879         if (!fpid)
2880                 goto out_put;
2881
2882         list_add(&fpid->list, &ftrace_pids);
2883         fpid->pid = pid;
2884
2885         set_ftrace_pid_task(pid);
2886
2887         ftrace_update_pid_func();
2888         ftrace_startup_enable(0);
2889
2890         mutex_unlock(&ftrace_lock);
2891         return 0;
2892
2893 out_put:
2894         if (pid != ftrace_swapper_pid)
2895                 put_pid(pid);
2896
2897 out:
2898         mutex_unlock(&ftrace_lock);
2899         return ret;
2900 }
2901
2902 static void ftrace_pid_reset(void)
2903 {
2904         struct ftrace_pid *fpid, *safe;
2905
2906         mutex_lock(&ftrace_lock);
2907         list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
2908                 struct pid *pid = fpid->pid;
2909
2910                 clear_ftrace_pid_task(pid);
2911
2912                 list_del(&fpid->list);
2913                 kfree(fpid);
2914         }
2915
2916         ftrace_update_pid_func();
2917         ftrace_startup_enable(0);
2918
2919         mutex_unlock(&ftrace_lock);
2920 }
2921
2922 static void *fpid_start(struct seq_file *m, loff_t *pos)
2923 {
2924         mutex_lock(&ftrace_lock);
2925
2926         if (list_empty(&ftrace_pids) && (!*pos))
2927                 return (void *) 1;
2928
2929         return seq_list_start(&ftrace_pids, *pos);
2930 }
2931
2932 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
2933 {
2934         if (v == (void *)1)
2935                 return NULL;
2936
2937         return seq_list_next(v, &ftrace_pids, pos);
2938 }
2939
2940 static void fpid_stop(struct seq_file *m, void *p)
2941 {
2942         mutex_unlock(&ftrace_lock);
2943 }
2944
2945 static int fpid_show(struct seq_file *m, void *v)
2946 {
2947         const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
2948
2949         if (v == (void *)1) {
2950                 seq_printf(m, "no pid\n");
2951                 return 0;
2952         }
2953
2954         if (fpid->pid == ftrace_swapper_pid)
2955                 seq_printf(m, "swapper tasks\n");
2956         else
2957                 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
2958
2959         return 0;
2960 }
2961
2962 static const struct seq_operations ftrace_pid_sops = {
2963         .start = fpid_start,
2964         .next = fpid_next,
2965         .stop = fpid_stop,
2966         .show = fpid_show,
2967 };
2968
2969 static int
2970 ftrace_pid_open(struct inode *inode, struct file *file)
2971 {
2972         int ret = 0;
2973
2974         if ((file->f_mode & FMODE_WRITE) &&
2975             (file->f_flags & O_TRUNC))
2976                 ftrace_pid_reset();
2977
2978         if (file->f_mode & FMODE_READ)
2979                 ret = seq_open(file, &ftrace_pid_sops);
2980
2981         return ret;
2982 }
2983
2984 static ssize_t
2985 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2986                    size_t cnt, loff_t *ppos)
2987 {
2988         char buf[64];
2989         long val;
2990         int ret;
2991
2992         if (cnt >= sizeof(buf))
2993                 return -EINVAL;
2994
2995         if (copy_from_user(&buf, ubuf, cnt))
2996                 return -EFAULT;
2997
2998         buf[cnt] = 0;
2999
3000         /*
3001          * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
3002          * to clean the filter quietly.
3003          */
3004         strstrip(buf);
3005         if (strlen(buf) == 0)
3006                 return 1;
3007
3008         ret = strict_strtol(buf, 10, &val);
3009         if (ret < 0)
3010                 return ret;
3011
3012         ret = ftrace_pid_add(val);
3013
3014         return ret ? ret : cnt;
3015 }
3016
3017 static int
3018 ftrace_pid_release(struct inode *inode, struct file *file)
3019 {
3020         if (file->f_mode & FMODE_READ)
3021                 seq_release(inode, file);
3022
3023         return 0;
3024 }
3025
3026 static const struct file_operations ftrace_pid_fops = {
3027         .open           = ftrace_pid_open,
3028         .write          = ftrace_pid_write,
3029         .read           = seq_read,
3030         .llseek         = seq_lseek,
3031         .release        = ftrace_pid_release,
3032 };
3033
3034 static __init int ftrace_init_debugfs(void)
3035 {
3036         struct dentry *d_tracer;
3037
3038         d_tracer = tracing_init_dentry();
3039         if (!d_tracer)
3040                 return 0;
3041
3042         ftrace_init_dyn_debugfs(d_tracer);
3043
3044         trace_create_file("set_ftrace_pid", 0644, d_tracer,
3045                             NULL, &ftrace_pid_fops);
3046
3047         ftrace_profile_debugfs(d_tracer);
3048
3049         return 0;
3050 }
3051 fs_initcall(ftrace_init_debugfs);
3052
3053 /**
3054  * ftrace_kill - kill ftrace
3055  *
3056  * This function should be used by panic code. It stops ftrace
3057  * but in a not so nice way. If you need to simply kill ftrace
3058  * from a non-atomic section, use ftrace_kill.
3059  */
3060 void ftrace_kill(void)
3061 {
3062         ftrace_disabled = 1;
3063         ftrace_enabled = 0;
3064         clear_ftrace_function();
3065 }
3066
3067 /**
3068  * register_ftrace_function - register a function for profiling
3069  * @ops - ops structure that holds the function for profiling.
3070  *
3071  * Register a function to be called by all functions in the
3072  * kernel.
3073  *
3074  * Note: @ops->func and all the functions it calls must be labeled
3075  *       with "notrace", otherwise it will go into a
3076  *       recursive loop.
3077  */
3078 int register_ftrace_function(struct ftrace_ops *ops)
3079 {
3080         int ret;
3081
3082         if (unlikely(ftrace_disabled))
3083                 return -1;
3084
3085         mutex_lock(&ftrace_lock);
3086
3087         ret = __register_ftrace_function(ops);
3088         ftrace_startup(0);
3089
3090         mutex_unlock(&ftrace_lock);
3091         return ret;
3092 }
3093
3094 /**
3095  * unregister_ftrace_function - unregister a function for profiling.
3096  * @ops - ops structure that holds the function to unregister
3097  *
3098  * Unregister a function that was added to be called by ftrace profiling.
3099  */
3100 int unregister_ftrace_function(struct ftrace_ops *ops)
3101 {
3102         int ret;
3103
3104         mutex_lock(&ftrace_lock);
3105         ret = __unregister_ftrace_function(ops);
3106         ftrace_shutdown(0);
3107         mutex_unlock(&ftrace_lock);
3108
3109         return ret;
3110 }
3111
3112 int
3113 ftrace_enable_sysctl(struct ctl_table *table, int write,
3114                      void __user *buffer, size_t *lenp,
3115                      loff_t *ppos)
3116 {
3117         int ret;
3118
3119         if (unlikely(ftrace_disabled))
3120                 return -ENODEV;
3121
3122         mutex_lock(&ftrace_lock);
3123
3124         ret  = proc_dointvec(table, write, buffer, lenp, ppos);
3125
3126         if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3127                 goto out;
3128
3129         last_ftrace_enabled = !!ftrace_enabled;
3130
3131         if (ftrace_enabled) {
3132
3133                 ftrace_startup_sysctl();
3134
3135                 /* we are starting ftrace again */
3136                 if (ftrace_list != &ftrace_list_end) {
3137                         if (ftrace_list->next == &ftrace_list_end)
3138                                 ftrace_trace_function = ftrace_list->func;
3139                         else
3140                                 ftrace_trace_function = ftrace_list_func;
3141                 }
3142
3143         } else {
3144                 /* stopping ftrace calls (just send to ftrace_stub) */
3145                 ftrace_trace_function = ftrace_stub;
3146
3147                 ftrace_shutdown_sysctl();
3148         }
3149
3150  out:
3151         mutex_unlock(&ftrace_lock);
3152         return ret;
3153 }
3154
3155 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3156
3157 static int ftrace_graph_active;
3158 static struct notifier_block ftrace_suspend_notifier;
3159
3160 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3161 {
3162         return 0;
3163 }
3164
3165 /* The callbacks that hook a function */
3166 trace_func_graph_ret_t ftrace_graph_return =
3167                         (trace_func_graph_ret_t)ftrace_stub;
3168 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3169
3170 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3171 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3172 {
3173         int i;
3174         int ret = 0;
3175         unsigned long flags;
3176         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3177         struct task_struct *g, *t;
3178
3179         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3180                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3181                                         * sizeof(struct ftrace_ret_stack),
3182                                         GFP_KERNEL);
3183                 if (!ret_stack_list[i]) {
3184                         start = 0;
3185                         end = i;
3186                         ret = -ENOMEM;
3187                         goto free;
3188                 }
3189         }
3190
3191         read_lock_irqsave(&tasklist_lock, flags);
3192         do_each_thread(g, t) {
3193                 if (start == end) {
3194                         ret = -EAGAIN;
3195                         goto unlock;
3196                 }
3197
3198                 if (t->ret_stack == NULL) {
3199                         atomic_set(&t->tracing_graph_pause, 0);
3200                         atomic_set(&t->trace_overrun, 0);
3201                         t->curr_ret_stack = -1;
3202                         /* Make sure the tasks see the -1 first: */
3203                         smp_wmb();
3204                         t->ret_stack = ret_stack_list[start++];
3205                 }
3206         } while_each_thread(g, t);
3207
3208 unlock:
3209         read_unlock_irqrestore(&tasklist_lock, flags);
3210 free:
3211         for (i = start; i < end; i++)
3212                 kfree(ret_stack_list[i]);
3213         return ret;
3214 }
3215
3216 static void
3217 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
3218                                 struct task_struct *next)
3219 {
3220         unsigned long long timestamp;
3221         int index;
3222
3223         /*
3224          * Does the user want to count the time a function was asleep.
3225          * If so, do not update the time stamps.
3226          */
3227         if (trace_flags & TRACE_ITER_SLEEP_TIME)
3228                 return;
3229
3230         timestamp = trace_clock_local();
3231
3232         prev->ftrace_timestamp = timestamp;
3233
3234         /* only process tasks that we timestamped */
3235         if (!next->ftrace_timestamp)
3236                 return;
3237
3238         /*
3239          * Update all the counters in next to make up for the
3240          * time next was sleeping.
3241          */
3242         timestamp -= next->ftrace_timestamp;
3243
3244         for (index = next->curr_ret_stack; index >= 0; index--)
3245                 next->ret_stack[index].calltime += timestamp;
3246 }
3247
3248 /* Allocate a return stack for each task */
3249 static int start_graph_tracing(void)
3250 {
3251         struct ftrace_ret_stack **ret_stack_list;
3252         int ret, cpu;
3253
3254         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
3255                                 sizeof(struct ftrace_ret_stack *),
3256                                 GFP_KERNEL);
3257
3258         if (!ret_stack_list)
3259                 return -ENOMEM;
3260
3261         /* The cpu_boot init_task->ret_stack will never be freed */
3262         for_each_online_cpu(cpu) {
3263                 if (!idle_task(cpu)->ret_stack)
3264                         ftrace_graph_init_task(idle_task(cpu));
3265         }
3266
3267         do {
3268                 ret = alloc_retstack_tasklist(ret_stack_list);
3269         } while (ret == -EAGAIN);
3270
3271         if (!ret) {
3272                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
3273                 if (ret)
3274                         pr_info("ftrace_graph: Couldn't activate tracepoint"
3275                                 " probe to kernel_sched_switch\n");
3276         }
3277
3278         kfree(ret_stack_list);
3279         return ret;
3280 }
3281
3282 /*
3283  * Hibernation protection.
3284  * The state of the current task is too much unstable during
3285  * suspend/restore to disk. We want to protect against that.
3286  */
3287 static int
3288 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
3289                                                         void *unused)
3290 {
3291         switch (state) {
3292         case PM_HIBERNATION_PREPARE:
3293                 pause_graph_tracing();
3294                 break;
3295
3296         case PM_POST_HIBERNATION:
3297                 unpause_graph_tracing();
3298                 break;
3299         }
3300         return NOTIFY_DONE;
3301 }
3302
3303 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3304                         trace_func_graph_ent_t entryfunc)
3305 {
3306         int ret = 0;
3307
3308         mutex_lock(&ftrace_lock);
3309
3310         /* we currently allow only one tracer registered at a time */
3311         if (ftrace_graph_active) {
3312                 ret = -EBUSY;
3313                 goto out;
3314         }
3315
3316         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
3317         register_pm_notifier(&ftrace_suspend_notifier);
3318
3319         ftrace_graph_active++;
3320         ret = start_graph_tracing();
3321         if (ret) {
3322                 ftrace_graph_active--;
3323                 goto out;
3324         }
3325
3326         ftrace_graph_return = retfunc;
3327         ftrace_graph_entry = entryfunc;
3328
3329         ftrace_startup(FTRACE_START_FUNC_RET);
3330
3331 out:
3332         mutex_unlock(&ftrace_lock);
3333         return ret;
3334 }
3335
3336 void unregister_ftrace_graph(void)
3337 {
3338         mutex_lock(&ftrace_lock);
3339
3340         if (unlikely(!ftrace_graph_active))
3341                 goto out;
3342
3343         ftrace_graph_active--;
3344         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
3345         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3346         ftrace_graph_entry = ftrace_graph_entry_stub;
3347         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3348         unregister_pm_notifier(&ftrace_suspend_notifier);
3349
3350  out:
3351         mutex_unlock(&ftrace_lock);
3352 }
3353
3354 /* Allocate a return stack for newly created task */
3355 void ftrace_graph_init_task(struct task_struct *t)
3356 {
3357         /* Make sure we do not use the parent ret_stack */
3358         t->ret_stack = NULL;
3359
3360         if (ftrace_graph_active) {
3361                 struct ftrace_ret_stack *ret_stack;
3362
3363                 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3364                                 * sizeof(struct ftrace_ret_stack),
3365                                 GFP_KERNEL);
3366                 if (!ret_stack)
3367                         return;
3368                 t->curr_ret_stack = -1;
3369                 atomic_set(&t->tracing_graph_pause, 0);
3370                 atomic_set(&t->trace_overrun, 0);
3371                 t->ftrace_timestamp = 0;
3372                 /* make curr_ret_stack visable before we add the ret_stack */
3373                 smp_wmb();
3374                 t->ret_stack = ret_stack;
3375         }
3376 }
3377
3378 void ftrace_graph_exit_task(struct task_struct *t)
3379 {
3380         struct ftrace_ret_stack *ret_stack = t->ret_stack;
3381
3382         t->ret_stack = NULL;
3383         /* NULL must become visible to IRQs before we free it: */
3384         barrier();
3385
3386         kfree(ret_stack);
3387 }
3388
3389 void ftrace_graph_stop(void)
3390 {
3391         ftrace_stop();
3392 }
3393 #endif
3394