Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/percpu
[linux-2.6.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 William Lee Irwin III
13  */
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/smp_lock.h>
21 #include <linux/notifier.h>
22 #include <linux/irqflags.h>
23 #include <linux/debugfs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/rwsem.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/gfp.h>
40 #include <linux/fs.h>
41
42 #include "trace.h"
43 #include "trace_output.h"
44
45 #define TRACE_BUFFER_FLAGS      (RB_FL_OVERWRITE)
46
47 /*
48  * On boot up, the ring buffer is set to the minimum size, so that
49  * we do not waste memory on systems that are not using tracing.
50  */
51 int ring_buffer_expanded;
52
53 /*
54  * We need to change this state when a selftest is running.
55  * A selftest will lurk into the ring-buffer to count the
56  * entries inserted during the selftest although some concurrent
57  * insertions into the ring-buffer such as trace_printk could occurred
58  * at the same time, giving false positive or negative results.
59  */
60 static bool __read_mostly tracing_selftest_running;
61
62 /*
63  * If a tracer is running, we do not want to run SELFTEST.
64  */
65 bool __read_mostly tracing_selftest_disabled;
66
67 /* For tracers that don't implement custom flags */
68 static struct tracer_opt dummy_tracer_opt[] = {
69         { }
70 };
71
72 static struct tracer_flags dummy_tracer_flags = {
73         .val = 0,
74         .opts = dummy_tracer_opt
75 };
76
77 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
78 {
79         return 0;
80 }
81
82 /*
83  * Kill all tracing for good (never come back).
84  * It is initialized to 1 but will turn to zero if the initialization
85  * of the tracer is successful. But that is the only place that sets
86  * this back to zero.
87  */
88 static int tracing_disabled = 1;
89
90 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
91
92 static inline void ftrace_disable_cpu(void)
93 {
94         preempt_disable();
95         __this_cpu_inc(ftrace_cpu_disabled);
96 }
97
98 static inline void ftrace_enable_cpu(void)
99 {
100         __this_cpu_dec(ftrace_cpu_disabled);
101         preempt_enable();
102 }
103
104 static cpumask_var_t __read_mostly      tracing_buffer_mask;
105
106 #define for_each_tracing_cpu(cpu)       \
107         for_each_cpu(cpu, tracing_buffer_mask)
108
109 /*
110  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
111  *
112  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
113  * is set, then ftrace_dump is called. This will output the contents
114  * of the ftrace buffers to the console.  This is very useful for
115  * capturing traces that lead to crashes and outputing it to a
116  * serial console.
117  *
118  * It is default off, but you can enable it with either specifying
119  * "ftrace_dump_on_oops" in the kernel command line, or setting
120  * /proc/sys/kernel/ftrace_dump_on_oops to true.
121  */
122 int ftrace_dump_on_oops;
123
124 static int tracing_set_tracer(const char *buf);
125
126 #define MAX_TRACER_SIZE         100
127 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
128 static char *default_bootup_tracer;
129
130 static int __init set_cmdline_ftrace(char *str)
131 {
132         strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
133         default_bootup_tracer = bootup_tracer_buf;
134         /* We are using ftrace early, expand it */
135         ring_buffer_expanded = 1;
136         return 1;
137 }
138 __setup("ftrace=", set_cmdline_ftrace);
139
140 static int __init set_ftrace_dump_on_oops(char *str)
141 {
142         ftrace_dump_on_oops = 1;
143         return 1;
144 }
145 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
146
147 unsigned long long ns2usecs(cycle_t nsec)
148 {
149         nsec += 500;
150         do_div(nsec, 1000);
151         return nsec;
152 }
153
154 /*
155  * The global_trace is the descriptor that holds the tracing
156  * buffers for the live tracing. For each CPU, it contains
157  * a link list of pages that will store trace entries. The
158  * page descriptor of the pages in the memory is used to hold
159  * the link list by linking the lru item in the page descriptor
160  * to each of the pages in the buffer per CPU.
161  *
162  * For each active CPU there is a data field that holds the
163  * pages for the buffer for that CPU. Each CPU has the same number
164  * of pages allocated for its buffer.
165  */
166 static struct trace_array       global_trace;
167
168 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
169
170 int filter_current_check_discard(struct ring_buffer *buffer,
171                                  struct ftrace_event_call *call, void *rec,
172                                  struct ring_buffer_event *event)
173 {
174         return filter_check_discard(call, rec, buffer, event);
175 }
176 EXPORT_SYMBOL_GPL(filter_current_check_discard);
177
178 cycle_t ftrace_now(int cpu)
179 {
180         u64 ts;
181
182         /* Early boot up does not have a buffer yet */
183         if (!global_trace.buffer)
184                 return trace_clock_local();
185
186         ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
187         ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
188
189         return ts;
190 }
191
192 /*
193  * The max_tr is used to snapshot the global_trace when a maximum
194  * latency is reached. Some tracers will use this to store a maximum
195  * trace while it continues examining live traces.
196  *
197  * The buffers for the max_tr are set up the same as the global_trace.
198  * When a snapshot is taken, the link list of the max_tr is swapped
199  * with the link list of the global_trace and the buffers are reset for
200  * the global_trace so the tracing can continue.
201  */
202 static struct trace_array       max_tr;
203
204 static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
205
206 /* tracer_enabled is used to toggle activation of a tracer */
207 static int                      tracer_enabled = 1;
208
209 /**
210  * tracing_is_enabled - return tracer_enabled status
211  *
212  * This function is used by other tracers to know the status
213  * of the tracer_enabled flag.  Tracers may use this function
214  * to know if it should enable their features when starting
215  * up. See irqsoff tracer for an example (start_irqsoff_tracer).
216  */
217 int tracing_is_enabled(void)
218 {
219         return tracer_enabled;
220 }
221
222 /*
223  * trace_buf_size is the size in bytes that is allocated
224  * for a buffer. Note, the number of bytes is always rounded
225  * to page size.
226  *
227  * This number is purposely set to a low number of 16384.
228  * If the dump on oops happens, it will be much appreciated
229  * to not have to wait for all that output. Anyway this can be
230  * boot time and run time configurable.
231  */
232 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
233
234 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
235
236 /* trace_types holds a link list of available tracers. */
237 static struct tracer            *trace_types __read_mostly;
238
239 /* current_trace points to the tracer that is currently active */
240 static struct tracer            *current_trace __read_mostly;
241
242 /*
243  * trace_types_lock is used to protect the trace_types list.
244  */
245 static DEFINE_MUTEX(trace_types_lock);
246
247 /*
248  * serialize the access of the ring buffer
249  *
250  * ring buffer serializes readers, but it is low level protection.
251  * The validity of the events (which returns by ring_buffer_peek() ..etc)
252  * are not protected by ring buffer.
253  *
254  * The content of events may become garbage if we allow other process consumes
255  * these events concurrently:
256  *   A) the page of the consumed events may become a normal page
257  *      (not reader page) in ring buffer, and this page will be rewrited
258  *      by events producer.
259  *   B) The page of the consumed events may become a page for splice_read,
260  *      and this page will be returned to system.
261  *
262  * These primitives allow multi process access to different cpu ring buffer
263  * concurrently.
264  *
265  * These primitives don't distinguish read-only and read-consume access.
266  * Multi read-only access are also serialized.
267  */
268
269 #ifdef CONFIG_SMP
270 static DECLARE_RWSEM(all_cpu_access_lock);
271 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
272
273 static inline void trace_access_lock(int cpu)
274 {
275         if (cpu == TRACE_PIPE_ALL_CPU) {
276                 /* gain it for accessing the whole ring buffer. */
277                 down_write(&all_cpu_access_lock);
278         } else {
279                 /* gain it for accessing a cpu ring buffer. */
280
281                 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
282                 down_read(&all_cpu_access_lock);
283
284                 /* Secondly block other access to this @cpu ring buffer. */
285                 mutex_lock(&per_cpu(cpu_access_lock, cpu));
286         }
287 }
288
289 static inline void trace_access_unlock(int cpu)
290 {
291         if (cpu == TRACE_PIPE_ALL_CPU) {
292                 up_write(&all_cpu_access_lock);
293         } else {
294                 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
295                 up_read(&all_cpu_access_lock);
296         }
297 }
298
299 static inline void trace_access_lock_init(void)
300 {
301         int cpu;
302
303         for_each_possible_cpu(cpu)
304                 mutex_init(&per_cpu(cpu_access_lock, cpu));
305 }
306
307 #else
308
309 static DEFINE_MUTEX(access_lock);
310
311 static inline void trace_access_lock(int cpu)
312 {
313         (void)cpu;
314         mutex_lock(&access_lock);
315 }
316
317 static inline void trace_access_unlock(int cpu)
318 {
319         (void)cpu;
320         mutex_unlock(&access_lock);
321 }
322
323 static inline void trace_access_lock_init(void)
324 {
325 }
326
327 #endif
328
329 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
330 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
331
332 /* trace_flags holds trace_options default values */
333 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
334         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
335         TRACE_ITER_GRAPH_TIME;
336
337 static int trace_stop_count;
338 static DEFINE_SPINLOCK(tracing_start_lock);
339
340 /**
341  * trace_wake_up - wake up tasks waiting for trace input
342  *
343  * Simply wakes up any task that is blocked on the trace_wait
344  * queue. These is used with trace_poll for tasks polling the trace.
345  */
346 void trace_wake_up(void)
347 {
348         int cpu;
349
350         if (trace_flags & TRACE_ITER_BLOCK)
351                 return;
352         /*
353          * The runqueue_is_locked() can fail, but this is the best we
354          * have for now:
355          */
356         cpu = get_cpu();
357         if (!runqueue_is_locked(cpu))
358                 wake_up(&trace_wait);
359         put_cpu();
360 }
361
362 static int __init set_buf_size(char *str)
363 {
364         unsigned long buf_size;
365
366         if (!str)
367                 return 0;
368         buf_size = memparse(str, &str);
369         /* nr_entries can not be zero */
370         if (buf_size == 0)
371                 return 0;
372         trace_buf_size = buf_size;
373         return 1;
374 }
375 __setup("trace_buf_size=", set_buf_size);
376
377 unsigned long nsecs_to_usecs(unsigned long nsecs)
378 {
379         return nsecs / 1000;
380 }
381
382 /* These must match the bit postions in trace_iterator_flags */
383 static const char *trace_options[] = {
384         "print-parent",
385         "sym-offset",
386         "sym-addr",
387         "verbose",
388         "raw",
389         "hex",
390         "bin",
391         "block",
392         "stacktrace",
393         "trace_printk",
394         "ftrace_preempt",
395         "branch",
396         "annotate",
397         "userstacktrace",
398         "sym-userobj",
399         "printk-msg-only",
400         "context-info",
401         "latency-format",
402         "sleep-time",
403         "graph-time",
404         NULL
405 };
406
407 static struct {
408         u64 (*func)(void);
409         const char *name;
410 } trace_clocks[] = {
411         { trace_clock_local,    "local" },
412         { trace_clock_global,   "global" },
413 };
414
415 int trace_clock_id;
416
417 /*
418  * trace_parser_get_init - gets the buffer for trace parser
419  */
420 int trace_parser_get_init(struct trace_parser *parser, int size)
421 {
422         memset(parser, 0, sizeof(*parser));
423
424         parser->buffer = kmalloc(size, GFP_KERNEL);
425         if (!parser->buffer)
426                 return 1;
427
428         parser->size = size;
429         return 0;
430 }
431
432 /*
433  * trace_parser_put - frees the buffer for trace parser
434  */
435 void trace_parser_put(struct trace_parser *parser)
436 {
437         kfree(parser->buffer);
438 }
439
440 /*
441  * trace_get_user - reads the user input string separated by  space
442  * (matched by isspace(ch))
443  *
444  * For each string found the 'struct trace_parser' is updated,
445  * and the function returns.
446  *
447  * Returns number of bytes read.
448  *
449  * See kernel/trace/trace.h for 'struct trace_parser' details.
450  */
451 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
452         size_t cnt, loff_t *ppos)
453 {
454         char ch;
455         size_t read = 0;
456         ssize_t ret;
457
458         if (!*ppos)
459                 trace_parser_clear(parser);
460
461         ret = get_user(ch, ubuf++);
462         if (ret)
463                 goto out;
464
465         read++;
466         cnt--;
467
468         /*
469          * The parser is not finished with the last write,
470          * continue reading the user input without skipping spaces.
471          */
472         if (!parser->cont) {
473                 /* skip white space */
474                 while (cnt && isspace(ch)) {
475                         ret = get_user(ch, ubuf++);
476                         if (ret)
477                                 goto out;
478                         read++;
479                         cnt--;
480                 }
481
482                 /* only spaces were written */
483                 if (isspace(ch)) {
484                         *ppos += read;
485                         ret = read;
486                         goto out;
487                 }
488
489                 parser->idx = 0;
490         }
491
492         /* read the non-space input */
493         while (cnt && !isspace(ch)) {
494                 if (parser->idx < parser->size - 1)
495                         parser->buffer[parser->idx++] = ch;
496                 else {
497                         ret = -EINVAL;
498                         goto out;
499                 }
500                 ret = get_user(ch, ubuf++);
501                 if (ret)
502                         goto out;
503                 read++;
504                 cnt--;
505         }
506
507         /* We either got finished input or we have to wait for another call. */
508         if (isspace(ch)) {
509                 parser->buffer[parser->idx] = 0;
510                 parser->cont = false;
511         } else {
512                 parser->cont = true;
513                 parser->buffer[parser->idx++] = ch;
514         }
515
516         *ppos += read;
517         ret = read;
518
519 out:
520         return ret;
521 }
522
523 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
524 {
525         int len;
526         int ret;
527
528         if (!cnt)
529                 return 0;
530
531         if (s->len <= s->readpos)
532                 return -EBUSY;
533
534         len = s->len - s->readpos;
535         if (cnt > len)
536                 cnt = len;
537         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
538         if (ret == cnt)
539                 return -EFAULT;
540
541         cnt -= ret;
542
543         s->readpos += cnt;
544         return cnt;
545 }
546
547 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
548 {
549         int len;
550         void *ret;
551
552         if (s->len <= s->readpos)
553                 return -EBUSY;
554
555         len = s->len - s->readpos;
556         if (cnt > len)
557                 cnt = len;
558         ret = memcpy(buf, s->buffer + s->readpos, cnt);
559         if (!ret)
560                 return -EFAULT;
561
562         s->readpos += cnt;
563         return cnt;
564 }
565
566 /*
567  * ftrace_max_lock is used to protect the swapping of buffers
568  * when taking a max snapshot. The buffers themselves are
569  * protected by per_cpu spinlocks. But the action of the swap
570  * needs its own lock.
571  *
572  * This is defined as a arch_spinlock_t in order to help
573  * with performance when lockdep debugging is enabled.
574  *
575  * It is also used in other places outside the update_max_tr
576  * so it needs to be defined outside of the
577  * CONFIG_TRACER_MAX_TRACE.
578  */
579 static arch_spinlock_t ftrace_max_lock =
580         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
581
582 #ifdef CONFIG_TRACER_MAX_TRACE
583 unsigned long __read_mostly     tracing_max_latency;
584 unsigned long __read_mostly     tracing_thresh;
585
586 /*
587  * Copy the new maximum trace into the separate maximum-trace
588  * structure. (this way the maximum trace is permanently saved,
589  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
590  */
591 static void
592 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
593 {
594         struct trace_array_cpu *data = tr->data[cpu];
595         struct trace_array_cpu *max_data = tr->data[cpu];
596
597         max_tr.cpu = cpu;
598         max_tr.time_start = data->preempt_timestamp;
599
600         max_data = max_tr.data[cpu];
601         max_data->saved_latency = tracing_max_latency;
602         max_data->critical_start = data->critical_start;
603         max_data->critical_end = data->critical_end;
604
605         memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
606         max_data->pid = tsk->pid;
607         max_data->uid = task_uid(tsk);
608         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
609         max_data->policy = tsk->policy;
610         max_data->rt_priority = tsk->rt_priority;
611
612         /* record this tasks comm */
613         tracing_record_cmdline(tsk);
614 }
615
616 /**
617  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
618  * @tr: tracer
619  * @tsk: the task with the latency
620  * @cpu: The cpu that initiated the trace.
621  *
622  * Flip the buffers between the @tr and the max_tr and record information
623  * about which task was the cause of this latency.
624  */
625 void
626 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
627 {
628         struct ring_buffer *buf = tr->buffer;
629
630         if (trace_stop_count)
631                 return;
632
633         WARN_ON_ONCE(!irqs_disabled());
634         arch_spin_lock(&ftrace_max_lock);
635
636         tr->buffer = max_tr.buffer;
637         max_tr.buffer = buf;
638
639         __update_max_tr(tr, tsk, cpu);
640         arch_spin_unlock(&ftrace_max_lock);
641 }
642
643 /**
644  * update_max_tr_single - only copy one trace over, and reset the rest
645  * @tr - tracer
646  * @tsk - task with the latency
647  * @cpu - the cpu of the buffer to copy.
648  *
649  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
650  */
651 void
652 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
653 {
654         int ret;
655
656         if (trace_stop_count)
657                 return;
658
659         WARN_ON_ONCE(!irqs_disabled());
660         arch_spin_lock(&ftrace_max_lock);
661
662         ftrace_disable_cpu();
663
664         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
665
666         if (ret == -EBUSY) {
667                 /*
668                  * We failed to swap the buffer due to a commit taking
669                  * place on this CPU. We fail to record, but we reset
670                  * the max trace buffer (no one writes directly to it)
671                  * and flag that it failed.
672                  */
673                 trace_array_printk(&max_tr, _THIS_IP_,
674                         "Failed to swap buffers due to commit in progress\n");
675         }
676
677         ftrace_enable_cpu();
678
679         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
680
681         __update_max_tr(tr, tsk, cpu);
682         arch_spin_unlock(&ftrace_max_lock);
683 }
684 #endif /* CONFIG_TRACER_MAX_TRACE */
685
686 /**
687  * register_tracer - register a tracer with the ftrace system.
688  * @type - the plugin for the tracer
689  *
690  * Register a new plugin tracer.
691  */
692 int register_tracer(struct tracer *type)
693 __releases(kernel_lock)
694 __acquires(kernel_lock)
695 {
696         struct tracer *t;
697         int ret = 0;
698
699         if (!type->name) {
700                 pr_info("Tracer must have a name\n");
701                 return -1;
702         }
703
704         if (strlen(type->name) > MAX_TRACER_SIZE) {
705                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
706                 return -1;
707         }
708
709         /*
710          * When this gets called we hold the BKL which means that
711          * preemption is disabled. Various trace selftests however
712          * need to disable and enable preemption for successful tests.
713          * So we drop the BKL here and grab it after the tests again.
714          */
715         unlock_kernel();
716         mutex_lock(&trace_types_lock);
717
718         tracing_selftest_running = true;
719
720         for (t = trace_types; t; t = t->next) {
721                 if (strcmp(type->name, t->name) == 0) {
722                         /* already found */
723                         pr_info("Tracer %s already registered\n",
724                                 type->name);
725                         ret = -1;
726                         goto out;
727                 }
728         }
729
730         if (!type->set_flag)
731                 type->set_flag = &dummy_set_flag;
732         if (!type->flags)
733                 type->flags = &dummy_tracer_flags;
734         else
735                 if (!type->flags->opts)
736                         type->flags->opts = dummy_tracer_opt;
737         if (!type->wait_pipe)
738                 type->wait_pipe = default_wait_pipe;
739
740
741 #ifdef CONFIG_FTRACE_STARTUP_TEST
742         if (type->selftest && !tracing_selftest_disabled) {
743                 struct tracer *saved_tracer = current_trace;
744                 struct trace_array *tr = &global_trace;
745
746                 /*
747                  * Run a selftest on this tracer.
748                  * Here we reset the trace buffer, and set the current
749                  * tracer to be this tracer. The tracer can then run some
750                  * internal tracing to verify that everything is in order.
751                  * If we fail, we do not register this tracer.
752                  */
753                 tracing_reset_online_cpus(tr);
754
755                 current_trace = type;
756                 /* the test is responsible for initializing and enabling */
757                 pr_info("Testing tracer %s: ", type->name);
758                 ret = type->selftest(type, tr);
759                 /* the test is responsible for resetting too */
760                 current_trace = saved_tracer;
761                 if (ret) {
762                         printk(KERN_CONT "FAILED!\n");
763                         goto out;
764                 }
765                 /* Only reset on passing, to avoid touching corrupted buffers */
766                 tracing_reset_online_cpus(tr);
767
768                 printk(KERN_CONT "PASSED\n");
769         }
770 #endif
771
772         type->next = trace_types;
773         trace_types = type;
774
775  out:
776         tracing_selftest_running = false;
777         mutex_unlock(&trace_types_lock);
778
779         if (ret || !default_bootup_tracer)
780                 goto out_unlock;
781
782         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
783                 goto out_unlock;
784
785         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
786         /* Do we want this tracer to start on bootup? */
787         tracing_set_tracer(type->name);
788         default_bootup_tracer = NULL;
789         /* disable other selftests, since this will break it. */
790         tracing_selftest_disabled = 1;
791 #ifdef CONFIG_FTRACE_STARTUP_TEST
792         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
793                type->name);
794 #endif
795
796  out_unlock:
797         lock_kernel();
798         return ret;
799 }
800
801 void unregister_tracer(struct tracer *type)
802 {
803         struct tracer **t;
804
805         mutex_lock(&trace_types_lock);
806         for (t = &trace_types; *t; t = &(*t)->next) {
807                 if (*t == type)
808                         goto found;
809         }
810         pr_info("Tracer %s not registered\n", type->name);
811         goto out;
812
813  found:
814         *t = (*t)->next;
815
816         if (type == current_trace && tracer_enabled) {
817                 tracer_enabled = 0;
818                 tracing_stop();
819                 if (current_trace->stop)
820                         current_trace->stop(&global_trace);
821                 current_trace = &nop_trace;
822         }
823 out:
824         mutex_unlock(&trace_types_lock);
825 }
826
827 static void __tracing_reset(struct trace_array *tr, int cpu)
828 {
829         ftrace_disable_cpu();
830         ring_buffer_reset_cpu(tr->buffer, cpu);
831         ftrace_enable_cpu();
832 }
833
834 void tracing_reset(struct trace_array *tr, int cpu)
835 {
836         struct ring_buffer *buffer = tr->buffer;
837
838         ring_buffer_record_disable(buffer);
839
840         /* Make sure all commits have finished */
841         synchronize_sched();
842         __tracing_reset(tr, cpu);
843
844         ring_buffer_record_enable(buffer);
845 }
846
847 void tracing_reset_online_cpus(struct trace_array *tr)
848 {
849         struct ring_buffer *buffer = tr->buffer;
850         int cpu;
851
852         ring_buffer_record_disable(buffer);
853
854         /* Make sure all commits have finished */
855         synchronize_sched();
856
857         tr->time_start = ftrace_now(tr->cpu);
858
859         for_each_online_cpu(cpu)
860                 __tracing_reset(tr, cpu);
861
862         ring_buffer_record_enable(buffer);
863 }
864
865 void tracing_reset_current(int cpu)
866 {
867         tracing_reset(&global_trace, cpu);
868 }
869
870 void tracing_reset_current_online_cpus(void)
871 {
872         tracing_reset_online_cpus(&global_trace);
873 }
874
875 #define SAVED_CMDLINES 128
876 #define NO_CMDLINE_MAP UINT_MAX
877 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
878 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
879 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
880 static int cmdline_idx;
881 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
882
883 /* temporary disable recording */
884 static atomic_t trace_record_cmdline_disabled __read_mostly;
885
886 static void trace_init_cmdlines(void)
887 {
888         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
889         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
890         cmdline_idx = 0;
891 }
892
893 int is_tracing_stopped(void)
894 {
895         return trace_stop_count;
896 }
897
898 /**
899  * ftrace_off_permanent - disable all ftrace code permanently
900  *
901  * This should only be called when a serious anomally has
902  * been detected.  This will turn off the function tracing,
903  * ring buffers, and other tracing utilites. It takes no
904  * locks and can be called from any context.
905  */
906 void ftrace_off_permanent(void)
907 {
908         tracing_disabled = 1;
909         ftrace_stop();
910         tracing_off_permanent();
911 }
912
913 /**
914  * tracing_start - quick start of the tracer
915  *
916  * If tracing is enabled but was stopped by tracing_stop,
917  * this will start the tracer back up.
918  */
919 void tracing_start(void)
920 {
921         struct ring_buffer *buffer;
922         unsigned long flags;
923
924         if (tracing_disabled)
925                 return;
926
927         spin_lock_irqsave(&tracing_start_lock, flags);
928         if (--trace_stop_count) {
929                 if (trace_stop_count < 0) {
930                         /* Someone screwed up their debugging */
931                         WARN_ON_ONCE(1);
932                         trace_stop_count = 0;
933                 }
934                 goto out;
935         }
936
937
938         buffer = global_trace.buffer;
939         if (buffer)
940                 ring_buffer_record_enable(buffer);
941
942         buffer = max_tr.buffer;
943         if (buffer)
944                 ring_buffer_record_enable(buffer);
945
946         ftrace_start();
947  out:
948         spin_unlock_irqrestore(&tracing_start_lock, flags);
949 }
950
951 /**
952  * tracing_stop - quick stop of the tracer
953  *
954  * Light weight way to stop tracing. Use in conjunction with
955  * tracing_start.
956  */
957 void tracing_stop(void)
958 {
959         struct ring_buffer *buffer;
960         unsigned long flags;
961
962         ftrace_stop();
963         spin_lock_irqsave(&tracing_start_lock, flags);
964         if (trace_stop_count++)
965                 goto out;
966
967         buffer = global_trace.buffer;
968         if (buffer)
969                 ring_buffer_record_disable(buffer);
970
971         buffer = max_tr.buffer;
972         if (buffer)
973                 ring_buffer_record_disable(buffer);
974
975  out:
976         spin_unlock_irqrestore(&tracing_start_lock, flags);
977 }
978
979 void trace_stop_cmdline_recording(void);
980
981 static void trace_save_cmdline(struct task_struct *tsk)
982 {
983         unsigned pid, idx;
984
985         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
986                 return;
987
988         /*
989          * It's not the end of the world if we don't get
990          * the lock, but we also don't want to spin
991          * nor do we want to disable interrupts,
992          * so if we miss here, then better luck next time.
993          */
994         if (!arch_spin_trylock(&trace_cmdline_lock))
995                 return;
996
997         idx = map_pid_to_cmdline[tsk->pid];
998         if (idx == NO_CMDLINE_MAP) {
999                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1000
1001                 /*
1002                  * Check whether the cmdline buffer at idx has a pid
1003                  * mapped. We are going to overwrite that entry so we
1004                  * need to clear the map_pid_to_cmdline. Otherwise we
1005                  * would read the new comm for the old pid.
1006                  */
1007                 pid = map_cmdline_to_pid[idx];
1008                 if (pid != NO_CMDLINE_MAP)
1009                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1010
1011                 map_cmdline_to_pid[idx] = tsk->pid;
1012                 map_pid_to_cmdline[tsk->pid] = idx;
1013
1014                 cmdline_idx = idx;
1015         }
1016
1017         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1018
1019         arch_spin_unlock(&trace_cmdline_lock);
1020 }
1021
1022 void trace_find_cmdline(int pid, char comm[])
1023 {
1024         unsigned map;
1025
1026         if (!pid) {
1027                 strcpy(comm, "<idle>");
1028                 return;
1029         }
1030
1031         if (WARN_ON_ONCE(pid < 0)) {
1032                 strcpy(comm, "<XXX>");
1033                 return;
1034         }
1035
1036         if (pid > PID_MAX_DEFAULT) {
1037                 strcpy(comm, "<...>");
1038                 return;
1039         }
1040
1041         preempt_disable();
1042         arch_spin_lock(&trace_cmdline_lock);
1043         map = map_pid_to_cmdline[pid];
1044         if (map != NO_CMDLINE_MAP)
1045                 strcpy(comm, saved_cmdlines[map]);
1046         else
1047                 strcpy(comm, "<...>");
1048
1049         arch_spin_unlock(&trace_cmdline_lock);
1050         preempt_enable();
1051 }
1052
1053 void tracing_record_cmdline(struct task_struct *tsk)
1054 {
1055         if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1056             !tracing_is_on())
1057                 return;
1058
1059         trace_save_cmdline(tsk);
1060 }
1061
1062 void
1063 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1064                              int pc)
1065 {
1066         struct task_struct *tsk = current;
1067
1068         entry->preempt_count            = pc & 0xff;
1069         entry->pid                      = (tsk) ? tsk->pid : 0;
1070         entry->lock_depth               = (tsk) ? tsk->lock_depth : 0;
1071         entry->flags =
1072 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1073                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1074 #else
1075                 TRACE_FLAG_IRQS_NOSUPPORT |
1076 #endif
1077                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1078                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1079                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1080 }
1081 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1082
1083 struct ring_buffer_event *
1084 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1085                           int type,
1086                           unsigned long len,
1087                           unsigned long flags, int pc)
1088 {
1089         struct ring_buffer_event *event;
1090
1091         event = ring_buffer_lock_reserve(buffer, len);
1092         if (event != NULL) {
1093                 struct trace_entry *ent = ring_buffer_event_data(event);
1094
1095                 tracing_generic_entry_update(ent, flags, pc);
1096                 ent->type = type;
1097         }
1098
1099         return event;
1100 }
1101
1102 static inline void
1103 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1104                              struct ring_buffer_event *event,
1105                              unsigned long flags, int pc,
1106                              int wake)
1107 {
1108         ring_buffer_unlock_commit(buffer, event);
1109
1110         ftrace_trace_stack(buffer, flags, 6, pc);
1111         ftrace_trace_userstack(buffer, flags, pc);
1112
1113         if (wake)
1114                 trace_wake_up();
1115 }
1116
1117 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1118                                 struct ring_buffer_event *event,
1119                                 unsigned long flags, int pc)
1120 {
1121         __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1122 }
1123
1124 struct ring_buffer_event *
1125 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1126                                   int type, unsigned long len,
1127                                   unsigned long flags, int pc)
1128 {
1129         *current_rb = global_trace.buffer;
1130         return trace_buffer_lock_reserve(*current_rb,
1131                                          type, len, flags, pc);
1132 }
1133 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1134
1135 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1136                                         struct ring_buffer_event *event,
1137                                         unsigned long flags, int pc)
1138 {
1139         __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
1140 }
1141 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1142
1143 void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1144                                        struct ring_buffer_event *event,
1145                                        unsigned long flags, int pc)
1146 {
1147         __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
1148 }
1149 EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
1150
1151 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1152                                          struct ring_buffer_event *event)
1153 {
1154         ring_buffer_discard_commit(buffer, event);
1155 }
1156 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1157
1158 void
1159 trace_function(struct trace_array *tr,
1160                unsigned long ip, unsigned long parent_ip, unsigned long flags,
1161                int pc)
1162 {
1163         struct ftrace_event_call *call = &event_function;
1164         struct ring_buffer *buffer = tr->buffer;
1165         struct ring_buffer_event *event;
1166         struct ftrace_entry *entry;
1167
1168         /* If we are reading the ring buffer, don't trace */
1169         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1170                 return;
1171
1172         event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1173                                           flags, pc);
1174         if (!event)
1175                 return;
1176         entry   = ring_buffer_event_data(event);
1177         entry->ip                       = ip;
1178         entry->parent_ip                = parent_ip;
1179
1180         if (!filter_check_discard(call, entry, buffer, event))
1181                 ring_buffer_unlock_commit(buffer, event);
1182 }
1183
1184 void
1185 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1186        unsigned long ip, unsigned long parent_ip, unsigned long flags,
1187        int pc)
1188 {
1189         if (likely(!atomic_read(&data->disabled)))
1190                 trace_function(tr, ip, parent_ip, flags, pc);
1191 }
1192
1193 #ifdef CONFIG_STACKTRACE
1194 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1195                                  unsigned long flags,
1196                                  int skip, int pc)
1197 {
1198         struct ftrace_event_call *call = &event_kernel_stack;
1199         struct ring_buffer_event *event;
1200         struct stack_entry *entry;
1201         struct stack_trace trace;
1202
1203         event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1204                                           sizeof(*entry), flags, pc);
1205         if (!event)
1206                 return;
1207         entry   = ring_buffer_event_data(event);
1208         memset(&entry->caller, 0, sizeof(entry->caller));
1209
1210         trace.nr_entries        = 0;
1211         trace.max_entries       = FTRACE_STACK_ENTRIES;
1212         trace.skip              = skip;
1213         trace.entries           = entry->caller;
1214
1215         save_stack_trace(&trace);
1216         if (!filter_check_discard(call, entry, buffer, event))
1217                 ring_buffer_unlock_commit(buffer, event);
1218 }
1219
1220 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1221                         int skip, int pc)
1222 {
1223         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1224                 return;
1225
1226         __ftrace_trace_stack(buffer, flags, skip, pc);
1227 }
1228
1229 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1230                    int pc)
1231 {
1232         __ftrace_trace_stack(tr->buffer, flags, skip, pc);
1233 }
1234
1235 /**
1236  * trace_dump_stack - record a stack back trace in the trace buffer
1237  */
1238 void trace_dump_stack(void)
1239 {
1240         unsigned long flags;
1241
1242         if (tracing_disabled || tracing_selftest_running)
1243                 return;
1244
1245         local_save_flags(flags);
1246
1247         /* skipping 3 traces, seems to get us at the caller of this function */
1248         __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count());
1249 }
1250
1251 void
1252 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1253 {
1254         struct ftrace_event_call *call = &event_user_stack;
1255         struct ring_buffer_event *event;
1256         struct userstack_entry *entry;
1257         struct stack_trace trace;
1258
1259         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1260                 return;
1261
1262         event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1263                                           sizeof(*entry), flags, pc);
1264         if (!event)
1265                 return;
1266         entry   = ring_buffer_event_data(event);
1267
1268         entry->tgid             = current->tgid;
1269         memset(&entry->caller, 0, sizeof(entry->caller));
1270
1271         trace.nr_entries        = 0;
1272         trace.max_entries       = FTRACE_STACK_ENTRIES;
1273         trace.skip              = 0;
1274         trace.entries           = entry->caller;
1275
1276         save_stack_trace_user(&trace);
1277         if (!filter_check_discard(call, entry, buffer, event))
1278                 ring_buffer_unlock_commit(buffer, event);
1279 }
1280
1281 #ifdef UNUSED
1282 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1283 {
1284         ftrace_trace_userstack(tr, flags, preempt_count());
1285 }
1286 #endif /* UNUSED */
1287
1288 #endif /* CONFIG_STACKTRACE */
1289
1290 static void
1291 ftrace_trace_special(void *__tr,
1292                      unsigned long arg1, unsigned long arg2, unsigned long arg3,
1293                      int pc)
1294 {
1295         struct ftrace_event_call *call = &event_special;
1296         struct ring_buffer_event *event;
1297         struct trace_array *tr = __tr;
1298         struct ring_buffer *buffer = tr->buffer;
1299         struct special_entry *entry;
1300
1301         event = trace_buffer_lock_reserve(buffer, TRACE_SPECIAL,
1302                                           sizeof(*entry), 0, pc);
1303         if (!event)
1304                 return;
1305         entry   = ring_buffer_event_data(event);
1306         entry->arg1                     = arg1;
1307         entry->arg2                     = arg2;
1308         entry->arg3                     = arg3;
1309
1310         if (!filter_check_discard(call, entry, buffer, event))
1311                 trace_buffer_unlock_commit(buffer, event, 0, pc);
1312 }
1313
1314 void
1315 __trace_special(void *__tr, void *__data,
1316                 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1317 {
1318         ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
1319 }
1320
1321 void
1322 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1323 {
1324         struct trace_array *tr = &global_trace;
1325         struct trace_array_cpu *data;
1326         unsigned long flags;
1327         int cpu;
1328         int pc;
1329
1330         if (tracing_disabled)
1331                 return;
1332
1333         pc = preempt_count();
1334         local_irq_save(flags);
1335         cpu = raw_smp_processor_id();
1336         data = tr->data[cpu];
1337
1338         if (likely(atomic_inc_return(&data->disabled) == 1))
1339                 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
1340
1341         atomic_dec(&data->disabled);
1342         local_irq_restore(flags);
1343 }
1344
1345 /**
1346  * trace_vbprintk - write binary msg to tracing buffer
1347  *
1348  */
1349 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1350 {
1351         static arch_spinlock_t trace_buf_lock =
1352                 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1353         static u32 trace_buf[TRACE_BUF_SIZE];
1354
1355         struct ftrace_event_call *call = &event_bprint;
1356         struct ring_buffer_event *event;
1357         struct ring_buffer *buffer;
1358         struct trace_array *tr = &global_trace;
1359         struct trace_array_cpu *data;
1360         struct bprint_entry *entry;
1361         unsigned long flags;
1362         int disable;
1363         int resched;
1364         int cpu, len = 0, size, pc;
1365
1366         if (unlikely(tracing_selftest_running || tracing_disabled))
1367                 return 0;
1368
1369         /* Don't pollute graph traces with trace_vprintk internals */
1370         pause_graph_tracing();
1371
1372         pc = preempt_count();
1373         resched = ftrace_preempt_disable();
1374         cpu = raw_smp_processor_id();
1375         data = tr->data[cpu];
1376
1377         disable = atomic_inc_return(&data->disabled);
1378         if (unlikely(disable != 1))
1379                 goto out;
1380
1381         /* Lockdep uses trace_printk for lock tracing */
1382         local_irq_save(flags);
1383         arch_spin_lock(&trace_buf_lock);
1384         len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1385
1386         if (len > TRACE_BUF_SIZE || len < 0)
1387                 goto out_unlock;
1388
1389         size = sizeof(*entry) + sizeof(u32) * len;
1390         buffer = tr->buffer;
1391         event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1392                                           flags, pc);
1393         if (!event)
1394                 goto out_unlock;
1395         entry = ring_buffer_event_data(event);
1396         entry->ip                       = ip;
1397         entry->fmt                      = fmt;
1398
1399         memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1400         if (!filter_check_discard(call, entry, buffer, event)) {
1401                 ring_buffer_unlock_commit(buffer, event);
1402                 ftrace_trace_stack(buffer, flags, 6, pc);
1403         }
1404
1405 out_unlock:
1406         arch_spin_unlock(&trace_buf_lock);
1407         local_irq_restore(flags);
1408
1409 out:
1410         atomic_dec_return(&data->disabled);
1411         ftrace_preempt_enable(resched);
1412         unpause_graph_tracing();
1413
1414         return len;
1415 }
1416 EXPORT_SYMBOL_GPL(trace_vbprintk);
1417
1418 int trace_array_printk(struct trace_array *tr,
1419                        unsigned long ip, const char *fmt, ...)
1420 {
1421         int ret;
1422         va_list ap;
1423
1424         if (!(trace_flags & TRACE_ITER_PRINTK))
1425                 return 0;
1426
1427         va_start(ap, fmt);
1428         ret = trace_array_vprintk(tr, ip, fmt, ap);
1429         va_end(ap);
1430         return ret;
1431 }
1432
1433 int trace_array_vprintk(struct trace_array *tr,
1434                         unsigned long ip, const char *fmt, va_list args)
1435 {
1436         static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1437         static char trace_buf[TRACE_BUF_SIZE];
1438
1439         struct ftrace_event_call *call = &event_print;
1440         struct ring_buffer_event *event;
1441         struct ring_buffer *buffer;
1442         struct trace_array_cpu *data;
1443         int cpu, len = 0, size, pc;
1444         struct print_entry *entry;
1445         unsigned long irq_flags;
1446         int disable;
1447
1448         if (tracing_disabled || tracing_selftest_running)
1449                 return 0;
1450
1451         pc = preempt_count();
1452         preempt_disable_notrace();
1453         cpu = raw_smp_processor_id();
1454         data = tr->data[cpu];
1455
1456         disable = atomic_inc_return(&data->disabled);
1457         if (unlikely(disable != 1))
1458                 goto out;
1459
1460         pause_graph_tracing();
1461         raw_local_irq_save(irq_flags);
1462         arch_spin_lock(&trace_buf_lock);
1463         len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1464
1465         size = sizeof(*entry) + len + 1;
1466         buffer = tr->buffer;
1467         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1468                                           irq_flags, pc);
1469         if (!event)
1470                 goto out_unlock;
1471         entry = ring_buffer_event_data(event);
1472         entry->ip = ip;
1473
1474         memcpy(&entry->buf, trace_buf, len);
1475         entry->buf[len] = '\0';
1476         if (!filter_check_discard(call, entry, buffer, event)) {
1477                 ring_buffer_unlock_commit(buffer, event);
1478                 ftrace_trace_stack(buffer, irq_flags, 6, pc);
1479         }
1480
1481  out_unlock:
1482         arch_spin_unlock(&trace_buf_lock);
1483         raw_local_irq_restore(irq_flags);
1484         unpause_graph_tracing();
1485  out:
1486         atomic_dec_return(&data->disabled);
1487         preempt_enable_notrace();
1488
1489         return len;
1490 }
1491
1492 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1493 {
1494         return trace_array_vprintk(&global_trace, ip, fmt, args);
1495 }
1496 EXPORT_SYMBOL_GPL(trace_vprintk);
1497
1498 enum trace_file_type {
1499         TRACE_FILE_LAT_FMT      = 1,
1500         TRACE_FILE_ANNOTATE     = 2,
1501 };
1502
1503 static void trace_iterator_increment(struct trace_iterator *iter)
1504 {
1505         /* Don't allow ftrace to trace into the ring buffers */
1506         ftrace_disable_cpu();
1507
1508         iter->idx++;
1509         if (iter->buffer_iter[iter->cpu])
1510                 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1511
1512         ftrace_enable_cpu();
1513 }
1514
1515 static struct trace_entry *
1516 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
1517 {
1518         struct ring_buffer_event *event;
1519         struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1520
1521         /* Don't allow ftrace to trace into the ring buffers */
1522         ftrace_disable_cpu();
1523
1524         if (buf_iter)
1525                 event = ring_buffer_iter_peek(buf_iter, ts);
1526         else
1527                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
1528
1529         ftrace_enable_cpu();
1530
1531         return event ? ring_buffer_event_data(event) : NULL;
1532 }
1533
1534 static struct trace_entry *
1535 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1536 {
1537         struct ring_buffer *buffer = iter->tr->buffer;
1538         struct trace_entry *ent, *next = NULL;
1539         int cpu_file = iter->cpu_file;
1540         u64 next_ts = 0, ts;
1541         int next_cpu = -1;
1542         int cpu;
1543
1544         /*
1545          * If we are in a per_cpu trace file, don't bother by iterating over
1546          * all cpu and peek directly.
1547          */
1548         if (cpu_file > TRACE_PIPE_ALL_CPU) {
1549                 if (ring_buffer_empty_cpu(buffer, cpu_file))
1550                         return NULL;
1551                 ent = peek_next_entry(iter, cpu_file, ent_ts);
1552                 if (ent_cpu)
1553                         *ent_cpu = cpu_file;
1554
1555                 return ent;
1556         }
1557
1558         for_each_tracing_cpu(cpu) {
1559
1560                 if (ring_buffer_empty_cpu(buffer, cpu))
1561                         continue;
1562
1563                 ent = peek_next_entry(iter, cpu, &ts);
1564
1565                 /*
1566                  * Pick the entry with the smallest timestamp:
1567                  */
1568                 if (ent && (!next || ts < next_ts)) {
1569                         next = ent;
1570                         next_cpu = cpu;
1571                         next_ts = ts;
1572                 }
1573         }
1574
1575         if (ent_cpu)
1576                 *ent_cpu = next_cpu;
1577
1578         if (ent_ts)
1579                 *ent_ts = next_ts;
1580
1581         return next;
1582 }
1583
1584 /* Find the next real entry, without updating the iterator itself */
1585 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1586                                           int *ent_cpu, u64 *ent_ts)
1587 {
1588         return __find_next_entry(iter, ent_cpu, ent_ts);
1589 }
1590
1591 /* Find the next real entry, and increment the iterator to the next entry */
1592 static void *find_next_entry_inc(struct trace_iterator *iter)
1593 {
1594         iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
1595
1596         if (iter->ent)
1597                 trace_iterator_increment(iter);
1598
1599         return iter->ent ? iter : NULL;
1600 }
1601
1602 static void trace_consume(struct trace_iterator *iter)
1603 {
1604         /* Don't allow ftrace to trace into the ring buffers */
1605         ftrace_disable_cpu();
1606         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1607         ftrace_enable_cpu();
1608 }
1609
1610 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1611 {
1612         struct trace_iterator *iter = m->private;
1613         int i = (int)*pos;
1614         void *ent;
1615
1616         WARN_ON_ONCE(iter->leftover);
1617
1618         (*pos)++;
1619
1620         /* can't go backwards */
1621         if (iter->idx > i)
1622                 return NULL;
1623
1624         if (iter->idx < 0)
1625                 ent = find_next_entry_inc(iter);
1626         else
1627                 ent = iter;
1628
1629         while (ent && iter->idx < i)
1630                 ent = find_next_entry_inc(iter);
1631
1632         iter->pos = *pos;
1633
1634         return ent;
1635 }
1636
1637 static void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1638 {
1639         struct trace_array *tr = iter->tr;
1640         struct ring_buffer_event *event;
1641         struct ring_buffer_iter *buf_iter;
1642         unsigned long entries = 0;
1643         u64 ts;
1644
1645         tr->data[cpu]->skipped_entries = 0;
1646
1647         if (!iter->buffer_iter[cpu])
1648                 return;
1649
1650         buf_iter = iter->buffer_iter[cpu];
1651         ring_buffer_iter_reset(buf_iter);
1652
1653         /*
1654          * We could have the case with the max latency tracers
1655          * that a reset never took place on a cpu. This is evident
1656          * by the timestamp being before the start of the buffer.
1657          */
1658         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1659                 if (ts >= iter->tr->time_start)
1660                         break;
1661                 entries++;
1662                 ring_buffer_read(buf_iter, NULL);
1663         }
1664
1665         tr->data[cpu]->skipped_entries = entries;
1666 }
1667
1668 /*
1669  * The current tracer is copied to avoid a global locking
1670  * all around.
1671  */
1672 static void *s_start(struct seq_file *m, loff_t *pos)
1673 {
1674         struct trace_iterator *iter = m->private;
1675         static struct tracer *old_tracer;
1676         int cpu_file = iter->cpu_file;
1677         void *p = NULL;
1678         loff_t l = 0;
1679         int cpu;
1680
1681         /* copy the tracer to avoid using a global lock all around */
1682         mutex_lock(&trace_types_lock);
1683         if (unlikely(old_tracer != current_trace && current_trace)) {
1684                 old_tracer = current_trace;
1685                 *iter->trace = *current_trace;
1686         }
1687         mutex_unlock(&trace_types_lock);
1688
1689         atomic_inc(&trace_record_cmdline_disabled);
1690
1691         if (*pos != iter->pos) {
1692                 iter->ent = NULL;
1693                 iter->cpu = 0;
1694                 iter->idx = -1;
1695
1696                 ftrace_disable_cpu();
1697
1698                 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1699                         for_each_tracing_cpu(cpu)
1700                                 tracing_iter_reset(iter, cpu);
1701                 } else
1702                         tracing_iter_reset(iter, cpu_file);
1703
1704                 ftrace_enable_cpu();
1705
1706                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1707                         ;
1708
1709         } else {
1710                 /*
1711                  * If we overflowed the seq_file before, then we want
1712                  * to just reuse the trace_seq buffer again.
1713                  */
1714                 if (iter->leftover)
1715                         p = iter;
1716                 else {
1717                         l = *pos - 1;
1718                         p = s_next(m, p, &l);
1719                 }
1720         }
1721
1722         trace_event_read_lock();
1723         trace_access_lock(cpu_file);
1724         return p;
1725 }
1726
1727 static void s_stop(struct seq_file *m, void *p)
1728 {
1729         struct trace_iterator *iter = m->private;
1730
1731         atomic_dec(&trace_record_cmdline_disabled);
1732         trace_access_unlock(iter->cpu_file);
1733         trace_event_read_unlock();
1734 }
1735
1736 static void print_lat_help_header(struct seq_file *m)
1737 {
1738         seq_puts(m, "#                  _------=> CPU#            \n");
1739         seq_puts(m, "#                 / _-----=> irqs-off        \n");
1740         seq_puts(m, "#                | / _----=> need-resched    \n");
1741         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1742         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1743         seq_puts(m, "#                |||| /_--=> lock-depth       \n");
1744         seq_puts(m, "#                |||||/     delay             \n");
1745         seq_puts(m, "#  cmd     pid   |||||| time  |   caller      \n");
1746         seq_puts(m, "#     \\   /      ||||||   \\   |   /           \n");
1747 }
1748
1749 static void print_func_help_header(struct seq_file *m)
1750 {
1751         seq_puts(m, "#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n");
1752         seq_puts(m, "#              | |       |          |         |\n");
1753 }
1754
1755
1756 static void
1757 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1758 {
1759         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1760         struct trace_array *tr = iter->tr;
1761         struct trace_array_cpu *data = tr->data[tr->cpu];
1762         struct tracer *type = current_trace;
1763         unsigned long entries = 0;
1764         unsigned long total = 0;
1765         unsigned long count;
1766         const char *name = "preemption";
1767         int cpu;
1768
1769         if (type)
1770                 name = type->name;
1771
1772
1773         for_each_tracing_cpu(cpu) {
1774                 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1775                 /*
1776                  * If this buffer has skipped entries, then we hold all
1777                  * entries for the trace and we need to ignore the
1778                  * ones before the time stamp.
1779                  */
1780                 if (tr->data[cpu]->skipped_entries) {
1781                         count -= tr->data[cpu]->skipped_entries;
1782                         /* total is the same as the entries */
1783                         total += count;
1784                 } else
1785                         total += count +
1786                                 ring_buffer_overrun_cpu(tr->buffer, cpu);
1787                 entries += count;
1788         }
1789
1790         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1791                    name, UTS_RELEASE);
1792         seq_puts(m, "# -----------------------------------"
1793                  "---------------------------------\n");
1794         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
1795                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1796                    nsecs_to_usecs(data->saved_latency),
1797                    entries,
1798                    total,
1799                    tr->cpu,
1800 #if defined(CONFIG_PREEMPT_NONE)
1801                    "server",
1802 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1803                    "desktop",
1804 #elif defined(CONFIG_PREEMPT)
1805                    "preempt",
1806 #else
1807                    "unknown",
1808 #endif
1809                    /* These are reserved for later use */
1810                    0, 0, 0, 0);
1811 #ifdef CONFIG_SMP
1812         seq_printf(m, " #P:%d)\n", num_online_cpus());
1813 #else
1814         seq_puts(m, ")\n");
1815 #endif
1816         seq_puts(m, "#    -----------------\n");
1817         seq_printf(m, "#    | task: %.16s-%d "
1818                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1819                    data->comm, data->pid, data->uid, data->nice,
1820                    data->policy, data->rt_priority);
1821         seq_puts(m, "#    -----------------\n");
1822
1823         if (data->critical_start) {
1824                 seq_puts(m, "#  => started at: ");
1825                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1826                 trace_print_seq(m, &iter->seq);
1827                 seq_puts(m, "\n#  => ended at:   ");
1828                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1829                 trace_print_seq(m, &iter->seq);
1830                 seq_puts(m, "\n#\n");
1831         }
1832
1833         seq_puts(m, "#\n");
1834 }
1835
1836 static void test_cpu_buff_start(struct trace_iterator *iter)
1837 {
1838         struct trace_seq *s = &iter->seq;
1839
1840         if (!(trace_flags & TRACE_ITER_ANNOTATE))
1841                 return;
1842
1843         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1844                 return;
1845
1846         if (cpumask_test_cpu(iter->cpu, iter->started))
1847                 return;
1848
1849         if (iter->tr->data[iter->cpu]->skipped_entries)
1850                 return;
1851
1852         cpumask_set_cpu(iter->cpu, iter->started);
1853
1854         /* Don't print started cpu buffer for the first entry of the trace */
1855         if (iter->idx > 1)
1856                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1857                                 iter->cpu);
1858 }
1859
1860 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1861 {
1862         struct trace_seq *s = &iter->seq;
1863         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1864         struct trace_entry *entry;
1865         struct trace_event *event;
1866
1867         entry = iter->ent;
1868
1869         test_cpu_buff_start(iter);
1870
1871         event = ftrace_find_event(entry->type);
1872
1873         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1874                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1875                         if (!trace_print_lat_context(iter))
1876                                 goto partial;
1877                 } else {
1878                         if (!trace_print_context(iter))
1879                                 goto partial;
1880                 }
1881         }
1882
1883         if (event)
1884                 return event->trace(iter, sym_flags);
1885
1886         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1887                 goto partial;
1888
1889         return TRACE_TYPE_HANDLED;
1890 partial:
1891         return TRACE_TYPE_PARTIAL_LINE;
1892 }
1893
1894 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1895 {
1896         struct trace_seq *s = &iter->seq;
1897         struct trace_entry *entry;
1898         struct trace_event *event;
1899
1900         entry = iter->ent;
1901
1902         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1903                 if (!trace_seq_printf(s, "%d %d %llu ",
1904                                       entry->pid, iter->cpu, iter->ts))
1905                         goto partial;
1906         }
1907
1908         event = ftrace_find_event(entry->type);
1909         if (event)
1910                 return event->raw(iter, 0);
1911
1912         if (!trace_seq_printf(s, "%d ?\n", entry->type))
1913                 goto partial;
1914
1915         return TRACE_TYPE_HANDLED;
1916 partial:
1917         return TRACE_TYPE_PARTIAL_LINE;
1918 }
1919
1920 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1921 {
1922         struct trace_seq *s = &iter->seq;
1923         unsigned char newline = '\n';
1924         struct trace_entry *entry;
1925         struct trace_event *event;
1926
1927         entry = iter->ent;
1928
1929         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1930                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1931                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1932                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1933         }
1934
1935         event = ftrace_find_event(entry->type);
1936         if (event) {
1937                 enum print_line_t ret = event->hex(iter, 0);
1938                 if (ret != TRACE_TYPE_HANDLED)
1939                         return ret;
1940         }
1941
1942         SEQ_PUT_FIELD_RET(s, newline);
1943
1944         return TRACE_TYPE_HANDLED;
1945 }
1946
1947 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1948 {
1949         struct trace_seq *s = &iter->seq;
1950         struct trace_entry *entry;
1951         struct trace_event *event;
1952
1953         entry = iter->ent;
1954
1955         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1956                 SEQ_PUT_FIELD_RET(s, entry->pid);
1957                 SEQ_PUT_FIELD_RET(s, iter->cpu);
1958                 SEQ_PUT_FIELD_RET(s, iter->ts);
1959         }
1960
1961         event = ftrace_find_event(entry->type);
1962         return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
1963 }
1964
1965 static int trace_empty(struct trace_iterator *iter)
1966 {
1967         int cpu;
1968
1969         /* If we are looking at one CPU buffer, only check that one */
1970         if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
1971                 cpu = iter->cpu_file;
1972                 if (iter->buffer_iter[cpu]) {
1973                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1974                                 return 0;
1975                 } else {
1976                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1977                                 return 0;
1978                 }
1979                 return 1;
1980         }
1981
1982         for_each_tracing_cpu(cpu) {
1983                 if (iter->buffer_iter[cpu]) {
1984                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1985                                 return 0;
1986                 } else {
1987                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1988                                 return 0;
1989                 }
1990         }
1991
1992         return 1;
1993 }
1994
1995 /*  Called with trace_event_read_lock() held. */
1996 static enum print_line_t print_trace_line(struct trace_iterator *iter)
1997 {
1998         enum print_line_t ret;
1999
2000         if (iter->trace && iter->trace->print_line) {
2001                 ret = iter->trace->print_line(iter);
2002                 if (ret != TRACE_TYPE_UNHANDLED)
2003                         return ret;
2004         }
2005
2006         if (iter->ent->type == TRACE_BPRINT &&
2007                         trace_flags & TRACE_ITER_PRINTK &&
2008                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2009                 return trace_print_bprintk_msg_only(iter);
2010
2011         if (iter->ent->type == TRACE_PRINT &&
2012                         trace_flags & TRACE_ITER_PRINTK &&
2013                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2014                 return trace_print_printk_msg_only(iter);
2015
2016         if (trace_flags & TRACE_ITER_BIN)
2017                 return print_bin_fmt(iter);
2018
2019         if (trace_flags & TRACE_ITER_HEX)
2020                 return print_hex_fmt(iter);
2021
2022         if (trace_flags & TRACE_ITER_RAW)
2023                 return print_raw_fmt(iter);
2024
2025         return print_trace_fmt(iter);
2026 }
2027
2028 static int s_show(struct seq_file *m, void *v)
2029 {
2030         struct trace_iterator *iter = v;
2031         int ret;
2032
2033         if (iter->ent == NULL) {
2034                 if (iter->tr) {
2035                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
2036                         seq_puts(m, "#\n");
2037                 }
2038                 if (iter->trace && iter->trace->print_header)
2039                         iter->trace->print_header(m);
2040                 else if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2041                         /* print nothing if the buffers are empty */
2042                         if (trace_empty(iter))
2043                                 return 0;
2044                         print_trace_header(m, iter);
2045                         if (!(trace_flags & TRACE_ITER_VERBOSE))
2046                                 print_lat_help_header(m);
2047                 } else {
2048                         if (!(trace_flags & TRACE_ITER_VERBOSE))
2049                                 print_func_help_header(m);
2050                 }
2051         } else if (iter->leftover) {
2052                 /*
2053                  * If we filled the seq_file buffer earlier, we
2054                  * want to just show it now.
2055                  */
2056                 ret = trace_print_seq(m, &iter->seq);
2057
2058                 /* ret should this time be zero, but you never know */
2059                 iter->leftover = ret;
2060
2061         } else {
2062                 print_trace_line(iter);
2063                 ret = trace_print_seq(m, &iter->seq);
2064                 /*
2065                  * If we overflow the seq_file buffer, then it will
2066                  * ask us for this data again at start up.
2067                  * Use that instead.
2068                  *  ret is 0 if seq_file write succeeded.
2069                  *        -1 otherwise.
2070                  */
2071                 iter->leftover = ret;
2072         }
2073
2074         return 0;
2075 }
2076
2077 static const struct seq_operations tracer_seq_ops = {
2078         .start          = s_start,
2079         .next           = s_next,
2080         .stop           = s_stop,
2081         .show           = s_show,
2082 };
2083
2084 static struct trace_iterator *
2085 __tracing_open(struct inode *inode, struct file *file)
2086 {
2087         long cpu_file = (long) inode->i_private;
2088         void *fail_ret = ERR_PTR(-ENOMEM);
2089         struct trace_iterator *iter;
2090         struct seq_file *m;
2091         int cpu, ret;
2092
2093         if (tracing_disabled)
2094                 return ERR_PTR(-ENODEV);
2095
2096         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2097         if (!iter)
2098                 return ERR_PTR(-ENOMEM);
2099
2100         /*
2101          * We make a copy of the current tracer to avoid concurrent
2102          * changes on it while we are reading.
2103          */
2104         mutex_lock(&trace_types_lock);
2105         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2106         if (!iter->trace)
2107                 goto fail;
2108
2109         if (current_trace)
2110                 *iter->trace = *current_trace;
2111
2112         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2113                 goto fail;
2114
2115         if (current_trace && current_trace->print_max)
2116                 iter->tr = &max_tr;
2117         else
2118                 iter->tr = &global_trace;
2119         iter->pos = -1;
2120         mutex_init(&iter->mutex);
2121         iter->cpu_file = cpu_file;
2122
2123         /* Notify the tracer early; before we stop tracing. */
2124         if (iter->trace && iter->trace->open)
2125                 iter->trace->open(iter);
2126
2127         /* Annotate start of buffers if we had overruns */
2128         if (ring_buffer_overruns(iter->tr->buffer))
2129                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2130
2131         /* stop the trace while dumping */
2132         tracing_stop();
2133
2134         if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2135                 for_each_tracing_cpu(cpu) {
2136
2137                         iter->buffer_iter[cpu] =
2138                                 ring_buffer_read_start(iter->tr->buffer, cpu);
2139                         tracing_iter_reset(iter, cpu);
2140                 }
2141         } else {
2142                 cpu = iter->cpu_file;
2143                 iter->buffer_iter[cpu] =
2144                                 ring_buffer_read_start(iter->tr->buffer, cpu);
2145                 tracing_iter_reset(iter, cpu);
2146         }
2147
2148         ret = seq_open(file, &tracer_seq_ops);
2149         if (ret < 0) {
2150                 fail_ret = ERR_PTR(ret);
2151                 goto fail_buffer;
2152         }
2153
2154         m = file->private_data;
2155         m->private = iter;
2156
2157         mutex_unlock(&trace_types_lock);
2158
2159         return iter;
2160
2161  fail_buffer:
2162         for_each_tracing_cpu(cpu) {
2163                 if (iter->buffer_iter[cpu])
2164                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2165         }
2166         free_cpumask_var(iter->started);
2167         tracing_start();
2168  fail:
2169         mutex_unlock(&trace_types_lock);
2170         kfree(iter->trace);
2171         kfree(iter);
2172
2173         return fail_ret;
2174 }
2175
2176 int tracing_open_generic(struct inode *inode, struct file *filp)
2177 {
2178         if (tracing_disabled)
2179                 return -ENODEV;
2180
2181         filp->private_data = inode->i_private;
2182         return 0;
2183 }
2184
2185 static int tracing_release(struct inode *inode, struct file *file)
2186 {
2187         struct seq_file *m = (struct seq_file *)file->private_data;
2188         struct trace_iterator *iter;
2189         int cpu;
2190
2191         if (!(file->f_mode & FMODE_READ))
2192                 return 0;
2193
2194         iter = m->private;
2195
2196         mutex_lock(&trace_types_lock);
2197         for_each_tracing_cpu(cpu) {
2198                 if (iter->buffer_iter[cpu])
2199                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2200         }
2201
2202         if (iter->trace && iter->trace->close)
2203                 iter->trace->close(iter);
2204
2205         /* reenable tracing if it was previously enabled */
2206         tracing_start();
2207         mutex_unlock(&trace_types_lock);
2208
2209         seq_release(inode, file);
2210         mutex_destroy(&iter->mutex);
2211         free_cpumask_var(iter->started);
2212         kfree(iter->trace);
2213         kfree(iter);
2214         return 0;
2215 }
2216
2217 static int tracing_open(struct inode *inode, struct file *file)
2218 {
2219         struct trace_iterator *iter;
2220         int ret = 0;
2221
2222         /* If this file was open for write, then erase contents */
2223         if ((file->f_mode & FMODE_WRITE) &&
2224             (file->f_flags & O_TRUNC)) {
2225                 long cpu = (long) inode->i_private;
2226
2227                 if (cpu == TRACE_PIPE_ALL_CPU)
2228                         tracing_reset_online_cpus(&global_trace);
2229                 else
2230                         tracing_reset(&global_trace, cpu);
2231         }
2232
2233         if (file->f_mode & FMODE_READ) {
2234                 iter = __tracing_open(inode, file);
2235                 if (IS_ERR(iter))
2236                         ret = PTR_ERR(iter);
2237                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2238                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
2239         }
2240         return ret;
2241 }
2242
2243 static void *
2244 t_next(struct seq_file *m, void *v, loff_t *pos)
2245 {
2246         struct tracer *t = v;
2247
2248         (*pos)++;
2249
2250         if (t)
2251                 t = t->next;
2252
2253         return t;
2254 }
2255
2256 static void *t_start(struct seq_file *m, loff_t *pos)
2257 {
2258         struct tracer *t;
2259         loff_t l = 0;
2260
2261         mutex_lock(&trace_types_lock);
2262         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2263                 ;
2264
2265         return t;
2266 }
2267
2268 static void t_stop(struct seq_file *m, void *p)
2269 {
2270         mutex_unlock(&trace_types_lock);
2271 }
2272
2273 static int t_show(struct seq_file *m, void *v)
2274 {
2275         struct tracer *t = v;
2276
2277         if (!t)
2278                 return 0;
2279
2280         seq_printf(m, "%s", t->name);
2281         if (t->next)
2282                 seq_putc(m, ' ');
2283         else
2284                 seq_putc(m, '\n');
2285
2286         return 0;
2287 }
2288
2289 static const struct seq_operations show_traces_seq_ops = {
2290         .start          = t_start,
2291         .next           = t_next,
2292         .stop           = t_stop,
2293         .show           = t_show,
2294 };
2295
2296 static int show_traces_open(struct inode *inode, struct file *file)
2297 {
2298         if (tracing_disabled)
2299                 return -ENODEV;
2300
2301         return seq_open(file, &show_traces_seq_ops);
2302 }
2303
2304 static ssize_t
2305 tracing_write_stub(struct file *filp, const char __user *ubuf,
2306                    size_t count, loff_t *ppos)
2307 {
2308         return count;
2309 }
2310
2311 static const struct file_operations tracing_fops = {
2312         .open           = tracing_open,
2313         .read           = seq_read,
2314         .write          = tracing_write_stub,
2315         .llseek         = seq_lseek,
2316         .release        = tracing_release,
2317 };
2318
2319 static const struct file_operations show_traces_fops = {
2320         .open           = show_traces_open,
2321         .read           = seq_read,
2322         .release        = seq_release,
2323 };
2324
2325 /*
2326  * Only trace on a CPU if the bitmask is set:
2327  */
2328 static cpumask_var_t tracing_cpumask;
2329
2330 /*
2331  * The tracer itself will not take this lock, but still we want
2332  * to provide a consistent cpumask to user-space:
2333  */
2334 static DEFINE_MUTEX(tracing_cpumask_update_lock);
2335
2336 /*
2337  * Temporary storage for the character representation of the
2338  * CPU bitmask (and one more byte for the newline):
2339  */
2340 static char mask_str[NR_CPUS + 1];
2341
2342 static ssize_t
2343 tracing_cpumask_read(struct file *filp, char __user *ubuf,
2344                      size_t count, loff_t *ppos)
2345 {
2346         int len;
2347
2348         mutex_lock(&tracing_cpumask_update_lock);
2349
2350         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2351         if (count - len < 2) {
2352                 count = -EINVAL;
2353                 goto out_err;
2354         }
2355         len += sprintf(mask_str + len, "\n");
2356         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2357
2358 out_err:
2359         mutex_unlock(&tracing_cpumask_update_lock);
2360
2361         return count;
2362 }
2363
2364 static ssize_t
2365 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2366                       size_t count, loff_t *ppos)
2367 {
2368         int err, cpu;
2369         cpumask_var_t tracing_cpumask_new;
2370
2371         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2372                 return -ENOMEM;
2373
2374         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2375         if (err)
2376                 goto err_unlock;
2377
2378         mutex_lock(&tracing_cpumask_update_lock);
2379
2380         local_irq_disable();
2381         arch_spin_lock(&ftrace_max_lock);
2382         for_each_tracing_cpu(cpu) {
2383                 /*
2384                  * Increase/decrease the disabled counter if we are
2385                  * about to flip a bit in the cpumask:
2386                  */
2387                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2388                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2389                         atomic_inc(&global_trace.data[cpu]->disabled);
2390                 }
2391                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2392                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2393                         atomic_dec(&global_trace.data[cpu]->disabled);
2394                 }
2395         }
2396         arch_spin_unlock(&ftrace_max_lock);
2397         local_irq_enable();
2398
2399         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2400
2401         mutex_unlock(&tracing_cpumask_update_lock);
2402         free_cpumask_var(tracing_cpumask_new);
2403
2404         return count;
2405
2406 err_unlock:
2407         free_cpumask_var(tracing_cpumask_new);
2408
2409         return err;
2410 }
2411
2412 static const struct file_operations tracing_cpumask_fops = {
2413         .open           = tracing_open_generic,
2414         .read           = tracing_cpumask_read,
2415         .write          = tracing_cpumask_write,
2416 };
2417
2418 static int tracing_trace_options_show(struct seq_file *m, void *v)
2419 {
2420         struct tracer_opt *trace_opts;
2421         u32 tracer_flags;
2422         int i;
2423
2424         mutex_lock(&trace_types_lock);
2425         tracer_flags = current_trace->flags->val;
2426         trace_opts = current_trace->flags->opts;
2427
2428         for (i = 0; trace_options[i]; i++) {
2429                 if (trace_flags & (1 << i))
2430                         seq_printf(m, "%s\n", trace_options[i]);
2431                 else
2432                         seq_printf(m, "no%s\n", trace_options[i]);
2433         }
2434
2435         for (i = 0; trace_opts[i].name; i++) {
2436                 if (tracer_flags & trace_opts[i].bit)
2437                         seq_printf(m, "%s\n", trace_opts[i].name);
2438                 else
2439                         seq_printf(m, "no%s\n", trace_opts[i].name);
2440         }
2441         mutex_unlock(&trace_types_lock);
2442
2443         return 0;
2444 }
2445
2446 static int __set_tracer_option(struct tracer *trace,
2447                                struct tracer_flags *tracer_flags,
2448                                struct tracer_opt *opts, int neg)
2449 {
2450         int ret;
2451
2452         ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2453         if (ret)
2454                 return ret;
2455
2456         if (neg)
2457                 tracer_flags->val &= ~opts->bit;
2458         else
2459                 tracer_flags->val |= opts->bit;
2460         return 0;
2461 }
2462
2463 /* Try to assign a tracer specific option */
2464 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2465 {
2466         struct tracer_flags *tracer_flags = trace->flags;
2467         struct tracer_opt *opts = NULL;
2468         int i;
2469
2470         for (i = 0; tracer_flags->opts[i].name; i++) {
2471                 opts = &tracer_flags->opts[i];
2472
2473                 if (strcmp(cmp, opts->name) == 0)
2474                         return __set_tracer_option(trace, trace->flags,
2475                                                    opts, neg);
2476         }
2477
2478         return -EINVAL;
2479 }
2480
2481 static void set_tracer_flags(unsigned int mask, int enabled)
2482 {
2483         /* do nothing if flag is already set */
2484         if (!!(trace_flags & mask) == !!enabled)
2485                 return;
2486
2487         if (enabled)
2488                 trace_flags |= mask;
2489         else
2490                 trace_flags &= ~mask;
2491 }
2492
2493 static ssize_t
2494 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2495                         size_t cnt, loff_t *ppos)
2496 {
2497         char buf[64];
2498         char *cmp;
2499         int neg = 0;
2500         int ret;
2501         int i;
2502
2503         if (cnt >= sizeof(buf))
2504                 return -EINVAL;
2505
2506         if (copy_from_user(&buf, ubuf, cnt))
2507                 return -EFAULT;
2508
2509         buf[cnt] = 0;
2510         cmp = strstrip(buf);
2511
2512         if (strncmp(cmp, "no", 2) == 0) {
2513                 neg = 1;
2514                 cmp += 2;
2515         }
2516
2517         for (i = 0; trace_options[i]; i++) {
2518                 if (strcmp(cmp, trace_options[i]) == 0) {
2519                         set_tracer_flags(1 << i, !neg);
2520                         break;
2521                 }
2522         }
2523
2524         /* If no option could be set, test the specific tracer options */
2525         if (!trace_options[i]) {
2526                 mutex_lock(&trace_types_lock);
2527                 ret = set_tracer_option(current_trace, cmp, neg);
2528                 mutex_unlock(&trace_types_lock);
2529                 if (ret)
2530                         return ret;
2531         }
2532
2533         *ppos += cnt;
2534
2535         return cnt;
2536 }
2537
2538 static int tracing_trace_options_open(struct inode *inode, struct file *file)
2539 {
2540         if (tracing_disabled)
2541                 return -ENODEV;
2542         return single_open(file, tracing_trace_options_show, NULL);
2543 }
2544
2545 static const struct file_operations tracing_iter_fops = {
2546         .open           = tracing_trace_options_open,
2547         .read           = seq_read,
2548         .llseek         = seq_lseek,
2549         .release        = single_release,
2550         .write          = tracing_trace_options_write,
2551 };
2552
2553 static const char readme_msg[] =
2554         "tracing mini-HOWTO:\n\n"
2555         "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2556         "# cat /sys/kernel/debug/tracing/available_tracers\n"
2557         "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
2558         "# cat /sys/kernel/debug/tracing/current_tracer\n"
2559         "nop\n"
2560         "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2561         "# cat /sys/kernel/debug/tracing/current_tracer\n"
2562         "sched_switch\n"
2563         "# cat /sys/kernel/debug/tracing/trace_options\n"
2564         "noprint-parent nosym-offset nosym-addr noverbose\n"
2565         "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2566         "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
2567         "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2568         "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
2569 ;
2570
2571 static ssize_t
2572 tracing_readme_read(struct file *filp, char __user *ubuf,
2573                        size_t cnt, loff_t *ppos)
2574 {
2575         return simple_read_from_buffer(ubuf, cnt, ppos,
2576                                         readme_msg, strlen(readme_msg));
2577 }
2578
2579 static const struct file_operations tracing_readme_fops = {
2580         .open           = tracing_open_generic,
2581         .read           = tracing_readme_read,
2582 };
2583
2584 static ssize_t
2585 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2586                                 size_t cnt, loff_t *ppos)
2587 {
2588         char *buf_comm;
2589         char *file_buf;
2590         char *buf;
2591         int len = 0;
2592         int pid;
2593         int i;
2594
2595         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2596         if (!file_buf)
2597                 return -ENOMEM;
2598
2599         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2600         if (!buf_comm) {
2601                 kfree(file_buf);
2602                 return -ENOMEM;
2603         }
2604
2605         buf = file_buf;
2606
2607         for (i = 0; i < SAVED_CMDLINES; i++) {
2608                 int r;
2609
2610                 pid = map_cmdline_to_pid[i];
2611                 if (pid == -1 || pid == NO_CMDLINE_MAP)
2612                         continue;
2613
2614                 trace_find_cmdline(pid, buf_comm);
2615                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2616                 buf += r;
2617                 len += r;
2618         }
2619
2620         len = simple_read_from_buffer(ubuf, cnt, ppos,
2621                                       file_buf, len);
2622
2623         kfree(file_buf);
2624         kfree(buf_comm);
2625
2626         return len;
2627 }
2628
2629 static const struct file_operations tracing_saved_cmdlines_fops = {
2630     .open       = tracing_open_generic,
2631     .read       = tracing_saved_cmdlines_read,
2632 };
2633
2634 static ssize_t
2635 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2636                   size_t cnt, loff_t *ppos)
2637 {
2638         char buf[64];
2639         int r;
2640
2641         r = sprintf(buf, "%u\n", tracer_enabled);
2642         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2643 }
2644
2645 static ssize_t
2646 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2647                    size_t cnt, loff_t *ppos)
2648 {
2649         struct trace_array *tr = filp->private_data;
2650         char buf[64];
2651         unsigned long val;
2652         int ret;
2653
2654         if (cnt >= sizeof(buf))
2655                 return -EINVAL;
2656
2657         if (copy_from_user(&buf, ubuf, cnt))
2658                 return -EFAULT;
2659
2660         buf[cnt] = 0;
2661
2662         ret = strict_strtoul(buf, 10, &val);
2663         if (ret < 0)
2664                 return ret;
2665
2666         val = !!val;
2667
2668         mutex_lock(&trace_types_lock);
2669         if (tracer_enabled ^ val) {
2670                 if (val) {
2671                         tracer_enabled = 1;
2672                         if (current_trace->start)
2673                                 current_trace->start(tr);
2674                         tracing_start();
2675                 } else {
2676                         tracer_enabled = 0;
2677                         tracing_stop();
2678                         if (current_trace->stop)
2679                                 current_trace->stop(tr);
2680                 }
2681         }
2682         mutex_unlock(&trace_types_lock);
2683
2684         *ppos += cnt;
2685
2686         return cnt;
2687 }
2688
2689 static ssize_t
2690 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2691                        size_t cnt, loff_t *ppos)
2692 {
2693         char buf[MAX_TRACER_SIZE+2];
2694         int r;
2695
2696         mutex_lock(&trace_types_lock);
2697         if (current_trace)
2698                 r = sprintf(buf, "%s\n", current_trace->name);
2699         else
2700                 r = sprintf(buf, "\n");
2701         mutex_unlock(&trace_types_lock);
2702
2703         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2704 }
2705
2706 int tracer_init(struct tracer *t, struct trace_array *tr)
2707 {
2708         tracing_reset_online_cpus(tr);
2709         return t->init(tr);
2710 }
2711
2712 static int tracing_resize_ring_buffer(unsigned long size)
2713 {
2714         int ret;
2715
2716         /*
2717          * If kernel or user changes the size of the ring buffer
2718          * we use the size that was given, and we can forget about
2719          * expanding it later.
2720          */
2721         ring_buffer_expanded = 1;
2722
2723         ret = ring_buffer_resize(global_trace.buffer, size);
2724         if (ret < 0)
2725                 return ret;
2726
2727         ret = ring_buffer_resize(max_tr.buffer, size);
2728         if (ret < 0) {
2729                 int r;
2730
2731                 r = ring_buffer_resize(global_trace.buffer,
2732                                        global_trace.entries);
2733                 if (r < 0) {
2734                         /*
2735                          * AARGH! We are left with different
2736                          * size max buffer!!!!
2737                          * The max buffer is our "snapshot" buffer.
2738                          * When a tracer needs a snapshot (one of the
2739                          * latency tracers), it swaps the max buffer
2740                          * with the saved snap shot. We succeeded to
2741                          * update the size of the main buffer, but failed to
2742                          * update the size of the max buffer. But when we tried
2743                          * to reset the main buffer to the original size, we
2744                          * failed there too. This is very unlikely to
2745                          * happen, but if it does, warn and kill all
2746                          * tracing.
2747                          */
2748                         WARN_ON(1);
2749                         tracing_disabled = 1;
2750                 }
2751                 return ret;
2752         }
2753
2754         global_trace.entries = size;
2755
2756         return ret;
2757 }
2758
2759 /**
2760  * tracing_update_buffers - used by tracing facility to expand ring buffers
2761  *
2762  * To save on memory when the tracing is never used on a system with it
2763  * configured in. The ring buffers are set to a minimum size. But once
2764  * a user starts to use the tracing facility, then they need to grow
2765  * to their default size.
2766  *
2767  * This function is to be called when a tracer is about to be used.
2768  */
2769 int tracing_update_buffers(void)
2770 {
2771         int ret = 0;
2772
2773         mutex_lock(&trace_types_lock);
2774         if (!ring_buffer_expanded)
2775                 ret = tracing_resize_ring_buffer(trace_buf_size);
2776         mutex_unlock(&trace_types_lock);
2777
2778         return ret;
2779 }
2780
2781 struct trace_option_dentry;
2782
2783 static struct trace_option_dentry *
2784 create_trace_option_files(struct tracer *tracer);
2785
2786 static void
2787 destroy_trace_option_files(struct trace_option_dentry *topts);
2788
2789 static int tracing_set_tracer(const char *buf)
2790 {
2791         static struct trace_option_dentry *topts;
2792         struct trace_array *tr = &global_trace;
2793         struct tracer *t;
2794         int ret = 0;
2795
2796         mutex_lock(&trace_types_lock);
2797
2798         if (!ring_buffer_expanded) {
2799                 ret = tracing_resize_ring_buffer(trace_buf_size);
2800                 if (ret < 0)
2801                         goto out;
2802                 ret = 0;
2803         }
2804
2805         for (t = trace_types; t; t = t->next) {
2806                 if (strcmp(t->name, buf) == 0)
2807                         break;
2808         }
2809         if (!t) {
2810                 ret = -EINVAL;
2811                 goto out;
2812         }
2813         if (t == current_trace)
2814                 goto out;
2815
2816         trace_branch_disable();
2817         if (current_trace && current_trace->reset)
2818                 current_trace->reset(tr);
2819
2820         destroy_trace_option_files(topts);
2821
2822         current_trace = t;
2823
2824         topts = create_trace_option_files(current_trace);
2825
2826         if (t->init) {
2827                 ret = tracer_init(t, tr);
2828                 if (ret)
2829                         goto out;
2830         }
2831
2832         trace_branch_enable(tr);
2833  out:
2834         mutex_unlock(&trace_types_lock);
2835
2836         return ret;
2837 }
2838
2839 static ssize_t
2840 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2841                         size_t cnt, loff_t *ppos)
2842 {
2843         char buf[MAX_TRACER_SIZE+1];
2844         int i;
2845         size_t ret;
2846         int err;
2847
2848         ret = cnt;
2849
2850         if (cnt > MAX_TRACER_SIZE)
2851                 cnt = MAX_TRACER_SIZE;
2852
2853         if (copy_from_user(&buf, ubuf, cnt))
2854                 return -EFAULT;
2855
2856         buf[cnt] = 0;
2857
2858         /* strip ending whitespace. */
2859         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2860                 buf[i] = 0;
2861
2862         err = tracing_set_tracer(buf);
2863         if (err)
2864                 return err;
2865
2866         *ppos += ret;
2867
2868         return ret;
2869 }
2870
2871 static ssize_t
2872 tracing_max_lat_read(struct file *filp, char __user *ubuf,
2873                      size_t cnt, loff_t *ppos)
2874 {
2875         unsigned long *ptr = filp->private_data;
2876         char buf[64];
2877         int r;
2878
2879         r = snprintf(buf, sizeof(buf), "%ld\n",
2880                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2881         if (r > sizeof(buf))
2882                 r = sizeof(buf);
2883         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2884 }
2885
2886 static ssize_t
2887 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2888                       size_t cnt, loff_t *ppos)
2889 {
2890         unsigned long *ptr = filp->private_data;
2891         char buf[64];
2892         unsigned long val;
2893         int ret;
2894
2895         if (cnt >= sizeof(buf))
2896                 return -EINVAL;
2897
2898         if (copy_from_user(&buf, ubuf, cnt))
2899                 return -EFAULT;
2900
2901         buf[cnt] = 0;
2902
2903         ret = strict_strtoul(buf, 10, &val);
2904         if (ret < 0)
2905                 return ret;
2906
2907         *ptr = val * 1000;
2908
2909         return cnt;
2910 }
2911
2912 static int tracing_open_pipe(struct inode *inode, struct file *filp)
2913 {
2914         long cpu_file = (long) inode->i_private;
2915         struct trace_iterator *iter;
2916         int ret = 0;
2917
2918         if (tracing_disabled)
2919                 return -ENODEV;
2920
2921         mutex_lock(&trace_types_lock);
2922
2923         /* create a buffer to store the information to pass to userspace */
2924         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2925         if (!iter) {
2926                 ret = -ENOMEM;
2927                 goto out;
2928         }
2929
2930         /*
2931          * We make a copy of the current tracer to avoid concurrent
2932          * changes on it while we are reading.
2933          */
2934         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
2935         if (!iter->trace) {
2936                 ret = -ENOMEM;
2937                 goto fail;
2938         }
2939         if (current_trace)
2940                 *iter->trace = *current_trace;
2941
2942         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
2943                 ret = -ENOMEM;
2944                 goto fail;
2945         }
2946
2947         /* trace pipe does not show start of buffer */
2948         cpumask_setall(iter->started);
2949
2950         if (trace_flags & TRACE_ITER_LATENCY_FMT)
2951                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2952
2953         iter->cpu_file = cpu_file;
2954         iter->tr = &global_trace;
2955         mutex_init(&iter->mutex);
2956         filp->private_data = iter;
2957
2958         if (iter->trace->pipe_open)
2959                 iter->trace->pipe_open(iter);
2960
2961 out:
2962         mutex_unlock(&trace_types_lock);
2963         return ret;
2964
2965 fail:
2966         kfree(iter->trace);
2967         kfree(iter);
2968         mutex_unlock(&trace_types_lock);
2969         return ret;
2970 }
2971
2972 static int tracing_release_pipe(struct inode *inode, struct file *file)
2973 {
2974         struct trace_iterator *iter = file->private_data;
2975
2976         mutex_lock(&trace_types_lock);
2977
2978         if (iter->trace->pipe_close)
2979                 iter->trace->pipe_close(iter);
2980
2981         mutex_unlock(&trace_types_lock);
2982
2983         free_cpumask_var(iter->started);
2984         mutex_destroy(&iter->mutex);
2985         kfree(iter->trace);
2986         kfree(iter);
2987
2988         return 0;
2989 }
2990
2991 static unsigned int
2992 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2993 {
2994         struct trace_iterator *iter = filp->private_data;
2995
2996         if (trace_flags & TRACE_ITER_BLOCK) {
2997                 /*
2998                  * Always select as readable when in blocking mode
2999                  */
3000                 return POLLIN | POLLRDNORM;
3001         } else {
3002                 if (!trace_empty(iter))
3003                         return POLLIN | POLLRDNORM;
3004                 poll_wait(filp, &trace_wait, poll_table);
3005                 if (!trace_empty(iter))
3006                         return POLLIN | POLLRDNORM;
3007
3008                 return 0;
3009         }
3010 }
3011
3012
3013 void default_wait_pipe(struct trace_iterator *iter)
3014 {
3015         DEFINE_WAIT(wait);
3016
3017         prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3018
3019         if (trace_empty(iter))
3020                 schedule();
3021
3022         finish_wait(&trace_wait, &wait);
3023 }
3024
3025 /*
3026  * This is a make-shift waitqueue.
3027  * A tracer might use this callback on some rare cases:
3028  *
3029  *  1) the current tracer might hold the runqueue lock when it wakes up
3030  *     a reader, hence a deadlock (sched, function, and function graph tracers)
3031  *  2) the function tracers, trace all functions, we don't want
3032  *     the overhead of calling wake_up and friends
3033  *     (and tracing them too)
3034  *
3035  *     Anyway, this is really very primitive wakeup.
3036  */
3037 void poll_wait_pipe(struct trace_iterator *iter)
3038 {
3039         set_current_state(TASK_INTERRUPTIBLE);
3040         /* sleep for 100 msecs, and try again. */
3041         schedule_timeout(HZ / 10);
3042 }
3043
3044 /* Must be called with trace_types_lock mutex held. */
3045 static int tracing_wait_pipe(struct file *filp)
3046 {
3047         struct trace_iterator *iter = filp->private_data;
3048
3049         while (trace_empty(iter)) {
3050
3051                 if ((filp->f_flags & O_NONBLOCK)) {
3052                         return -EAGAIN;
3053                 }
3054
3055                 mutex_unlock(&iter->mutex);
3056
3057                 iter->trace->wait_pipe(iter);
3058
3059                 mutex_lock(&iter->mutex);
3060
3061                 if (signal_pending(current))
3062                         return -EINTR;
3063
3064                 /*
3065                  * We block until we read something and tracing is disabled.
3066                  * We still block if tracing is disabled, but we have never
3067                  * read anything. This allows a user to cat this file, and
3068                  * then enable tracing. But after we have read something,
3069                  * we give an EOF when tracing is again disabled.
3070                  *
3071                  * iter->pos will be 0 if we haven't read anything.
3072                  */
3073                 if (!tracer_enabled && iter->pos)
3074                         break;
3075         }
3076
3077         return 1;
3078 }
3079
3080 /*
3081  * Consumer reader.
3082  */
3083 static ssize_t
3084 tracing_read_pipe(struct file *filp, char __user *ubuf,
3085                   size_t cnt, loff_t *ppos)
3086 {
3087         struct trace_iterator *iter = filp->private_data;
3088         static struct tracer *old_tracer;
3089         ssize_t sret;
3090
3091         /* return any leftover data */
3092         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3093         if (sret != -EBUSY)
3094                 return sret;
3095
3096         trace_seq_init(&iter->seq);
3097
3098         /* copy the tracer to avoid using a global lock all around */
3099         mutex_lock(&trace_types_lock);
3100         if (unlikely(old_tracer != current_trace && current_trace)) {
3101                 old_tracer = current_trace;
3102                 *iter->trace = *current_trace;
3103         }
3104         mutex_unlock(&trace_types_lock);
3105
3106         /*
3107          * Avoid more than one consumer on a single file descriptor
3108          * This is just a matter of traces coherency, the ring buffer itself
3109          * is protected.
3110          */
3111         mutex_lock(&iter->mutex);
3112         if (iter->trace->read) {
3113                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3114                 if (sret)
3115                         goto out;
3116         }
3117
3118 waitagain:
3119         sret = tracing_wait_pipe(filp);
3120         if (sret <= 0)
3121                 goto out;
3122
3123         /* stop when tracing is finished */
3124         if (trace_empty(iter)) {
3125                 sret = 0;
3126                 goto out;
3127         }
3128
3129         if (cnt >= PAGE_SIZE)
3130                 cnt = PAGE_SIZE - 1;
3131
3132         /* reset all but tr, trace, and overruns */
3133         memset(&iter->seq, 0,
3134                sizeof(struct trace_iterator) -
3135                offsetof(struct trace_iterator, seq));
3136         iter->pos = -1;
3137
3138         trace_event_read_lock();
3139         trace_access_lock(iter->cpu_file);
3140         while (find_next_entry_inc(iter) != NULL) {
3141                 enum print_line_t ret;
3142                 int len = iter->seq.len;
3143
3144                 ret = print_trace_line(iter);
3145                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3146                         /* don't print partial lines */
3147                         iter->seq.len = len;
3148                         break;
3149                 }
3150                 if (ret != TRACE_TYPE_NO_CONSUME)
3151                         trace_consume(iter);
3152
3153                 if (iter->seq.len >= cnt)
3154                         break;
3155         }
3156         trace_access_unlock(iter->cpu_file);
3157         trace_event_read_unlock();
3158
3159         /* Now copy what we have to the user */
3160         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3161         if (iter->seq.readpos >= iter->seq.len)
3162                 trace_seq_init(&iter->seq);
3163
3164         /*
3165          * If there was nothing to send to user, inspite of consuming trace
3166          * entries, go back to wait for more entries.
3167          */
3168         if (sret == -EBUSY)
3169                 goto waitagain;
3170
3171 out:
3172         mutex_unlock(&iter->mutex);
3173
3174         return sret;
3175 }
3176
3177 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3178                                      struct pipe_buffer *buf)
3179 {
3180         __free_page(buf->page);
3181 }
3182
3183 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3184                                      unsigned int idx)
3185 {
3186         __free_page(spd->pages[idx]);
3187 }
3188
3189 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3190         .can_merge              = 0,
3191         .map                    = generic_pipe_buf_map,
3192         .unmap                  = generic_pipe_buf_unmap,
3193         .confirm                = generic_pipe_buf_confirm,
3194         .release                = tracing_pipe_buf_release,
3195         .steal                  = generic_pipe_buf_steal,
3196         .get                    = generic_pipe_buf_get,
3197 };
3198
3199 static size_t
3200 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3201 {
3202         size_t count;
3203         int ret;
3204
3205         /* Seq buffer is page-sized, exactly what we need. */
3206         for (;;) {
3207                 count = iter->seq.len;
3208                 ret = print_trace_line(iter);
3209                 count = iter->seq.len - count;
3210                 if (rem < count) {
3211                         rem = 0;
3212                         iter->seq.len -= count;
3213                         break;
3214                 }
3215                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3216                         iter->seq.len -= count;
3217                         break;
3218                 }
3219
3220                 if (ret != TRACE_TYPE_NO_CONSUME)
3221                         trace_consume(iter);
3222                 rem -= count;
3223                 if (!find_next_entry_inc(iter)) {
3224                         rem = 0;
3225                         iter->ent = NULL;
3226                         break;
3227                 }
3228         }
3229
3230         return rem;
3231 }
3232
3233 static ssize_t tracing_splice_read_pipe(struct file *filp,
3234                                         loff_t *ppos,
3235                                         struct pipe_inode_info *pipe,
3236                                         size_t len,
3237                                         unsigned int flags)
3238 {
3239         struct page *pages[PIPE_BUFFERS];
3240         struct partial_page partial[PIPE_BUFFERS];
3241         struct trace_iterator *iter = filp->private_data;
3242         struct splice_pipe_desc spd = {
3243                 .pages          = pages,
3244                 .partial        = partial,
3245                 .nr_pages       = 0, /* This gets updated below. */
3246                 .flags          = flags,
3247                 .ops            = &tracing_pipe_buf_ops,
3248                 .spd_release    = tracing_spd_release_pipe,
3249         };
3250         static struct tracer *old_tracer;
3251         ssize_t ret;
3252         size_t rem;
3253         unsigned int i;
3254
3255         /* copy the tracer to avoid using a global lock all around */
3256         mutex_lock(&trace_types_lock);
3257         if (unlikely(old_tracer != current_trace && current_trace)) {
3258                 old_tracer = current_trace;
3259                 *iter->trace = *current_trace;
3260         }
3261         mutex_unlock(&trace_types_lock);
3262
3263         mutex_lock(&iter->mutex);
3264
3265         if (iter->trace->splice_read) {
3266                 ret = iter->trace->splice_read(iter, filp,
3267                                                ppos, pipe, len, flags);
3268                 if (ret)
3269                         goto out_err;
3270         }
3271
3272         ret = tracing_wait_pipe(filp);
3273         if (ret <= 0)
3274                 goto out_err;
3275
3276         if (!iter->ent && !find_next_entry_inc(iter)) {
3277                 ret = -EFAULT;
3278                 goto out_err;
3279         }
3280
3281         trace_event_read_lock();
3282         trace_access_lock(iter->cpu_file);
3283
3284         /* Fill as many pages as possible. */
3285         for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
3286                 pages[i] = alloc_page(GFP_KERNEL);
3287                 if (!pages[i])
3288                         break;
3289
3290                 rem = tracing_fill_pipe_page(rem, iter);
3291
3292                 /* Copy the data into the page, so we can start over. */
3293                 ret = trace_seq_to_buffer(&iter->seq,
3294                                           page_address(pages[i]),
3295                                           iter->seq.len);
3296                 if (ret < 0) {
3297                         __free_page(pages[i]);
3298                         break;
3299                 }
3300                 partial[i].offset = 0;
3301                 partial[i].len = iter->seq.len;
3302
3303                 trace_seq_init(&iter->seq);
3304         }
3305
3306         trace_access_unlock(iter->cpu_file);
3307         trace_event_read_unlock();
3308         mutex_unlock(&iter->mutex);
3309
3310         spd.nr_pages = i;
3311
3312         return splice_to_pipe(pipe, &spd);
3313
3314 out_err:
3315         mutex_unlock(&iter->mutex);
3316
3317         return ret;
3318 }
3319
3320 static ssize_t
3321 tracing_entries_read(struct file *filp, char __user *ubuf,
3322                      size_t cnt, loff_t *ppos)
3323 {
3324         struct trace_array *tr = filp->private_data;
3325         char buf[96];
3326         int r;
3327
3328         mutex_lock(&trace_types_lock);
3329         if (!ring_buffer_expanded)
3330                 r = sprintf(buf, "%lu (expanded: %lu)\n",
3331                             tr->entries >> 10,
3332                             trace_buf_size >> 10);
3333         else
3334                 r = sprintf(buf, "%lu\n", tr->entries >> 10);
3335         mutex_unlock(&trace_types_lock);
3336
3337         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3338 }
3339
3340 static ssize_t
3341 tracing_entries_write(struct file *filp, const char __user *ubuf,
3342                       size_t cnt, loff_t *ppos)
3343 {
3344         unsigned long val;
3345         char buf[64];
3346         int ret, cpu;
3347
3348         if (cnt >= sizeof(buf))
3349                 return -EINVAL;
3350
3351         if (copy_from_user(&buf, ubuf, cnt))
3352                 return -EFAULT;
3353
3354         buf[cnt] = 0;
3355
3356         ret = strict_strtoul(buf, 10, &val);
3357         if (ret < 0)
3358                 return ret;
3359
3360         /* must have at least 1 entry */
3361         if (!val)
3362                 return -EINVAL;
3363
3364         mutex_lock(&trace_types_lock);
3365
3366         tracing_stop();
3367
3368         /* disable all cpu buffers */
3369         for_each_tracing_cpu(cpu) {
3370                 if (global_trace.data[cpu])
3371                         atomic_inc(&global_trace.data[cpu]->disabled);
3372                 if (max_tr.data[cpu])
3373                         atomic_inc(&max_tr.data[cpu]->disabled);
3374         }
3375
3376         /* value is in KB */
3377         val <<= 10;
3378
3379         if (val != global_trace.entries) {
3380                 ret = tracing_resize_ring_buffer(val);
3381                 if (ret < 0) {
3382                         cnt = ret;
3383                         goto out;
3384                 }
3385         }
3386
3387         *ppos += cnt;
3388
3389         /* If check pages failed, return ENOMEM */
3390         if (tracing_disabled)
3391                 cnt = -ENOMEM;
3392  out:
3393         for_each_tracing_cpu(cpu) {
3394                 if (global_trace.data[cpu])
3395                         atomic_dec(&global_trace.data[cpu]->disabled);
3396                 if (max_tr.data[cpu])
3397                         atomic_dec(&max_tr.data[cpu]->disabled);
3398         }
3399
3400         tracing_start();
3401         max_tr.entries = global_trace.entries;
3402         mutex_unlock(&trace_types_lock);
3403
3404         return cnt;
3405 }
3406
3407 static int mark_printk(const char *fmt, ...)
3408 {
3409         int ret;
3410         va_list args;
3411         va_start(args, fmt);
3412         ret = trace_vprintk(0, fmt, args);
3413         va_end(args);
3414         return ret;
3415 }
3416
3417 static ssize_t
3418 tracing_mark_write(struct file *filp, const char __user *ubuf,
3419                                         size_t cnt, loff_t *fpos)
3420 {
3421         char *buf;
3422
3423         if (tracing_disabled)
3424                 return -EINVAL;
3425
3426         if (cnt > TRACE_BUF_SIZE)
3427                 cnt = TRACE_BUF_SIZE;
3428
3429         buf = kmalloc(cnt + 2, GFP_KERNEL);
3430         if (buf == NULL)
3431                 return -ENOMEM;
3432
3433         if (copy_from_user(buf, ubuf, cnt)) {
3434                 kfree(buf);
3435                 return -EFAULT;
3436         }
3437         if (buf[cnt-1] != '\n') {
3438                 buf[cnt] = '\n';
3439                 buf[cnt+1] = '\0';
3440         } else
3441                 buf[cnt] = '\0';
3442
3443         cnt = mark_printk("%s", buf);
3444         kfree(buf);
3445         *fpos += cnt;
3446
3447         return cnt;
3448 }
3449
3450 static int tracing_clock_show(struct seq_file *m, void *v)
3451 {
3452         int i;
3453
3454         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3455                 seq_printf(m,
3456                         "%s%s%s%s", i ? " " : "",
3457                         i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3458                         i == trace_clock_id ? "]" : "");
3459         seq_putc(m, '\n');
3460
3461         return 0;
3462 }
3463
3464 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3465                                    size_t cnt, loff_t *fpos)
3466 {
3467         char buf[64];
3468         const char *clockstr;
3469         int i;
3470
3471         if (cnt >= sizeof(buf))
3472                 return -EINVAL;
3473
3474         if (copy_from_user(&buf, ubuf, cnt))
3475                 return -EFAULT;
3476
3477         buf[cnt] = 0;
3478
3479         clockstr = strstrip(buf);
3480
3481         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3482                 if (strcmp(trace_clocks[i].name, clockstr) == 0)
3483                         break;
3484         }
3485         if (i == ARRAY_SIZE(trace_clocks))
3486                 return -EINVAL;
3487
3488         trace_clock_id = i;
3489
3490         mutex_lock(&trace_types_lock);
3491
3492         ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3493         if (max_tr.buffer)
3494                 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3495
3496         mutex_unlock(&trace_types_lock);
3497
3498         *fpos += cnt;
3499
3500         return cnt;
3501 }
3502
3503 static int tracing_clock_open(struct inode *inode, struct file *file)
3504 {
3505         if (tracing_disabled)
3506                 return -ENODEV;
3507         return single_open(file, tracing_clock_show, NULL);
3508 }
3509
3510 static const struct file_operations tracing_max_lat_fops = {
3511         .open           = tracing_open_generic,
3512         .read           = tracing_max_lat_read,
3513         .write          = tracing_max_lat_write,
3514 };
3515
3516 static const struct file_operations tracing_ctrl_fops = {
3517         .open           = tracing_open_generic,
3518         .read           = tracing_ctrl_read,
3519         .write          = tracing_ctrl_write,
3520 };
3521
3522 static const struct file_operations set_tracer_fops = {
3523         .open           = tracing_open_generic,
3524         .read           = tracing_set_trace_read,
3525         .write          = tracing_set_trace_write,
3526 };
3527
3528 static const struct file_operations tracing_pipe_fops = {
3529         .open           = tracing_open_pipe,
3530         .poll           = tracing_poll_pipe,
3531         .read           = tracing_read_pipe,
3532         .splice_read    = tracing_splice_read_pipe,
3533         .release        = tracing_release_pipe,
3534 };
3535
3536 static const struct file_operations tracing_entries_fops = {
3537         .open           = tracing_open_generic,
3538         .read           = tracing_entries_read,
3539         .write          = tracing_entries_write,
3540 };
3541
3542 static const struct file_operations tracing_mark_fops = {
3543         .open           = tracing_open_generic,
3544         .write          = tracing_mark_write,
3545 };
3546
3547 static const struct file_operations trace_clock_fops = {
3548         .open           = tracing_clock_open,
3549         .read           = seq_read,
3550         .llseek         = seq_lseek,
3551         .release        = single_release,
3552         .write          = tracing_clock_write,
3553 };
3554
3555 struct ftrace_buffer_info {
3556         struct trace_array      *tr;
3557         void                    *spare;
3558         int                     cpu;
3559         unsigned int            read;
3560 };
3561
3562 static int tracing_buffers_open(struct inode *inode, struct file *filp)
3563 {
3564         int cpu = (int)(long)inode->i_private;
3565         struct ftrace_buffer_info *info;
3566
3567         if (tracing_disabled)
3568                 return -ENODEV;
3569
3570         info = kzalloc(sizeof(*info), GFP_KERNEL);
3571         if (!info)
3572                 return -ENOMEM;
3573
3574         info->tr        = &global_trace;
3575         info->cpu       = cpu;
3576         info->spare     = NULL;
3577         /* Force reading ring buffer for first read */
3578         info->read      = (unsigned int)-1;
3579
3580         filp->private_data = info;
3581
3582         return nonseekable_open(inode, filp);
3583 }
3584
3585 static ssize_t
3586 tracing_buffers_read(struct file *filp, char __user *ubuf,
3587                      size_t count, loff_t *ppos)
3588 {
3589         struct ftrace_buffer_info *info = filp->private_data;
3590         unsigned int pos;
3591         ssize_t ret;
3592         size_t size;
3593
3594         if (!count)
3595                 return 0;
3596
3597         if (!info->spare)
3598                 info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3599         if (!info->spare)
3600                 return -ENOMEM;
3601
3602         /* Do we have previous read data to read? */
3603         if (info->read < PAGE_SIZE)
3604                 goto read;
3605
3606         info->read = 0;
3607
3608         trace_access_lock(info->cpu);
3609         ret = ring_buffer_read_page(info->tr->buffer,
3610                                     &info->spare,
3611                                     count,
3612                                     info->cpu, 0);
3613         trace_access_unlock(info->cpu);
3614         if (ret < 0)
3615                 return 0;
3616
3617         pos = ring_buffer_page_len(info->spare);
3618
3619         if (pos < PAGE_SIZE)
3620                 memset(info->spare + pos, 0, PAGE_SIZE - pos);
3621
3622 read:
3623         size = PAGE_SIZE - info->read;
3624         if (size > count)
3625                 size = count;
3626
3627         ret = copy_to_user(ubuf, info->spare + info->read, size);
3628         if (ret == size)
3629                 return -EFAULT;
3630         size -= ret;
3631
3632         *ppos += size;
3633         info->read += size;
3634
3635         return size;
3636 }
3637
3638 static int tracing_buffers_release(struct inode *inode, struct file *file)
3639 {
3640         struct ftrace_buffer_info *info = file->private_data;
3641
3642         if (info->spare)
3643                 ring_buffer_free_read_page(info->tr->buffer, info->spare);
3644         kfree(info);
3645
3646         return 0;
3647 }
3648
3649 struct buffer_ref {
3650         struct ring_buffer      *buffer;
3651         void                    *page;
3652         int                     ref;
3653 };
3654
3655 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3656                                     struct pipe_buffer *buf)
3657 {
3658         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3659
3660         if (--ref->ref)
3661                 return;
3662
3663         ring_buffer_free_read_page(ref->buffer, ref->page);
3664         kfree(ref);
3665         buf->private = 0;
3666 }
3667
3668 static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3669                                  struct pipe_buffer *buf)
3670 {
3671         return 1;
3672 }
3673
3674 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3675                                 struct pipe_buffer *buf)
3676 {
3677         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3678
3679         ref->ref++;
3680 }
3681
3682 /* Pipe buffer operations for a buffer. */
3683 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
3684         .can_merge              = 0,
3685         .map                    = generic_pipe_buf_map,
3686         .unmap                  = generic_pipe_buf_unmap,
3687         .confirm                = generic_pipe_buf_confirm,
3688         .release                = buffer_pipe_buf_release,
3689         .steal                  = buffer_pipe_buf_steal,
3690         .get                    = buffer_pipe_buf_get,
3691 };
3692
3693 /*
3694  * Callback from splice_to_pipe(), if we need to release some pages
3695  * at the end of the spd in case we error'ed out in filling the pipe.
3696  */
3697 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3698 {
3699         struct buffer_ref *ref =
3700                 (struct buffer_ref *)spd->partial[i].private;
3701
3702         if (--ref->ref)
3703                 return;
3704
3705         ring_buffer_free_read_page(ref->buffer, ref->page);
3706         kfree(ref);
3707         spd->partial[i].private = 0;
3708 }
3709
3710 static ssize_t
3711 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
3712                             struct pipe_inode_info *pipe, size_t len,
3713                             unsigned int flags)
3714 {
3715         struct ftrace_buffer_info *info = file->private_data;
3716         struct partial_page partial[PIPE_BUFFERS];
3717         struct page *pages[PIPE_BUFFERS];
3718         struct splice_pipe_desc spd = {
3719                 .pages          = pages,
3720                 .partial        = partial,
3721                 .flags          = flags,
3722                 .ops            = &buffer_pipe_buf_ops,
3723                 .spd_release    = buffer_spd_release,
3724         };
3725         struct buffer_ref *ref;
3726         int entries, size, i;
3727         size_t ret;
3728
3729         if (*ppos & (PAGE_SIZE - 1)) {
3730                 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
3731                 return -EINVAL;
3732         }
3733
3734         if (len & (PAGE_SIZE - 1)) {
3735                 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
3736                 if (len < PAGE_SIZE)
3737                         return -EINVAL;
3738                 len &= PAGE_MASK;
3739         }
3740
3741         trace_access_lock(info->cpu);
3742         entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3743
3744         for (i = 0; i < PIPE_BUFFERS && len && entries; i++, len -= PAGE_SIZE) {
3745                 struct page *page;
3746                 int r;
3747
3748                 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3749                 if (!ref)
3750                         break;
3751
3752                 ref->ref = 1;
3753                 ref->buffer = info->tr->buffer;
3754                 ref->page = ring_buffer_alloc_read_page(ref->buffer);
3755                 if (!ref->page) {
3756                         kfree(ref);
3757                         break;
3758                 }
3759
3760                 r = ring_buffer_read_page(ref->buffer, &ref->page,
3761                                           len, info->cpu, 1);
3762                 if (r < 0) {
3763                         ring_buffer_free_read_page(ref->buffer,
3764                                                    ref->page);
3765                         kfree(ref);
3766                         break;
3767                 }
3768
3769                 /*
3770                  * zero out any left over data, this is going to
3771                  * user land.
3772                  */
3773                 size = ring_buffer_page_len(ref->page);
3774                 if (size < PAGE_SIZE)
3775                         memset(ref->page + size, 0, PAGE_SIZE - size);
3776
3777                 page = virt_to_page(ref->page);
3778
3779                 spd.pages[i] = page;
3780                 spd.partial[i].len = PAGE_SIZE;
3781                 spd.partial[i].offset = 0;
3782                 spd.partial[i].private = (unsigned long)ref;
3783                 spd.nr_pages++;
3784                 *ppos += PAGE_SIZE;
3785
3786                 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3787         }
3788
3789         trace_access_unlock(info->cpu);
3790         spd.nr_pages = i;
3791
3792         /* did we read anything? */
3793         if (!spd.nr_pages) {
3794                 if (flags & SPLICE_F_NONBLOCK)
3795                         ret = -EAGAIN;
3796                 else
3797                         ret = 0;
3798                 /* TODO: block */
3799                 return ret;
3800         }
3801
3802         ret = splice_to_pipe(pipe, &spd);
3803
3804         return ret;
3805 }
3806
3807 static const struct file_operations tracing_buffers_fops = {
3808         .open           = tracing_buffers_open,
3809         .read           = tracing_buffers_read,
3810         .release        = tracing_buffers_release,
3811         .splice_read    = tracing_buffers_splice_read,
3812         .llseek         = no_llseek,
3813 };
3814
3815 static ssize_t
3816 tracing_stats_read(struct file *filp, char __user *ubuf,
3817                    size_t count, loff_t *ppos)
3818 {
3819         unsigned long cpu = (unsigned long)filp->private_data;
3820         struct trace_array *tr = &global_trace;
3821         struct trace_seq *s;
3822         unsigned long cnt;
3823
3824         s = kmalloc(sizeof(*s), GFP_KERNEL);
3825         if (!s)
3826                 return -ENOMEM;
3827
3828         trace_seq_init(s);
3829
3830         cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
3831         trace_seq_printf(s, "entries: %ld\n", cnt);
3832
3833         cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
3834         trace_seq_printf(s, "overrun: %ld\n", cnt);
3835
3836         cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
3837         trace_seq_printf(s, "commit overrun: %ld\n", cnt);
3838
3839         count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
3840
3841         kfree(s);
3842
3843         return count;
3844 }
3845
3846 static const struct file_operations tracing_stats_fops = {
3847         .open           = tracing_open_generic,
3848         .read           = tracing_stats_read,
3849 };
3850
3851 #ifdef CONFIG_DYNAMIC_FTRACE
3852
3853 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3854 {
3855         return 0;
3856 }
3857
3858 static ssize_t
3859 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
3860                   size_t cnt, loff_t *ppos)
3861 {
3862         static char ftrace_dyn_info_buffer[1024];
3863         static DEFINE_MUTEX(dyn_info_mutex);
3864         unsigned long *p = filp->private_data;
3865         char *buf = ftrace_dyn_info_buffer;
3866         int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
3867         int r;
3868
3869         mutex_lock(&dyn_info_mutex);
3870         r = sprintf(buf, "%ld ", *p);
3871
3872         r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
3873         buf[r++] = '\n';
3874
3875         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3876
3877         mutex_unlock(&dyn_info_mutex);
3878
3879         return r;
3880 }
3881
3882 static const struct file_operations tracing_dyn_info_fops = {
3883         .open           = tracing_open_generic,
3884         .read           = tracing_read_dyn_info,
3885 };
3886 #endif
3887
3888 static struct dentry *d_tracer;
3889
3890 struct dentry *tracing_init_dentry(void)
3891 {
3892         static int once;
3893
3894         if (d_tracer)
3895                 return d_tracer;
3896
3897         if (!debugfs_initialized())
3898                 return NULL;
3899
3900         d_tracer = debugfs_create_dir("tracing", NULL);
3901
3902         if (!d_tracer && !once) {
3903                 once = 1;
3904                 pr_warning("Could not create debugfs directory 'tracing'\n");
3905                 return NULL;
3906         }
3907
3908         return d_tracer;
3909 }
3910
3911 static struct dentry *d_percpu;
3912
3913 struct dentry *tracing_dentry_percpu(void)
3914 {
3915         static int once;
3916         struct dentry *d_tracer;
3917
3918         if (d_percpu)
3919                 return d_percpu;
3920
3921         d_tracer = tracing_init_dentry();
3922
3923         if (!d_tracer)
3924                 return NULL;
3925
3926         d_percpu = debugfs_create_dir("per_cpu", d_tracer);
3927
3928         if (!d_percpu && !once) {
3929                 once = 1;
3930                 pr_warning("Could not create debugfs directory 'per_cpu'\n");
3931                 return NULL;
3932         }
3933
3934         return d_percpu;
3935 }
3936
3937 static void tracing_init_debugfs_percpu(long cpu)
3938 {
3939         struct dentry *d_percpu = tracing_dentry_percpu();
3940         struct dentry *d_cpu;
3941         /* strlen(cpu) + MAX(log10(cpu)) + '\0' */
3942         char cpu_dir[7];
3943
3944         if (cpu > 999 || cpu < 0)
3945                 return;
3946
3947         sprintf(cpu_dir, "cpu%ld", cpu);
3948         d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
3949         if (!d_cpu) {
3950                 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
3951                 return;
3952         }
3953
3954         /* per cpu trace_pipe */
3955         trace_create_file("trace_pipe", 0444, d_cpu,
3956                         (void *) cpu, &tracing_pipe_fops);
3957
3958         /* per cpu trace */
3959         trace_create_file("trace", 0644, d_cpu,
3960                         (void *) cpu, &tracing_fops);
3961
3962         trace_create_file("trace_pipe_raw", 0444, d_cpu,
3963                         (void *) cpu, &tracing_buffers_fops);
3964
3965         trace_create_file("stats", 0444, d_cpu,
3966                         (void *) cpu, &tracing_stats_fops);
3967 }
3968
3969 #ifdef CONFIG_FTRACE_SELFTEST
3970 /* Let selftest have access to static functions in this file */
3971 #include "trace_selftest.c"
3972 #endif
3973
3974 struct trace_option_dentry {
3975         struct tracer_opt               *opt;
3976         struct tracer_flags             *flags;
3977         struct dentry                   *entry;
3978 };
3979
3980 static ssize_t
3981 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
3982                         loff_t *ppos)
3983 {
3984         struct trace_option_dentry *topt = filp->private_data;
3985         char *buf;
3986
3987         if (topt->flags->val & topt->opt->bit)
3988                 buf = "1\n";
3989         else
3990                 buf = "0\n";
3991
3992         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3993 }
3994
3995 static ssize_t
3996 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
3997                          loff_t *ppos)
3998 {
3999         struct trace_option_dentry *topt = filp->private_data;
4000         unsigned long val;
4001         char buf[64];
4002         int ret;
4003
4004         if (cnt >= sizeof(buf))
4005                 return -EINVAL;
4006
4007         if (copy_from_user(&buf, ubuf, cnt))
4008                 return -EFAULT;
4009
4010         buf[cnt] = 0;
4011
4012         ret = strict_strtoul(buf, 10, &val);
4013         if (ret < 0)
4014                 return ret;
4015
4016         if (val != 0 && val != 1)
4017                 return -EINVAL;
4018
4019         if (!!(topt->flags->val & topt->opt->bit) != val) {
4020                 mutex_lock(&trace_types_lock);
4021                 ret = __set_tracer_option(current_trace, topt->flags,
4022                                           topt->opt, !val);
4023                 mutex_unlock(&trace_types_lock);
4024                 if (ret)
4025                         return ret;
4026         }
4027
4028         *ppos += cnt;
4029
4030         return cnt;
4031 }
4032
4033
4034 static const struct file_operations trace_options_fops = {
4035         .open = tracing_open_generic,
4036         .read = trace_options_read,
4037         .write = trace_options_write,
4038 };
4039
4040 static ssize_t
4041 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4042                         loff_t *ppos)
4043 {
4044         long index = (long)filp->private_data;
4045         char *buf;
4046
4047         if (trace_flags & (1 << index))
4048                 buf = "1\n";
4049         else
4050                 buf = "0\n";
4051
4052         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4053 }
4054
4055 static ssize_t
4056 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4057                          loff_t *ppos)
4058 {
4059         long index = (long)filp->private_data;
4060         char buf[64];
4061         unsigned long val;
4062         int ret;
4063
4064         if (cnt >= sizeof(buf))
4065                 return -EINVAL;
4066
4067         if (copy_from_user(&buf, ubuf, cnt))
4068                 return -EFAULT;
4069
4070         buf[cnt] = 0;
4071
4072         ret = strict_strtoul(buf, 10, &val);
4073         if (ret < 0)
4074                 return ret;
4075
4076         if (val != 0 && val != 1)
4077                 return -EINVAL;
4078         set_tracer_flags(1 << index, val);
4079
4080         *ppos += cnt;
4081
4082         return cnt;
4083 }
4084
4085 static const struct file_operations trace_options_core_fops = {
4086         .open = tracing_open_generic,
4087         .read = trace_options_core_read,
4088         .write = trace_options_core_write,
4089 };
4090
4091 struct dentry *trace_create_file(const char *name,
4092                                  mode_t mode,
4093                                  struct dentry *parent,
4094                                  void *data,
4095                                  const struct file_operations *fops)
4096 {
4097         struct dentry *ret;
4098
4099         ret = debugfs_create_file(name, mode, parent, data, fops);
4100         if (!ret)
4101                 pr_warning("Could not create debugfs '%s' entry\n", name);
4102
4103         return ret;
4104 }
4105
4106
4107 static struct dentry *trace_options_init_dentry(void)
4108 {
4109         struct dentry *d_tracer;
4110         static struct dentry *t_options;
4111
4112         if (t_options)
4113                 return t_options;
4114
4115         d_tracer = tracing_init_dentry();
4116         if (!d_tracer)
4117                 return NULL;
4118
4119         t_options = debugfs_create_dir("options", d_tracer);
4120         if (!t_options) {
4121                 pr_warning("Could not create debugfs directory 'options'\n");
4122                 return NULL;
4123         }
4124
4125         return t_options;
4126 }
4127
4128 static void
4129 create_trace_option_file(struct trace_option_dentry *topt,
4130                          struct tracer_flags *flags,
4131                          struct tracer_opt *opt)
4132 {
4133         struct dentry *t_options;
4134
4135         t_options = trace_options_init_dentry();
4136         if (!t_options)
4137                 return;
4138
4139         topt->flags = flags;
4140         topt->opt = opt;
4141
4142         topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4143                                     &trace_options_fops);
4144
4145 }
4146
4147 static struct trace_option_dentry *
4148 create_trace_option_files(struct tracer *tracer)
4149 {
4150         struct trace_option_dentry *topts;
4151         struct tracer_flags *flags;
4152         struct tracer_opt *opts;
4153         int cnt;
4154
4155         if (!tracer)
4156                 return NULL;
4157
4158         flags = tracer->flags;
4159
4160         if (!flags || !flags->opts)
4161                 return NULL;
4162
4163         opts = flags->opts;
4164
4165         for (cnt = 0; opts[cnt].name; cnt++)
4166                 ;
4167
4168         topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4169         if (!topts)
4170                 return NULL;
4171
4172         for (cnt = 0; opts[cnt].name; cnt++)
4173                 create_trace_option_file(&topts[cnt], flags,
4174                                          &opts[cnt]);
4175
4176         return topts;
4177 }
4178
4179 static void
4180 destroy_trace_option_files(struct trace_option_dentry *topts)
4181 {
4182         int cnt;
4183
4184         if (!topts)
4185                 return;
4186
4187         for (cnt = 0; topts[cnt].opt; cnt++) {
4188                 if (topts[cnt].entry)
4189                         debugfs_remove(topts[cnt].entry);
4190         }
4191
4192         kfree(topts);
4193 }
4194
4195 static struct dentry *
4196 create_trace_option_core_file(const char *option, long index)
4197 {
4198         struct dentry *t_options;
4199
4200         t_options = trace_options_init_dentry();
4201         if (!t_options)
4202                 return NULL;
4203
4204         return trace_create_file(option, 0644, t_options, (void *)index,
4205                                     &trace_options_core_fops);
4206 }
4207
4208 static __init void create_trace_options_dir(void)
4209 {
4210         struct dentry *t_options;
4211         int i;
4212
4213         t_options = trace_options_init_dentry();
4214         if (!t_options)
4215                 return;
4216
4217         for (i = 0; trace_options[i]; i++)
4218                 create_trace_option_core_file(trace_options[i], i);
4219 }
4220
4221 static __init int tracer_init_debugfs(void)
4222 {
4223         struct dentry *d_tracer;
4224         int cpu;
4225
4226         trace_access_lock_init();
4227
4228         d_tracer = tracing_init_dentry();
4229
4230         trace_create_file("tracing_enabled", 0644, d_tracer,
4231                         &global_trace, &tracing_ctrl_fops);
4232
4233         trace_create_file("trace_options", 0644, d_tracer,
4234                         NULL, &tracing_iter_fops);
4235
4236         trace_create_file("tracing_cpumask", 0644, d_tracer,
4237                         NULL, &tracing_cpumask_fops);
4238
4239         trace_create_file("trace", 0644, d_tracer,
4240                         (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
4241
4242         trace_create_file("available_tracers", 0444, d_tracer,
4243                         &global_trace, &show_traces_fops);
4244
4245         trace_create_file("current_tracer", 0644, d_tracer,
4246                         &global_trace, &set_tracer_fops);
4247
4248 #ifdef CONFIG_TRACER_MAX_TRACE
4249         trace_create_file("tracing_max_latency", 0644, d_tracer,
4250                         &tracing_max_latency, &tracing_max_lat_fops);
4251
4252         trace_create_file("tracing_thresh", 0644, d_tracer,
4253                         &tracing_thresh, &tracing_max_lat_fops);
4254 #endif
4255
4256         trace_create_file("README", 0444, d_tracer,
4257                         NULL, &tracing_readme_fops);
4258
4259         trace_create_file("trace_pipe", 0444, d_tracer,
4260                         (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
4261
4262         trace_create_file("buffer_size_kb", 0644, d_tracer,
4263                         &global_trace, &tracing_entries_fops);
4264
4265         trace_create_file("trace_marker", 0220, d_tracer,
4266                         NULL, &tracing_mark_fops);
4267
4268         trace_create_file("saved_cmdlines", 0444, d_tracer,
4269                         NULL, &tracing_saved_cmdlines_fops);
4270
4271         trace_create_file("trace_clock", 0644, d_tracer, NULL,
4272                           &trace_clock_fops);
4273
4274 #ifdef CONFIG_DYNAMIC_FTRACE
4275         trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4276                         &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
4277 #endif
4278 #ifdef CONFIG_SYSPROF_TRACER
4279         init_tracer_sysprof_debugfs(d_tracer);
4280 #endif
4281
4282         create_trace_options_dir();
4283
4284         for_each_tracing_cpu(cpu)
4285                 tracing_init_debugfs_percpu(cpu);
4286
4287         return 0;
4288 }
4289
4290 static int trace_panic_handler(struct notifier_block *this,
4291                                unsigned long event, void *unused)
4292 {
4293         if (ftrace_dump_on_oops)
4294                 ftrace_dump();
4295         return NOTIFY_OK;
4296 }
4297
4298 static struct notifier_block trace_panic_notifier = {
4299         .notifier_call  = trace_panic_handler,
4300         .next           = NULL,
4301         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
4302 };
4303
4304 static int trace_die_handler(struct notifier_block *self,
4305                              unsigned long val,
4306                              void *data)
4307 {
4308         switch (val) {
4309         case DIE_OOPS:
4310                 if (ftrace_dump_on_oops)
4311                         ftrace_dump();
4312                 break;
4313         default:
4314                 break;
4315         }
4316         return NOTIFY_OK;
4317 }
4318
4319 static struct notifier_block trace_die_notifier = {
4320         .notifier_call = trace_die_handler,
4321         .priority = 200
4322 };
4323
4324 /*
4325  * printk is set to max of 1024, we really don't need it that big.
4326  * Nothing should be printing 1000 characters anyway.
4327  */
4328 #define TRACE_MAX_PRINT         1000
4329
4330 /*
4331  * Define here KERN_TRACE so that we have one place to modify
4332  * it if we decide to change what log level the ftrace dump
4333  * should be at.
4334  */
4335 #define KERN_TRACE              KERN_EMERG
4336
4337 static void
4338 trace_printk_seq(struct trace_seq *s)
4339 {
4340         /* Probably should print a warning here. */
4341         if (s->len >= 1000)
4342                 s->len = 1000;
4343
4344         /* should be zero ended, but we are paranoid. */
4345         s->buffer[s->len] = 0;
4346
4347         printk(KERN_TRACE "%s", s->buffer);
4348
4349         trace_seq_init(s);
4350 }
4351
4352 static void __ftrace_dump(bool disable_tracing)
4353 {
4354         static arch_spinlock_t ftrace_dump_lock =
4355                 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
4356         /* use static because iter can be a bit big for the stack */
4357         static struct trace_iterator iter;
4358         unsigned int old_userobj;
4359         static int dump_ran;
4360         unsigned long flags;
4361         int cnt = 0, cpu;
4362
4363         /* only one dump */
4364         local_irq_save(flags);
4365         arch_spin_lock(&ftrace_dump_lock);
4366         if (dump_ran)
4367                 goto out;
4368
4369         dump_ran = 1;
4370
4371         tracing_off();
4372
4373         if (disable_tracing)
4374                 ftrace_kill();
4375
4376         for_each_tracing_cpu(cpu) {
4377                 atomic_inc(&global_trace.data[cpu]->disabled);
4378         }
4379
4380         old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4381
4382         /* don't look at user memory in panic mode */
4383         trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4384
4385         printk(KERN_TRACE "Dumping ftrace buffer:\n");
4386
4387         /* Simulate the iterator */
4388         iter.tr = &global_trace;
4389         iter.trace = current_trace;
4390         iter.cpu_file = TRACE_PIPE_ALL_CPU;
4391
4392         /*
4393          * We need to stop all tracing on all CPUS to read the
4394          * the next buffer. This is a bit expensive, but is
4395          * not done often. We fill all what we can read,
4396          * and then release the locks again.
4397          */
4398
4399         while (!trace_empty(&iter)) {
4400
4401                 if (!cnt)
4402                         printk(KERN_TRACE "---------------------------------\n");
4403
4404                 cnt++;
4405
4406                 /* reset all but tr, trace, and overruns */
4407                 memset(&iter.seq, 0,
4408                        sizeof(struct trace_iterator) -
4409                        offsetof(struct trace_iterator, seq));
4410                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
4411                 iter.pos = -1;
4412
4413                 if (find_next_entry_inc(&iter) != NULL) {
4414                         int ret;
4415
4416                         ret = print_trace_line(&iter);
4417                         if (ret != TRACE_TYPE_NO_CONSUME)
4418                                 trace_consume(&iter);
4419                 }
4420
4421                 trace_printk_seq(&iter.seq);
4422         }
4423
4424         if (!cnt)
4425                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
4426         else
4427                 printk(KERN_TRACE "---------------------------------\n");
4428
4429         /* Re-enable tracing if requested */
4430         if (!disable_tracing) {
4431                 trace_flags |= old_userobj;
4432
4433                 for_each_tracing_cpu(cpu) {
4434                         atomic_dec(&global_trace.data[cpu]->disabled);
4435                 }
4436                 tracing_on();
4437         }
4438
4439  out:
4440         arch_spin_unlock(&ftrace_dump_lock);
4441         local_irq_restore(flags);
4442 }
4443
4444 /* By default: disable tracing after the dump */
4445 void ftrace_dump(void)
4446 {
4447         __ftrace_dump(true);
4448 }
4449
4450 __init static int tracer_alloc_buffers(void)
4451 {
4452         int ring_buf_size;
4453         int i;
4454         int ret = -ENOMEM;
4455
4456         if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4457                 goto out;
4458
4459         if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4460                 goto out_free_buffer_mask;
4461
4462         /* To save memory, keep the ring buffer size to its minimum */
4463         if (ring_buffer_expanded)
4464                 ring_buf_size = trace_buf_size;
4465         else
4466                 ring_buf_size = 1;
4467
4468         cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4469         cpumask_copy(tracing_cpumask, cpu_all_mask);
4470
4471         /* TODO: make the number of buffers hot pluggable with CPUS */
4472         global_trace.buffer = ring_buffer_alloc(ring_buf_size,
4473                                                    TRACE_BUFFER_FLAGS);
4474         if (!global_trace.buffer) {
4475                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4476                 WARN_ON(1);
4477                 goto out_free_cpumask;
4478         }
4479         global_trace.entries = ring_buffer_size(global_trace.buffer);
4480
4481
4482 #ifdef CONFIG_TRACER_MAX_TRACE
4483         max_tr.buffer = ring_buffer_alloc(ring_buf_size,
4484                                              TRACE_BUFFER_FLAGS);
4485         if (!max_tr.buffer) {
4486                 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4487                 WARN_ON(1);
4488                 ring_buffer_free(global_trace.buffer);
4489                 goto out_free_cpumask;
4490         }
4491         max_tr.entries = ring_buffer_size(max_tr.buffer);
4492         WARN_ON(max_tr.entries != global_trace.entries);
4493 #endif
4494
4495         /* Allocate the first page for all buffers */
4496         for_each_tracing_cpu(i) {
4497                 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
4498                 max_tr.data[i] = &per_cpu(max_tr_data, i);
4499         }
4500
4501         trace_init_cmdlines();
4502
4503         register_tracer(&nop_trace);
4504         current_trace = &nop_trace;
4505 #ifdef CONFIG_BOOT_TRACER
4506         register_tracer(&boot_tracer);
4507 #endif
4508         /* All seems OK, enable tracing */
4509         tracing_disabled = 0;
4510
4511         atomic_notifier_chain_register(&panic_notifier_list,
4512                                        &trace_panic_notifier);
4513
4514         register_die_notifier(&trace_die_notifier);
4515
4516         return 0;
4517
4518 out_free_cpumask:
4519         free_cpumask_var(tracing_cpumask);
4520 out_free_buffer_mask:
4521         free_cpumask_var(tracing_buffer_mask);
4522 out:
4523         return ret;
4524 }
4525
4526 __init static int clear_boot_tracer(void)
4527 {
4528         /*
4529          * The default tracer at boot buffer is an init section.
4530          * This function is called in lateinit. If we did not
4531          * find the boot tracer, then clear it out, to prevent
4532          * later registration from accessing the buffer that is
4533          * about to be freed.
4534          */
4535         if (!default_bootup_tracer)
4536                 return 0;
4537
4538         printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4539                default_bootup_tracer);
4540         default_bootup_tracer = NULL;
4541
4542         return 0;
4543 }
4544
4545 early_initcall(tracer_alloc_buffers);
4546 fs_initcall(tracer_init_debugfs);
4547 late_initcall(clear_boot_tracer);