tracing: make trace_seq_reset global and rename to trace_seq_init
[linux-2.6.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 William Lee Irwin III
13  */
14 #include <linux/utsrelease.h>
15 #include <linux/kallsyms.h>
16 #include <linux/seq_file.h>
17 #include <linux/notifier.h>
18 #include <linux/debugfs.h>
19 #include <linux/pagemap.h>
20 #include <linux/hardirq.h>
21 #include <linux/linkage.h>
22 #include <linux/uaccess.h>
23 #include <linux/ftrace.h>
24 #include <linux/module.h>
25 #include <linux/percpu.h>
26 #include <linux/kdebug.h>
27 #include <linux/ctype.h>
28 #include <linux/init.h>
29 #include <linux/poll.h>
30 #include <linux/gfp.h>
31 #include <linux/fs.h>
32 #include <linux/kprobes.h>
33 #include <linux/writeback.h>
34 #include <linux/splice.h>
35
36 #include <linux/stacktrace.h>
37 #include <linux/ring_buffer.h>
38 #include <linux/irqflags.h>
39
40 #include "trace.h"
41 #include "trace_output.h"
42
43 #define TRACE_BUFFER_FLAGS      (RB_FL_OVERWRITE)
44
45 unsigned long __read_mostly     tracing_max_latency;
46 unsigned long __read_mostly     tracing_thresh;
47
48 /*
49  * We need to change this state when a selftest is running.
50  * A selftest will lurk into the ring-buffer to count the
51  * entries inserted during the selftest although some concurrent
52  * insertions into the ring-buffer such as ftrace_printk could occurred
53  * at the same time, giving false positive or negative results.
54  */
55 static bool __read_mostly tracing_selftest_running;
56
57 /*
58  * If a tracer is running, we do not want to run SELFTEST.
59  */
60 static bool __read_mostly tracing_selftest_disabled;
61
62 /* For tracers that don't implement custom flags */
63 static struct tracer_opt dummy_tracer_opt[] = {
64         { }
65 };
66
67 static struct tracer_flags dummy_tracer_flags = {
68         .val = 0,
69         .opts = dummy_tracer_opt
70 };
71
72 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
73 {
74         return 0;
75 }
76
77 /*
78  * Kill all tracing for good (never come back).
79  * It is initialized to 1 but will turn to zero if the initialization
80  * of the tracer is successful. But that is the only place that sets
81  * this back to zero.
82  */
83 static int tracing_disabled = 1;
84
85 static DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
86
87 static inline void ftrace_disable_cpu(void)
88 {
89         preempt_disable();
90         local_inc(&__get_cpu_var(ftrace_cpu_disabled));
91 }
92
93 static inline void ftrace_enable_cpu(void)
94 {
95         local_dec(&__get_cpu_var(ftrace_cpu_disabled));
96         preempt_enable();
97 }
98
99 static cpumask_var_t __read_mostly      tracing_buffer_mask;
100
101 /* Define which cpu buffers are currently read in trace_pipe */
102 static cpumask_var_t                    tracing_reader_cpumask;
103
104 #define for_each_tracing_cpu(cpu)       \
105         for_each_cpu(cpu, tracing_buffer_mask)
106
107 /*
108  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
109  *
110  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
111  * is set, then ftrace_dump is called. This will output the contents
112  * of the ftrace buffers to the console.  This is very useful for
113  * capturing traces that lead to crashes and outputing it to a
114  * serial console.
115  *
116  * It is default off, but you can enable it with either specifying
117  * "ftrace_dump_on_oops" in the kernel command line, or setting
118  * /proc/sys/kernel/ftrace_dump_on_oops to true.
119  */
120 int ftrace_dump_on_oops;
121
122 static int tracing_set_tracer(const char *buf);
123
124 #define BOOTUP_TRACER_SIZE              100
125 static char bootup_tracer_buf[BOOTUP_TRACER_SIZE] __initdata;
126 static char *default_bootup_tracer;
127
128 static int __init set_ftrace(char *str)
129 {
130         strncpy(bootup_tracer_buf, str, BOOTUP_TRACER_SIZE);
131         default_bootup_tracer = bootup_tracer_buf;
132         return 1;
133 }
134 __setup("ftrace=", set_ftrace);
135
136 static int __init set_ftrace_dump_on_oops(char *str)
137 {
138         ftrace_dump_on_oops = 1;
139         return 1;
140 }
141 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
142
143 long
144 ns2usecs(cycle_t nsec)
145 {
146         nsec += 500;
147         do_div(nsec, 1000);
148         return nsec;
149 }
150
151 cycle_t ftrace_now(int cpu)
152 {
153         u64 ts = ring_buffer_time_stamp(cpu);
154         ring_buffer_normalize_time_stamp(cpu, &ts);
155         return ts;
156 }
157
158 /*
159  * The global_trace is the descriptor that holds the tracing
160  * buffers for the live tracing. For each CPU, it contains
161  * a link list of pages that will store trace entries. The
162  * page descriptor of the pages in the memory is used to hold
163  * the link list by linking the lru item in the page descriptor
164  * to each of the pages in the buffer per CPU.
165  *
166  * For each active CPU there is a data field that holds the
167  * pages for the buffer for that CPU. Each CPU has the same number
168  * of pages allocated for its buffer.
169  */
170 static struct trace_array       global_trace;
171
172 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
173
174 /*
175  * The max_tr is used to snapshot the global_trace when a maximum
176  * latency is reached. Some tracers will use this to store a maximum
177  * trace while it continues examining live traces.
178  *
179  * The buffers for the max_tr are set up the same as the global_trace.
180  * When a snapshot is taken, the link list of the max_tr is swapped
181  * with the link list of the global_trace and the buffers are reset for
182  * the global_trace so the tracing can continue.
183  */
184 static struct trace_array       max_tr;
185
186 static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
187
188 /* tracer_enabled is used to toggle activation of a tracer */
189 static int                      tracer_enabled = 1;
190
191 /**
192  * tracing_is_enabled - return tracer_enabled status
193  *
194  * This function is used by other tracers to know the status
195  * of the tracer_enabled flag.  Tracers may use this function
196  * to know if it should enable their features when starting
197  * up. See irqsoff tracer for an example (start_irqsoff_tracer).
198  */
199 int tracing_is_enabled(void)
200 {
201         return tracer_enabled;
202 }
203
204 /*
205  * trace_buf_size is the size in bytes that is allocated
206  * for a buffer. Note, the number of bytes is always rounded
207  * to page size.
208  *
209  * This number is purposely set to a low number of 16384.
210  * If the dump on oops happens, it will be much appreciated
211  * to not have to wait for all that output. Anyway this can be
212  * boot time and run time configurable.
213  */
214 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
215
216 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
217
218 /* trace_types holds a link list of available tracers. */
219 static struct tracer            *trace_types __read_mostly;
220
221 /* current_trace points to the tracer that is currently active */
222 static struct tracer            *current_trace __read_mostly;
223
224 /*
225  * max_tracer_type_len is used to simplify the allocating of
226  * buffers to read userspace tracer names. We keep track of
227  * the longest tracer name registered.
228  */
229 static int                      max_tracer_type_len;
230
231 /*
232  * trace_types_lock is used to protect the trace_types list.
233  * This lock is also used to keep user access serialized.
234  * Accesses from userspace will grab this lock while userspace
235  * activities happen inside the kernel.
236  */
237 static DEFINE_MUTEX(trace_types_lock);
238
239 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
240 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
241
242 /* trace_flags holds trace_options default values */
243 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
244         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO;
245
246 /**
247  * trace_wake_up - wake up tasks waiting for trace input
248  *
249  * Simply wakes up any task that is blocked on the trace_wait
250  * queue. These is used with trace_poll for tasks polling the trace.
251  */
252 void trace_wake_up(void)
253 {
254         /*
255          * The runqueue_is_locked() can fail, but this is the best we
256          * have for now:
257          */
258         if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
259                 wake_up(&trace_wait);
260 }
261
262 static int __init set_buf_size(char *str)
263 {
264         unsigned long buf_size;
265         int ret;
266
267         if (!str)
268                 return 0;
269         ret = strict_strtoul(str, 0, &buf_size);
270         /* nr_entries can not be zero */
271         if (ret < 0 || buf_size == 0)
272                 return 0;
273         trace_buf_size = buf_size;
274         return 1;
275 }
276 __setup("trace_buf_size=", set_buf_size);
277
278 unsigned long nsecs_to_usecs(unsigned long nsecs)
279 {
280         return nsecs / 1000;
281 }
282
283 /* These must match the bit postions in trace_iterator_flags */
284 static const char *trace_options[] = {
285         "print-parent",
286         "sym-offset",
287         "sym-addr",
288         "verbose",
289         "raw",
290         "hex",
291         "bin",
292         "block",
293         "stacktrace",
294         "sched-tree",
295         "ftrace_printk",
296         "ftrace_preempt",
297         "branch",
298         "annotate",
299         "userstacktrace",
300         "sym-userobj",
301         "printk-msg-only",
302         "context-info",
303         NULL
304 };
305
306 /*
307  * ftrace_max_lock is used to protect the swapping of buffers
308  * when taking a max snapshot. The buffers themselves are
309  * protected by per_cpu spinlocks. But the action of the swap
310  * needs its own lock.
311  *
312  * This is defined as a raw_spinlock_t in order to help
313  * with performance when lockdep debugging is enabled.
314  */
315 static raw_spinlock_t ftrace_max_lock =
316         (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
317
318 /*
319  * Copy the new maximum trace into the separate maximum-trace
320  * structure. (this way the maximum trace is permanently saved,
321  * for later retrieval via /debugfs/tracing/latency_trace)
322  */
323 static void
324 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
325 {
326         struct trace_array_cpu *data = tr->data[cpu];
327
328         max_tr.cpu = cpu;
329         max_tr.time_start = data->preempt_timestamp;
330
331         data = max_tr.data[cpu];
332         data->saved_latency = tracing_max_latency;
333
334         memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
335         data->pid = tsk->pid;
336         data->uid = task_uid(tsk);
337         data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
338         data->policy = tsk->policy;
339         data->rt_priority = tsk->rt_priority;
340
341         /* record this tasks comm */
342         tracing_record_cmdline(tsk);
343 }
344
345 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
346 {
347         int len;
348         int ret;
349
350         if (s->len <= s->readpos)
351                 return -EBUSY;
352
353         len = s->len - s->readpos;
354         if (cnt > len)
355                 cnt = len;
356         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
357         if (ret)
358                 return -EFAULT;
359
360         s->readpos += len;
361         return cnt;
362 }
363
364 ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
365 {
366         int len;
367         void *ret;
368
369         if (s->len <= s->readpos)
370                 return -EBUSY;
371
372         len = s->len - s->readpos;
373         if (cnt > len)
374                 cnt = len;
375         ret = memcpy(buf, s->buffer + s->readpos, cnt);
376         if (!ret)
377                 return -EFAULT;
378
379         s->readpos += len;
380         return cnt;
381 }
382
383 static void
384 trace_print_seq(struct seq_file *m, struct trace_seq *s)
385 {
386         int len = s->len >= PAGE_SIZE ? PAGE_SIZE - 1 : s->len;
387
388         s->buffer[len] = 0;
389         seq_puts(m, s->buffer);
390
391         trace_seq_init(s);
392 }
393
394 /**
395  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
396  * @tr: tracer
397  * @tsk: the task with the latency
398  * @cpu: The cpu that initiated the trace.
399  *
400  * Flip the buffers between the @tr and the max_tr and record information
401  * about which task was the cause of this latency.
402  */
403 void
404 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
405 {
406         struct ring_buffer *buf = tr->buffer;
407
408         WARN_ON_ONCE(!irqs_disabled());
409         __raw_spin_lock(&ftrace_max_lock);
410
411         tr->buffer = max_tr.buffer;
412         max_tr.buffer = buf;
413
414         ftrace_disable_cpu();
415         ring_buffer_reset(tr->buffer);
416         ftrace_enable_cpu();
417
418         __update_max_tr(tr, tsk, cpu);
419         __raw_spin_unlock(&ftrace_max_lock);
420 }
421
422 /**
423  * update_max_tr_single - only copy one trace over, and reset the rest
424  * @tr - tracer
425  * @tsk - task with the latency
426  * @cpu - the cpu of the buffer to copy.
427  *
428  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
429  */
430 void
431 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
432 {
433         int ret;
434
435         WARN_ON_ONCE(!irqs_disabled());
436         __raw_spin_lock(&ftrace_max_lock);
437
438         ftrace_disable_cpu();
439
440         ring_buffer_reset(max_tr.buffer);
441         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
442
443         ftrace_enable_cpu();
444
445         WARN_ON_ONCE(ret && ret != -EAGAIN);
446
447         __update_max_tr(tr, tsk, cpu);
448         __raw_spin_unlock(&ftrace_max_lock);
449 }
450
451 /**
452  * register_tracer - register a tracer with the ftrace system.
453  * @type - the plugin for the tracer
454  *
455  * Register a new plugin tracer.
456  */
457 int register_tracer(struct tracer *type)
458 __releases(kernel_lock)
459 __acquires(kernel_lock)
460 {
461         struct tracer *t;
462         int len;
463         int ret = 0;
464
465         if (!type->name) {
466                 pr_info("Tracer must have a name\n");
467                 return -1;
468         }
469
470         /*
471          * When this gets called we hold the BKL which means that
472          * preemption is disabled. Various trace selftests however
473          * need to disable and enable preemption for successful tests.
474          * So we drop the BKL here and grab it after the tests again.
475          */
476         unlock_kernel();
477         mutex_lock(&trace_types_lock);
478
479         tracing_selftest_running = true;
480
481         for (t = trace_types; t; t = t->next) {
482                 if (strcmp(type->name, t->name) == 0) {
483                         /* already found */
484                         pr_info("Trace %s already registered\n",
485                                 type->name);
486                         ret = -1;
487                         goto out;
488                 }
489         }
490
491         if (!type->set_flag)
492                 type->set_flag = &dummy_set_flag;
493         if (!type->flags)
494                 type->flags = &dummy_tracer_flags;
495         else
496                 if (!type->flags->opts)
497                         type->flags->opts = dummy_tracer_opt;
498         if (!type->wait_pipe)
499                 type->wait_pipe = default_wait_pipe;
500
501
502 #ifdef CONFIG_FTRACE_STARTUP_TEST
503         if (type->selftest && !tracing_selftest_disabled) {
504                 struct tracer *saved_tracer = current_trace;
505                 struct trace_array *tr = &global_trace;
506                 int i;
507
508                 /*
509                  * Run a selftest on this tracer.
510                  * Here we reset the trace buffer, and set the current
511                  * tracer to be this tracer. The tracer can then run some
512                  * internal tracing to verify that everything is in order.
513                  * If we fail, we do not register this tracer.
514                  */
515                 for_each_tracing_cpu(i)
516                         tracing_reset(tr, i);
517
518                 current_trace = type;
519                 /* the test is responsible for initializing and enabling */
520                 pr_info("Testing tracer %s: ", type->name);
521                 ret = type->selftest(type, tr);
522                 /* the test is responsible for resetting too */
523                 current_trace = saved_tracer;
524                 if (ret) {
525                         printk(KERN_CONT "FAILED!\n");
526                         goto out;
527                 }
528                 /* Only reset on passing, to avoid touching corrupted buffers */
529                 for_each_tracing_cpu(i)
530                         tracing_reset(tr, i);
531
532                 printk(KERN_CONT "PASSED\n");
533         }
534 #endif
535
536         type->next = trace_types;
537         trace_types = type;
538         len = strlen(type->name);
539         if (len > max_tracer_type_len)
540                 max_tracer_type_len = len;
541
542  out:
543         tracing_selftest_running = false;
544         mutex_unlock(&trace_types_lock);
545
546         if (ret || !default_bootup_tracer)
547                 goto out_unlock;
548
549         if (strncmp(default_bootup_tracer, type->name, BOOTUP_TRACER_SIZE))
550                 goto out_unlock;
551
552         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
553         /* Do we want this tracer to start on bootup? */
554         tracing_set_tracer(type->name);
555         default_bootup_tracer = NULL;
556         /* disable other selftests, since this will break it. */
557         tracing_selftest_disabled = 1;
558 #ifdef CONFIG_FTRACE_STARTUP_TEST
559         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
560                type->name);
561 #endif
562
563  out_unlock:
564         lock_kernel();
565         return ret;
566 }
567
568 void unregister_tracer(struct tracer *type)
569 {
570         struct tracer **t;
571         int len;
572
573         mutex_lock(&trace_types_lock);
574         for (t = &trace_types; *t; t = &(*t)->next) {
575                 if (*t == type)
576                         goto found;
577         }
578         pr_info("Trace %s not registered\n", type->name);
579         goto out;
580
581  found:
582         *t = (*t)->next;
583
584         if (type == current_trace && tracer_enabled) {
585                 tracer_enabled = 0;
586                 tracing_stop();
587                 if (current_trace->stop)
588                         current_trace->stop(&global_trace);
589                 current_trace = &nop_trace;
590         }
591
592         if (strlen(type->name) != max_tracer_type_len)
593                 goto out;
594
595         max_tracer_type_len = 0;
596         for (t = &trace_types; *t; t = &(*t)->next) {
597                 len = strlen((*t)->name);
598                 if (len > max_tracer_type_len)
599                         max_tracer_type_len = len;
600         }
601  out:
602         mutex_unlock(&trace_types_lock);
603 }
604
605 void tracing_reset(struct trace_array *tr, int cpu)
606 {
607         ftrace_disable_cpu();
608         ring_buffer_reset_cpu(tr->buffer, cpu);
609         ftrace_enable_cpu();
610 }
611
612 void tracing_reset_online_cpus(struct trace_array *tr)
613 {
614         int cpu;
615
616         tr->time_start = ftrace_now(tr->cpu);
617
618         for_each_online_cpu(cpu)
619                 tracing_reset(tr, cpu);
620 }
621
622 #define SAVED_CMDLINES 128
623 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
624 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
625 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
626 static int cmdline_idx;
627 static DEFINE_SPINLOCK(trace_cmdline_lock);
628
629 /* temporary disable recording */
630 static atomic_t trace_record_cmdline_disabled __read_mostly;
631
632 static void trace_init_cmdlines(void)
633 {
634         memset(&map_pid_to_cmdline, -1, sizeof(map_pid_to_cmdline));
635         memset(&map_cmdline_to_pid, -1, sizeof(map_cmdline_to_pid));
636         cmdline_idx = 0;
637 }
638
639 static int trace_stop_count;
640 static DEFINE_SPINLOCK(tracing_start_lock);
641
642 /**
643  * ftrace_off_permanent - disable all ftrace code permanently
644  *
645  * This should only be called when a serious anomally has
646  * been detected.  This will turn off the function tracing,
647  * ring buffers, and other tracing utilites. It takes no
648  * locks and can be called from any context.
649  */
650 void ftrace_off_permanent(void)
651 {
652         tracing_disabled = 1;
653         ftrace_stop();
654         tracing_off_permanent();
655 }
656
657 /**
658  * tracing_start - quick start of the tracer
659  *
660  * If tracing is enabled but was stopped by tracing_stop,
661  * this will start the tracer back up.
662  */
663 void tracing_start(void)
664 {
665         struct ring_buffer *buffer;
666         unsigned long flags;
667
668         if (tracing_disabled)
669                 return;
670
671         spin_lock_irqsave(&tracing_start_lock, flags);
672         if (--trace_stop_count) {
673                 if (trace_stop_count < 0) {
674                         /* Someone screwed up their debugging */
675                         WARN_ON_ONCE(1);
676                         trace_stop_count = 0;
677                 }
678                 goto out;
679         }
680
681
682         buffer = global_trace.buffer;
683         if (buffer)
684                 ring_buffer_record_enable(buffer);
685
686         buffer = max_tr.buffer;
687         if (buffer)
688                 ring_buffer_record_enable(buffer);
689
690         ftrace_start();
691  out:
692         spin_unlock_irqrestore(&tracing_start_lock, flags);
693 }
694
695 /**
696  * tracing_stop - quick stop of the tracer
697  *
698  * Light weight way to stop tracing. Use in conjunction with
699  * tracing_start.
700  */
701 void tracing_stop(void)
702 {
703         struct ring_buffer *buffer;
704         unsigned long flags;
705
706         ftrace_stop();
707         spin_lock_irqsave(&tracing_start_lock, flags);
708         if (trace_stop_count++)
709                 goto out;
710
711         buffer = global_trace.buffer;
712         if (buffer)
713                 ring_buffer_record_disable(buffer);
714
715         buffer = max_tr.buffer;
716         if (buffer)
717                 ring_buffer_record_disable(buffer);
718
719  out:
720         spin_unlock_irqrestore(&tracing_start_lock, flags);
721 }
722
723 void trace_stop_cmdline_recording(void);
724
725 static void trace_save_cmdline(struct task_struct *tsk)
726 {
727         unsigned map;
728         unsigned idx;
729
730         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
731                 return;
732
733         /*
734          * It's not the end of the world if we don't get
735          * the lock, but we also don't want to spin
736          * nor do we want to disable interrupts,
737          * so if we miss here, then better luck next time.
738          */
739         if (!spin_trylock(&trace_cmdline_lock))
740                 return;
741
742         idx = map_pid_to_cmdline[tsk->pid];
743         if (idx >= SAVED_CMDLINES) {
744                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
745
746                 map = map_cmdline_to_pid[idx];
747                 if (map <= PID_MAX_DEFAULT)
748                         map_pid_to_cmdline[map] = (unsigned)-1;
749
750                 map_pid_to_cmdline[tsk->pid] = idx;
751
752                 cmdline_idx = idx;
753         }
754
755         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
756
757         spin_unlock(&trace_cmdline_lock);
758 }
759
760 char *trace_find_cmdline(int pid)
761 {
762         char *cmdline = "<...>";
763         unsigned map;
764
765         if (!pid)
766                 return "<idle>";
767
768         if (pid > PID_MAX_DEFAULT)
769                 goto out;
770
771         map = map_pid_to_cmdline[pid];
772         if (map >= SAVED_CMDLINES)
773                 goto out;
774
775         cmdline = saved_cmdlines[map];
776
777  out:
778         return cmdline;
779 }
780
781 void tracing_record_cmdline(struct task_struct *tsk)
782 {
783         if (atomic_read(&trace_record_cmdline_disabled))
784                 return;
785
786         trace_save_cmdline(tsk);
787 }
788
789 void
790 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
791                              int pc)
792 {
793         struct task_struct *tsk = current;
794
795         entry->preempt_count            = pc & 0xff;
796         entry->pid                      = (tsk) ? tsk->pid : 0;
797         entry->tgid                     = (tsk) ? tsk->tgid : 0;
798         entry->flags =
799 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
800                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
801 #else
802                 TRACE_FLAG_IRQS_NOSUPPORT |
803 #endif
804                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
805                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
806                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
807 }
808
809 struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
810                                                     unsigned char type,
811                                                     unsigned long len,
812                                                     unsigned long flags, int pc)
813 {
814         struct ring_buffer_event *event;
815
816         event = ring_buffer_lock_reserve(tr->buffer, len);
817         if (event != NULL) {
818                 struct trace_entry *ent = ring_buffer_event_data(event);
819
820                 tracing_generic_entry_update(ent, flags, pc);
821                 ent->type = type;
822         }
823
824         return event;
825 }
826 static void ftrace_trace_stack(struct trace_array *tr,
827                                unsigned long flags, int skip, int pc);
828 static void ftrace_trace_userstack(struct trace_array *tr,
829                                    unsigned long flags, int pc);
830
831 void trace_buffer_unlock_commit(struct trace_array *tr,
832                                 struct ring_buffer_event *event,
833                                 unsigned long flags, int pc)
834 {
835         ring_buffer_unlock_commit(tr->buffer, event);
836
837         ftrace_trace_stack(tr, flags, 6, pc);
838         ftrace_trace_userstack(tr, flags, pc);
839         trace_wake_up();
840 }
841
842 struct ring_buffer_event *
843 trace_current_buffer_lock_reserve(unsigned char type, unsigned long len,
844                                   unsigned long flags, int pc)
845 {
846         return trace_buffer_lock_reserve(&global_trace,
847                                          type, len, flags, pc);
848 }
849
850 void trace_current_buffer_unlock_commit(struct ring_buffer_event *event,
851                                         unsigned long flags, int pc)
852 {
853         return trace_buffer_unlock_commit(&global_trace, event, flags, pc);
854 }
855
856 void
857 trace_function(struct trace_array *tr,
858                unsigned long ip, unsigned long parent_ip, unsigned long flags,
859                int pc)
860 {
861         struct ring_buffer_event *event;
862         struct ftrace_entry *entry;
863
864         /* If we are reading the ring buffer, don't trace */
865         if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
866                 return;
867
868         event = trace_buffer_lock_reserve(tr, TRACE_FN, sizeof(*entry),
869                                           flags, pc);
870         if (!event)
871                 return;
872         entry   = ring_buffer_event_data(event);
873         entry->ip                       = ip;
874         entry->parent_ip                = parent_ip;
875         ring_buffer_unlock_commit(tr->buffer, event);
876 }
877
878 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
879 static void __trace_graph_entry(struct trace_array *tr,
880                                 struct ftrace_graph_ent *trace,
881                                 unsigned long flags,
882                                 int pc)
883 {
884         struct ring_buffer_event *event;
885         struct ftrace_graph_ent_entry *entry;
886
887         if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
888                 return;
889
890         event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_ENT,
891                                           sizeof(*entry), flags, pc);
892         if (!event)
893                 return;
894         entry   = ring_buffer_event_data(event);
895         entry->graph_ent                        = *trace;
896         ring_buffer_unlock_commit(global_trace.buffer, event);
897 }
898
899 static void __trace_graph_return(struct trace_array *tr,
900                                 struct ftrace_graph_ret *trace,
901                                 unsigned long flags,
902                                 int pc)
903 {
904         struct ring_buffer_event *event;
905         struct ftrace_graph_ret_entry *entry;
906
907         if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
908                 return;
909
910         event = trace_buffer_lock_reserve(&global_trace, TRACE_GRAPH_RET,
911                                           sizeof(*entry), flags, pc);
912         if (!event)
913                 return;
914         entry   = ring_buffer_event_data(event);
915         entry->ret                              = *trace;
916         ring_buffer_unlock_commit(global_trace.buffer, event);
917 }
918 #endif
919
920 void
921 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
922        unsigned long ip, unsigned long parent_ip, unsigned long flags,
923        int pc)
924 {
925         if (likely(!atomic_read(&data->disabled)))
926                 trace_function(tr, ip, parent_ip, flags, pc);
927 }
928
929 static void __ftrace_trace_stack(struct trace_array *tr,
930                                  unsigned long flags,
931                                  int skip, int pc)
932 {
933 #ifdef CONFIG_STACKTRACE
934         struct ring_buffer_event *event;
935         struct stack_entry *entry;
936         struct stack_trace trace;
937
938         event = trace_buffer_lock_reserve(tr, TRACE_STACK,
939                                           sizeof(*entry), flags, pc);
940         if (!event)
941                 return;
942         entry   = ring_buffer_event_data(event);
943         memset(&entry->caller, 0, sizeof(entry->caller));
944
945         trace.nr_entries        = 0;
946         trace.max_entries       = FTRACE_STACK_ENTRIES;
947         trace.skip              = skip;
948         trace.entries           = entry->caller;
949
950         save_stack_trace(&trace);
951         ring_buffer_unlock_commit(tr->buffer, event);
952 #endif
953 }
954
955 static void ftrace_trace_stack(struct trace_array *tr,
956                                unsigned long flags,
957                                int skip, int pc)
958 {
959         if (!(trace_flags & TRACE_ITER_STACKTRACE))
960                 return;
961
962         __ftrace_trace_stack(tr, flags, skip, pc);
963 }
964
965 void __trace_stack(struct trace_array *tr,
966                    unsigned long flags,
967                    int skip, int pc)
968 {
969         __ftrace_trace_stack(tr, flags, skip, pc);
970 }
971
972 static void ftrace_trace_userstack(struct trace_array *tr,
973                                    unsigned long flags, int pc)
974 {
975 #ifdef CONFIG_STACKTRACE
976         struct ring_buffer_event *event;
977         struct userstack_entry *entry;
978         struct stack_trace trace;
979
980         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
981                 return;
982
983         event = trace_buffer_lock_reserve(tr, TRACE_USER_STACK,
984                                           sizeof(*entry), flags, pc);
985         if (!event)
986                 return;
987         entry   = ring_buffer_event_data(event);
988
989         memset(&entry->caller, 0, sizeof(entry->caller));
990
991         trace.nr_entries        = 0;
992         trace.max_entries       = FTRACE_STACK_ENTRIES;
993         trace.skip              = 0;
994         trace.entries           = entry->caller;
995
996         save_stack_trace_user(&trace);
997         ring_buffer_unlock_commit(tr->buffer, event);
998 #endif
999 }
1000
1001 #ifdef UNUSED
1002 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1003 {
1004         ftrace_trace_userstack(tr, flags, preempt_count());
1005 }
1006 #endif /* UNUSED */
1007
1008 static void
1009 ftrace_trace_special(void *__tr,
1010                      unsigned long arg1, unsigned long arg2, unsigned long arg3,
1011                      int pc)
1012 {
1013         struct ring_buffer_event *event;
1014         struct trace_array *tr = __tr;
1015         struct special_entry *entry;
1016
1017         event = trace_buffer_lock_reserve(tr, TRACE_SPECIAL,
1018                                           sizeof(*entry), 0, pc);
1019         if (!event)
1020                 return;
1021         entry   = ring_buffer_event_data(event);
1022         entry->arg1                     = arg1;
1023         entry->arg2                     = arg2;
1024         entry->arg3                     = arg3;
1025         trace_buffer_unlock_commit(tr, event, 0, pc);
1026 }
1027
1028 void
1029 __trace_special(void *__tr, void *__data,
1030                 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1031 {
1032         ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
1033 }
1034
1035 void
1036 tracing_sched_switch_trace(struct trace_array *tr,
1037                            struct task_struct *prev,
1038                            struct task_struct *next,
1039                            unsigned long flags, int pc)
1040 {
1041         struct ring_buffer_event *event;
1042         struct ctx_switch_entry *entry;
1043
1044         event = trace_buffer_lock_reserve(tr, TRACE_CTX,
1045                                           sizeof(*entry), flags, pc);
1046         if (!event)
1047                 return;
1048         entry   = ring_buffer_event_data(event);
1049         entry->prev_pid                 = prev->pid;
1050         entry->prev_prio                = prev->prio;
1051         entry->prev_state               = prev->state;
1052         entry->next_pid                 = next->pid;
1053         entry->next_prio                = next->prio;
1054         entry->next_state               = next->state;
1055         entry->next_cpu = task_cpu(next);
1056         trace_buffer_unlock_commit(tr, event, flags, pc);
1057 }
1058
1059 void
1060 tracing_sched_wakeup_trace(struct trace_array *tr,
1061                            struct task_struct *wakee,
1062                            struct task_struct *curr,
1063                            unsigned long flags, int pc)
1064 {
1065         struct ring_buffer_event *event;
1066         struct ctx_switch_entry *entry;
1067
1068         event = trace_buffer_lock_reserve(tr, TRACE_WAKE,
1069                                           sizeof(*entry), flags, pc);
1070         if (!event)
1071                 return;
1072         entry   = ring_buffer_event_data(event);
1073         entry->prev_pid                 = curr->pid;
1074         entry->prev_prio                = curr->prio;
1075         entry->prev_state               = curr->state;
1076         entry->next_pid                 = wakee->pid;
1077         entry->next_prio                = wakee->prio;
1078         entry->next_state               = wakee->state;
1079         entry->next_cpu                 = task_cpu(wakee);
1080
1081         ring_buffer_unlock_commit(tr->buffer, event);
1082         ftrace_trace_stack(tr, flags, 6, pc);
1083         ftrace_trace_userstack(tr, flags, pc);
1084 }
1085
1086 void
1087 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1088 {
1089         struct trace_array *tr = &global_trace;
1090         struct trace_array_cpu *data;
1091         unsigned long flags;
1092         int cpu;
1093         int pc;
1094
1095         if (tracing_disabled)
1096                 return;
1097
1098         pc = preempt_count();
1099         local_irq_save(flags);
1100         cpu = raw_smp_processor_id();
1101         data = tr->data[cpu];
1102
1103         if (likely(atomic_inc_return(&data->disabled) == 1))
1104                 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
1105
1106         atomic_dec(&data->disabled);
1107         local_irq_restore(flags);
1108 }
1109
1110 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1111 int trace_graph_entry(struct ftrace_graph_ent *trace)
1112 {
1113         struct trace_array *tr = &global_trace;
1114         struct trace_array_cpu *data;
1115         unsigned long flags;
1116         long disabled;
1117         int cpu;
1118         int pc;
1119
1120         if (!ftrace_trace_task(current))
1121                 return 0;
1122
1123         if (!ftrace_graph_addr(trace->func))
1124                 return 0;
1125
1126         local_irq_save(flags);
1127         cpu = raw_smp_processor_id();
1128         data = tr->data[cpu];
1129         disabled = atomic_inc_return(&data->disabled);
1130         if (likely(disabled == 1)) {
1131                 pc = preempt_count();
1132                 __trace_graph_entry(tr, trace, flags, pc);
1133         }
1134         /* Only do the atomic if it is not already set */
1135         if (!test_tsk_trace_graph(current))
1136                 set_tsk_trace_graph(current);
1137         atomic_dec(&data->disabled);
1138         local_irq_restore(flags);
1139
1140         return 1;
1141 }
1142
1143 void trace_graph_return(struct ftrace_graph_ret *trace)
1144 {
1145         struct trace_array *tr = &global_trace;
1146         struct trace_array_cpu *data;
1147         unsigned long flags;
1148         long disabled;
1149         int cpu;
1150         int pc;
1151
1152         local_irq_save(flags);
1153         cpu = raw_smp_processor_id();
1154         data = tr->data[cpu];
1155         disabled = atomic_inc_return(&data->disabled);
1156         if (likely(disabled == 1)) {
1157                 pc = preempt_count();
1158                 __trace_graph_return(tr, trace, flags, pc);
1159         }
1160         if (!trace->depth)
1161                 clear_tsk_trace_graph(current);
1162         atomic_dec(&data->disabled);
1163         local_irq_restore(flags);
1164 }
1165 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1166
1167 enum trace_file_type {
1168         TRACE_FILE_LAT_FMT      = 1,
1169         TRACE_FILE_ANNOTATE     = 2,
1170 };
1171
1172 static void trace_iterator_increment(struct trace_iterator *iter)
1173 {
1174         /* Don't allow ftrace to trace into the ring buffers */
1175         ftrace_disable_cpu();
1176
1177         iter->idx++;
1178         if (iter->buffer_iter[iter->cpu])
1179                 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1180
1181         ftrace_enable_cpu();
1182 }
1183
1184 static struct trace_entry *
1185 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
1186 {
1187         struct ring_buffer_event *event;
1188         struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1189
1190         /* Don't allow ftrace to trace into the ring buffers */
1191         ftrace_disable_cpu();
1192
1193         if (buf_iter)
1194                 event = ring_buffer_iter_peek(buf_iter, ts);
1195         else
1196                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
1197
1198         ftrace_enable_cpu();
1199
1200         return event ? ring_buffer_event_data(event) : NULL;
1201 }
1202
1203 static struct trace_entry *
1204 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1205 {
1206         struct ring_buffer *buffer = iter->tr->buffer;
1207         struct trace_entry *ent, *next = NULL;
1208         int cpu_file = iter->cpu_file;
1209         u64 next_ts = 0, ts;
1210         int next_cpu = -1;
1211         int cpu;
1212
1213         /*
1214          * If we are in a per_cpu trace file, don't bother by iterating over
1215          * all cpu and peek directly.
1216          */
1217         if (cpu_file > TRACE_PIPE_ALL_CPU) {
1218                 if (ring_buffer_empty_cpu(buffer, cpu_file))
1219                         return NULL;
1220                 ent = peek_next_entry(iter, cpu_file, ent_ts);
1221                 if (ent_cpu)
1222                         *ent_cpu = cpu_file;
1223
1224                 return ent;
1225         }
1226
1227         for_each_tracing_cpu(cpu) {
1228
1229                 if (ring_buffer_empty_cpu(buffer, cpu))
1230                         continue;
1231
1232                 ent = peek_next_entry(iter, cpu, &ts);
1233
1234                 /*
1235                  * Pick the entry with the smallest timestamp:
1236                  */
1237                 if (ent && (!next || ts < next_ts)) {
1238                         next = ent;
1239                         next_cpu = cpu;
1240                         next_ts = ts;
1241                 }
1242         }
1243
1244         if (ent_cpu)
1245                 *ent_cpu = next_cpu;
1246
1247         if (ent_ts)
1248                 *ent_ts = next_ts;
1249
1250         return next;
1251 }
1252
1253 /* Find the next real entry, without updating the iterator itself */
1254 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1255                                           int *ent_cpu, u64 *ent_ts)
1256 {
1257         return __find_next_entry(iter, ent_cpu, ent_ts);
1258 }
1259
1260 /* Find the next real entry, and increment the iterator to the next entry */
1261 static void *find_next_entry_inc(struct trace_iterator *iter)
1262 {
1263         iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
1264
1265         if (iter->ent)
1266                 trace_iterator_increment(iter);
1267
1268         return iter->ent ? iter : NULL;
1269 }
1270
1271 static void trace_consume(struct trace_iterator *iter)
1272 {
1273         /* Don't allow ftrace to trace into the ring buffers */
1274         ftrace_disable_cpu();
1275         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1276         ftrace_enable_cpu();
1277 }
1278
1279 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1280 {
1281         struct trace_iterator *iter = m->private;
1282         int i = (int)*pos;
1283         void *ent;
1284
1285         (*pos)++;
1286
1287         /* can't go backwards */
1288         if (iter->idx > i)
1289                 return NULL;
1290
1291         if (iter->idx < 0)
1292                 ent = find_next_entry_inc(iter);
1293         else
1294                 ent = iter;
1295
1296         while (ent && iter->idx < i)
1297                 ent = find_next_entry_inc(iter);
1298
1299         iter->pos = *pos;
1300
1301         return ent;
1302 }
1303
1304 /*
1305  * No necessary locking here. The worst thing which can
1306  * happen is loosing events consumed at the same time
1307  * by a trace_pipe reader.
1308  * Other than that, we don't risk to crash the ring buffer
1309  * because it serializes the readers.
1310  *
1311  * The current tracer is copied to avoid a global locking
1312  * all around.
1313  */
1314 static void *s_start(struct seq_file *m, loff_t *pos)
1315 {
1316         struct trace_iterator *iter = m->private;
1317         static struct tracer *old_tracer;
1318         int cpu_file = iter->cpu_file;
1319         void *p = NULL;
1320         loff_t l = 0;
1321         int cpu;
1322
1323         /* copy the tracer to avoid using a global lock all around */
1324         mutex_lock(&trace_types_lock);
1325         if (unlikely(old_tracer != current_trace && current_trace)) {
1326                 old_tracer = current_trace;
1327                 *iter->trace = *current_trace;
1328         }
1329         mutex_unlock(&trace_types_lock);
1330
1331         atomic_inc(&trace_record_cmdline_disabled);
1332
1333         if (*pos != iter->pos) {
1334                 iter->ent = NULL;
1335                 iter->cpu = 0;
1336                 iter->idx = -1;
1337
1338                 ftrace_disable_cpu();
1339
1340                 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1341                         for_each_tracing_cpu(cpu)
1342                                 ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1343                 } else
1344                         ring_buffer_iter_reset(iter->buffer_iter[cpu_file]);
1345
1346
1347                 ftrace_enable_cpu();
1348
1349                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1350                         ;
1351
1352         } else {
1353                 l = *pos - 1;
1354                 p = s_next(m, p, &l);
1355         }
1356
1357         return p;
1358 }
1359
1360 static void s_stop(struct seq_file *m, void *p)
1361 {
1362         atomic_dec(&trace_record_cmdline_disabled);
1363 }
1364
1365 static void print_lat_help_header(struct seq_file *m)
1366 {
1367         seq_puts(m, "#                  _------=> CPU#            \n");
1368         seq_puts(m, "#                 / _-----=> irqs-off        \n");
1369         seq_puts(m, "#                | / _----=> need-resched    \n");
1370         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1371         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1372         seq_puts(m, "#                |||| /                      \n");
1373         seq_puts(m, "#                |||||     delay             \n");
1374         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1375         seq_puts(m, "#     \\   /      |||||   \\   |   /           \n");
1376 }
1377
1378 static void print_func_help_header(struct seq_file *m)
1379 {
1380         seq_puts(m, "#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n");
1381         seq_puts(m, "#              | |       |          |         |\n");
1382 }
1383
1384
1385 static void
1386 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1387 {
1388         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1389         struct trace_array *tr = iter->tr;
1390         struct trace_array_cpu *data = tr->data[tr->cpu];
1391         struct tracer *type = current_trace;
1392         unsigned long total;
1393         unsigned long entries;
1394         const char *name = "preemption";
1395
1396         if (type)
1397                 name = type->name;
1398
1399         entries = ring_buffer_entries(iter->tr->buffer);
1400         total = entries +
1401                 ring_buffer_overruns(iter->tr->buffer);
1402
1403         seq_printf(m, "%s latency trace v1.1.5 on %s\n",
1404                    name, UTS_RELEASE);
1405         seq_puts(m, "-----------------------------------"
1406                  "---------------------------------\n");
1407         seq_printf(m, " latency: %lu us, #%lu/%lu, CPU#%d |"
1408                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1409                    nsecs_to_usecs(data->saved_latency),
1410                    entries,
1411                    total,
1412                    tr->cpu,
1413 #if defined(CONFIG_PREEMPT_NONE)
1414                    "server",
1415 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1416                    "desktop",
1417 #elif defined(CONFIG_PREEMPT)
1418                    "preempt",
1419 #else
1420                    "unknown",
1421 #endif
1422                    /* These are reserved for later use */
1423                    0, 0, 0, 0);
1424 #ifdef CONFIG_SMP
1425         seq_printf(m, " #P:%d)\n", num_online_cpus());
1426 #else
1427         seq_puts(m, ")\n");
1428 #endif
1429         seq_puts(m, "    -----------------\n");
1430         seq_printf(m, "    | task: %.16s-%d "
1431                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1432                    data->comm, data->pid, data->uid, data->nice,
1433                    data->policy, data->rt_priority);
1434         seq_puts(m, "    -----------------\n");
1435
1436         if (data->critical_start) {
1437                 seq_puts(m, " => started at: ");
1438                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1439                 trace_print_seq(m, &iter->seq);
1440                 seq_puts(m, "\n => ended at:   ");
1441                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1442                 trace_print_seq(m, &iter->seq);
1443                 seq_puts(m, "\n");
1444         }
1445
1446         seq_puts(m, "\n");
1447 }
1448
1449 static void test_cpu_buff_start(struct trace_iterator *iter)
1450 {
1451         struct trace_seq *s = &iter->seq;
1452
1453         if (!(trace_flags & TRACE_ITER_ANNOTATE))
1454                 return;
1455
1456         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1457                 return;
1458
1459         if (cpumask_test_cpu(iter->cpu, iter->started))
1460                 return;
1461
1462         cpumask_set_cpu(iter->cpu, iter->started);
1463         trace_seq_printf(s, "##### CPU %u buffer started ####\n", iter->cpu);
1464 }
1465
1466 static enum print_line_t print_lat_fmt(struct trace_iterator *iter)
1467 {
1468         struct trace_seq *s = &iter->seq;
1469         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1470         struct trace_event *event;
1471         struct trace_entry *entry = iter->ent;
1472
1473         test_cpu_buff_start(iter);
1474
1475         event = ftrace_find_event(entry->type);
1476
1477         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1478                 if (!trace_print_lat_context(iter))
1479                         goto partial;
1480         }
1481
1482         if (event)
1483                 return event->latency_trace(iter, sym_flags);
1484
1485         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1486                 goto partial;
1487
1488         return TRACE_TYPE_HANDLED;
1489 partial:
1490         return TRACE_TYPE_PARTIAL_LINE;
1491 }
1492
1493 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1494 {
1495         struct trace_seq *s = &iter->seq;
1496         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1497         struct trace_entry *entry;
1498         struct trace_event *event;
1499
1500         entry = iter->ent;
1501
1502         test_cpu_buff_start(iter);
1503
1504         event = ftrace_find_event(entry->type);
1505
1506         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1507                 if (!trace_print_context(iter))
1508                         goto partial;
1509         }
1510
1511         if (event)
1512                 return event->trace(iter, sym_flags);
1513
1514         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1515                 goto partial;
1516
1517         return TRACE_TYPE_HANDLED;
1518 partial:
1519         return TRACE_TYPE_PARTIAL_LINE;
1520 }
1521
1522 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1523 {
1524         struct trace_seq *s = &iter->seq;
1525         struct trace_entry *entry;
1526         struct trace_event *event;
1527
1528         entry = iter->ent;
1529
1530         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1531                 if (!trace_seq_printf(s, "%d %d %llu ",
1532                                       entry->pid, iter->cpu, iter->ts))
1533                         goto partial;
1534         }
1535
1536         event = ftrace_find_event(entry->type);
1537         if (event)
1538                 return event->raw(iter, 0);
1539
1540         if (!trace_seq_printf(s, "%d ?\n", entry->type))
1541                 goto partial;
1542
1543         return TRACE_TYPE_HANDLED;
1544 partial:
1545         return TRACE_TYPE_PARTIAL_LINE;
1546 }
1547
1548 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1549 {
1550         struct trace_seq *s = &iter->seq;
1551         unsigned char newline = '\n';
1552         struct trace_entry *entry;
1553         struct trace_event *event;
1554
1555         entry = iter->ent;
1556
1557         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1558                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1559                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1560                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1561         }
1562
1563         event = ftrace_find_event(entry->type);
1564         if (event) {
1565                 enum print_line_t ret = event->hex(iter, 0);
1566                 if (ret != TRACE_TYPE_HANDLED)
1567                         return ret;
1568         }
1569
1570         SEQ_PUT_FIELD_RET(s, newline);
1571
1572         return TRACE_TYPE_HANDLED;
1573 }
1574
1575 static enum print_line_t print_printk_msg_only(struct trace_iterator *iter)
1576 {
1577         struct trace_seq *s = &iter->seq;
1578         struct trace_entry *entry = iter->ent;
1579         struct print_entry *field;
1580         int ret;
1581
1582         trace_assign_type(field, entry);
1583
1584         ret = trace_seq_printf(s, "%s", field->buf);
1585         if (!ret)
1586                 return TRACE_TYPE_PARTIAL_LINE;
1587
1588         return TRACE_TYPE_HANDLED;
1589 }
1590
1591 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1592 {
1593         struct trace_seq *s = &iter->seq;
1594         struct trace_entry *entry;
1595         struct trace_event *event;
1596
1597         entry = iter->ent;
1598
1599         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1600                 SEQ_PUT_FIELD_RET(s, entry->pid);
1601                 SEQ_PUT_FIELD_RET(s, iter->cpu);
1602                 SEQ_PUT_FIELD_RET(s, iter->ts);
1603         }
1604
1605         event = ftrace_find_event(entry->type);
1606         return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
1607 }
1608
1609 static int trace_empty(struct trace_iterator *iter)
1610 {
1611         int cpu;
1612
1613         for_each_tracing_cpu(cpu) {
1614                 if (iter->buffer_iter[cpu]) {
1615                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1616                                 return 0;
1617                 } else {
1618                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1619                                 return 0;
1620                 }
1621         }
1622
1623         return 1;
1624 }
1625
1626 static enum print_line_t print_trace_line(struct trace_iterator *iter)
1627 {
1628         enum print_line_t ret;
1629
1630         if (iter->trace && iter->trace->print_line) {
1631                 ret = iter->trace->print_line(iter);
1632                 if (ret != TRACE_TYPE_UNHANDLED)
1633                         return ret;
1634         }
1635
1636         if (iter->ent->type == TRACE_PRINT &&
1637                         trace_flags & TRACE_ITER_PRINTK &&
1638                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
1639                 return print_printk_msg_only(iter);
1640
1641         if (trace_flags & TRACE_ITER_BIN)
1642                 return print_bin_fmt(iter);
1643
1644         if (trace_flags & TRACE_ITER_HEX)
1645                 return print_hex_fmt(iter);
1646
1647         if (trace_flags & TRACE_ITER_RAW)
1648                 return print_raw_fmt(iter);
1649
1650         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
1651                 return print_lat_fmt(iter);
1652
1653         return print_trace_fmt(iter);
1654 }
1655
1656 static int s_show(struct seq_file *m, void *v)
1657 {
1658         struct trace_iterator *iter = v;
1659
1660         if (iter->ent == NULL) {
1661                 if (iter->tr) {
1662                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
1663                         seq_puts(m, "#\n");
1664                 }
1665                 if (iter->trace && iter->trace->print_header)
1666                         iter->trace->print_header(m);
1667                 else if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1668                         /* print nothing if the buffers are empty */
1669                         if (trace_empty(iter))
1670                                 return 0;
1671                         print_trace_header(m, iter);
1672                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1673                                 print_lat_help_header(m);
1674                 } else {
1675                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1676                                 print_func_help_header(m);
1677                 }
1678         } else {
1679                 print_trace_line(iter);
1680                 trace_print_seq(m, &iter->seq);
1681         }
1682
1683         return 0;
1684 }
1685
1686 static struct seq_operations tracer_seq_ops = {
1687         .start          = s_start,
1688         .next           = s_next,
1689         .stop           = s_stop,
1690         .show           = s_show,
1691 };
1692
1693 static struct trace_iterator *
1694 __tracing_open(struct inode *inode, struct file *file)
1695 {
1696         long cpu_file = (long) inode->i_private;
1697         void *fail_ret = ERR_PTR(-ENOMEM);
1698         struct trace_iterator *iter;
1699         struct seq_file *m;
1700         int cpu, ret;
1701
1702         if (tracing_disabled)
1703                 return ERR_PTR(-ENODEV);
1704
1705         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1706         if (!iter)
1707                 return ERR_PTR(-ENOMEM);
1708
1709         /*
1710          * We make a copy of the current tracer to avoid concurrent
1711          * changes on it while we are reading.
1712          */
1713         mutex_lock(&trace_types_lock);
1714         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
1715         if (!iter->trace)
1716                 goto fail;
1717
1718         if (current_trace)
1719                 *iter->trace = *current_trace;
1720
1721         if (current_trace && current_trace->print_max)
1722                 iter->tr = &max_tr;
1723         else
1724                 iter->tr = &global_trace;
1725         iter->pos = -1;
1726         mutex_init(&iter->mutex);
1727         iter->cpu_file = cpu_file;
1728
1729         /* Notify the tracer early; before we stop tracing. */
1730         if (iter->trace && iter->trace->open)
1731                 iter->trace->open(iter);
1732
1733         /* Annotate start of buffers if we had overruns */
1734         if (ring_buffer_overruns(iter->tr->buffer))
1735                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
1736
1737         if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
1738                 for_each_tracing_cpu(cpu) {
1739
1740                         iter->buffer_iter[cpu] =
1741                                 ring_buffer_read_start(iter->tr->buffer, cpu);
1742
1743                         if (!iter->buffer_iter[cpu])
1744                                 goto fail_buffer;
1745                 }
1746         } else {
1747                 cpu = iter->cpu_file;
1748                 iter->buffer_iter[cpu] =
1749                                 ring_buffer_read_start(iter->tr->buffer, cpu);
1750
1751                 if (!iter->buffer_iter[cpu])
1752                         goto fail;
1753         }
1754
1755         /* TODO stop tracer */
1756         ret = seq_open(file, &tracer_seq_ops);
1757         if (ret < 0) {
1758                 fail_ret = ERR_PTR(ret);
1759                 goto fail_buffer;
1760         }
1761
1762         m = file->private_data;
1763         m->private = iter;
1764
1765         /* stop the trace while dumping */
1766         tracing_stop();
1767
1768         mutex_unlock(&trace_types_lock);
1769
1770         return iter;
1771
1772  fail_buffer:
1773         for_each_tracing_cpu(cpu) {
1774                 if (iter->buffer_iter[cpu])
1775                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1776         }
1777  fail:
1778         mutex_unlock(&trace_types_lock);
1779         kfree(iter->trace);
1780         kfree(iter);
1781
1782         return fail_ret;
1783 }
1784
1785 int tracing_open_generic(struct inode *inode, struct file *filp)
1786 {
1787         if (tracing_disabled)
1788                 return -ENODEV;
1789
1790         filp->private_data = inode->i_private;
1791         return 0;
1792 }
1793
1794 static int tracing_release(struct inode *inode, struct file *file)
1795 {
1796         struct seq_file *m = (struct seq_file *)file->private_data;
1797         struct trace_iterator *iter = m->private;
1798         int cpu;
1799
1800         mutex_lock(&trace_types_lock);
1801         for_each_tracing_cpu(cpu) {
1802                 if (iter->buffer_iter[cpu])
1803                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1804         }
1805
1806         if (iter->trace && iter->trace->close)
1807                 iter->trace->close(iter);
1808
1809         /* reenable tracing if it was previously enabled */
1810         tracing_start();
1811         mutex_unlock(&trace_types_lock);
1812
1813         seq_release(inode, file);
1814         mutex_destroy(&iter->mutex);
1815         kfree(iter->trace);
1816         kfree(iter);
1817         return 0;
1818 }
1819
1820 static int tracing_open(struct inode *inode, struct file *file)
1821 {
1822         struct trace_iterator *iter;
1823         int ret = 0;
1824
1825         iter = __tracing_open(inode, file);
1826         if (IS_ERR(iter))
1827                 ret = PTR_ERR(iter);
1828
1829         return ret;
1830 }
1831
1832 static int tracing_lt_open(struct inode *inode, struct file *file)
1833 {
1834         struct trace_iterator *iter;
1835         int ret = 0;
1836
1837         iter = __tracing_open(inode, file);
1838
1839         if (IS_ERR(iter))
1840                 ret = PTR_ERR(iter);
1841         else
1842                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
1843
1844         return ret;
1845 }
1846
1847
1848 static void *
1849 t_next(struct seq_file *m, void *v, loff_t *pos)
1850 {
1851         struct tracer *t = m->private;
1852
1853         (*pos)++;
1854
1855         if (t)
1856                 t = t->next;
1857
1858         m->private = t;
1859
1860         return t;
1861 }
1862
1863 static void *t_start(struct seq_file *m, loff_t *pos)
1864 {
1865         struct tracer *t = m->private;
1866         loff_t l = 0;
1867
1868         mutex_lock(&trace_types_lock);
1869         for (; t && l < *pos; t = t_next(m, t, &l))
1870                 ;
1871
1872         return t;
1873 }
1874
1875 static void t_stop(struct seq_file *m, void *p)
1876 {
1877         mutex_unlock(&trace_types_lock);
1878 }
1879
1880 static int t_show(struct seq_file *m, void *v)
1881 {
1882         struct tracer *t = v;
1883
1884         if (!t)
1885                 return 0;
1886
1887         seq_printf(m, "%s", t->name);
1888         if (t->next)
1889                 seq_putc(m, ' ');
1890         else
1891                 seq_putc(m, '\n');
1892
1893         return 0;
1894 }
1895
1896 static struct seq_operations show_traces_seq_ops = {
1897         .start          = t_start,
1898         .next           = t_next,
1899         .stop           = t_stop,
1900         .show           = t_show,
1901 };
1902
1903 static int show_traces_open(struct inode *inode, struct file *file)
1904 {
1905         int ret;
1906
1907         if (tracing_disabled)
1908                 return -ENODEV;
1909
1910         ret = seq_open(file, &show_traces_seq_ops);
1911         if (!ret) {
1912                 struct seq_file *m = file->private_data;
1913                 m->private = trace_types;
1914         }
1915
1916         return ret;
1917 }
1918
1919 static struct file_operations tracing_fops = {
1920         .open           = tracing_open,
1921         .read           = seq_read,
1922         .llseek         = seq_lseek,
1923         .release        = tracing_release,
1924 };
1925
1926 static struct file_operations tracing_lt_fops = {
1927         .open           = tracing_lt_open,
1928         .read           = seq_read,
1929         .llseek         = seq_lseek,
1930         .release        = tracing_release,
1931 };
1932
1933 static struct file_operations show_traces_fops = {
1934         .open           = show_traces_open,
1935         .read           = seq_read,
1936         .release        = seq_release,
1937 };
1938
1939 /*
1940  * Only trace on a CPU if the bitmask is set:
1941  */
1942 static cpumask_var_t tracing_cpumask;
1943
1944 /*
1945  * The tracer itself will not take this lock, but still we want
1946  * to provide a consistent cpumask to user-space:
1947  */
1948 static DEFINE_MUTEX(tracing_cpumask_update_lock);
1949
1950 /*
1951  * Temporary storage for the character representation of the
1952  * CPU bitmask (and one more byte for the newline):
1953  */
1954 static char mask_str[NR_CPUS + 1];
1955
1956 static ssize_t
1957 tracing_cpumask_read(struct file *filp, char __user *ubuf,
1958                      size_t count, loff_t *ppos)
1959 {
1960         int len;
1961
1962         mutex_lock(&tracing_cpumask_update_lock);
1963
1964         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
1965         if (count - len < 2) {
1966                 count = -EINVAL;
1967                 goto out_err;
1968         }
1969         len += sprintf(mask_str + len, "\n");
1970         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
1971
1972 out_err:
1973         mutex_unlock(&tracing_cpumask_update_lock);
1974
1975         return count;
1976 }
1977
1978 static ssize_t
1979 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
1980                       size_t count, loff_t *ppos)
1981 {
1982         int err, cpu;
1983         cpumask_var_t tracing_cpumask_new;
1984
1985         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
1986                 return -ENOMEM;
1987
1988         mutex_lock(&tracing_cpumask_update_lock);
1989         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
1990         if (err)
1991                 goto err_unlock;
1992
1993         local_irq_disable();
1994         __raw_spin_lock(&ftrace_max_lock);
1995         for_each_tracing_cpu(cpu) {
1996                 /*
1997                  * Increase/decrease the disabled counter if we are
1998                  * about to flip a bit in the cpumask:
1999                  */
2000                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2001                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2002                         atomic_inc(&global_trace.data[cpu]->disabled);
2003                 }
2004                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2005                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2006                         atomic_dec(&global_trace.data[cpu]->disabled);
2007                 }
2008         }
2009         __raw_spin_unlock(&ftrace_max_lock);
2010         local_irq_enable();
2011
2012         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2013
2014         mutex_unlock(&tracing_cpumask_update_lock);
2015         free_cpumask_var(tracing_cpumask_new);
2016
2017         return count;
2018
2019 err_unlock:
2020         mutex_unlock(&tracing_cpumask_update_lock);
2021         free_cpumask_var(tracing_cpumask);
2022
2023         return err;
2024 }
2025
2026 static struct file_operations tracing_cpumask_fops = {
2027         .open           = tracing_open_generic,
2028         .read           = tracing_cpumask_read,
2029         .write          = tracing_cpumask_write,
2030 };
2031
2032 static ssize_t
2033 tracing_trace_options_read(struct file *filp, char __user *ubuf,
2034                        size_t cnt, loff_t *ppos)
2035 {
2036         struct tracer_opt *trace_opts;
2037         u32 tracer_flags;
2038         int len = 0;
2039         char *buf;
2040         int r = 0;
2041         int i;
2042
2043
2044         /* calculate max size */
2045         for (i = 0; trace_options[i]; i++) {
2046                 len += strlen(trace_options[i]);
2047                 len += 3; /* "no" and newline */
2048         }
2049
2050         mutex_lock(&trace_types_lock);
2051         tracer_flags = current_trace->flags->val;
2052         trace_opts = current_trace->flags->opts;
2053
2054         /*
2055          * Increase the size with names of options specific
2056          * of the current tracer.
2057          */
2058         for (i = 0; trace_opts[i].name; i++) {
2059                 len += strlen(trace_opts[i].name);
2060                 len += 3; /* "no" and newline */
2061         }
2062
2063         /* +2 for \n and \0 */
2064         buf = kmalloc(len + 2, GFP_KERNEL);
2065         if (!buf) {
2066                 mutex_unlock(&trace_types_lock);
2067                 return -ENOMEM;
2068         }
2069
2070         for (i = 0; trace_options[i]; i++) {
2071                 if (trace_flags & (1 << i))
2072                         r += sprintf(buf + r, "%s\n", trace_options[i]);
2073                 else
2074                         r += sprintf(buf + r, "no%s\n", trace_options[i]);
2075         }
2076
2077         for (i = 0; trace_opts[i].name; i++) {
2078                 if (tracer_flags & trace_opts[i].bit)
2079                         r += sprintf(buf + r, "%s\n",
2080                                 trace_opts[i].name);
2081                 else
2082                         r += sprintf(buf + r, "no%s\n",
2083                                 trace_opts[i].name);
2084         }
2085         mutex_unlock(&trace_types_lock);
2086
2087         WARN_ON(r >= len + 2);
2088
2089         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2090
2091         kfree(buf);
2092         return r;
2093 }
2094
2095 /* Try to assign a tracer specific option */
2096 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2097 {
2098         struct tracer_flags *trace_flags = trace->flags;
2099         struct tracer_opt *opts = NULL;
2100         int ret = 0, i = 0;
2101         int len;
2102
2103         for (i = 0; trace_flags->opts[i].name; i++) {
2104                 opts = &trace_flags->opts[i];
2105                 len = strlen(opts->name);
2106
2107                 if (strncmp(cmp, opts->name, len) == 0) {
2108                         ret = trace->set_flag(trace_flags->val,
2109                                 opts->bit, !neg);
2110                         break;
2111                 }
2112         }
2113         /* Not found */
2114         if (!trace_flags->opts[i].name)
2115                 return -EINVAL;
2116
2117         /* Refused to handle */
2118         if (ret)
2119                 return ret;
2120
2121         if (neg)
2122                 trace_flags->val &= ~opts->bit;
2123         else
2124                 trace_flags->val |= opts->bit;
2125
2126         return 0;
2127 }
2128
2129 static ssize_t
2130 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2131                         size_t cnt, loff_t *ppos)
2132 {
2133         char buf[64];
2134         char *cmp = buf;
2135         int neg = 0;
2136         int ret;
2137         int i;
2138
2139         if (cnt >= sizeof(buf))
2140                 return -EINVAL;
2141
2142         if (copy_from_user(&buf, ubuf, cnt))
2143                 return -EFAULT;
2144
2145         buf[cnt] = 0;
2146
2147         if (strncmp(buf, "no", 2) == 0) {
2148                 neg = 1;
2149                 cmp += 2;
2150         }
2151
2152         for (i = 0; trace_options[i]; i++) {
2153                 int len = strlen(trace_options[i]);
2154
2155                 if (strncmp(cmp, trace_options[i], len) == 0) {
2156                         if (neg)
2157                                 trace_flags &= ~(1 << i);
2158                         else
2159                                 trace_flags |= (1 << i);
2160                         break;
2161                 }
2162         }
2163
2164         /* If no option could be set, test the specific tracer options */
2165         if (!trace_options[i]) {
2166                 mutex_lock(&trace_types_lock);
2167                 ret = set_tracer_option(current_trace, cmp, neg);
2168                 mutex_unlock(&trace_types_lock);
2169                 if (ret)
2170                         return ret;
2171         }
2172
2173         filp->f_pos += cnt;
2174
2175         return cnt;
2176 }
2177
2178 static struct file_operations tracing_iter_fops = {
2179         .open           = tracing_open_generic,
2180         .read           = tracing_trace_options_read,
2181         .write          = tracing_trace_options_write,
2182 };
2183
2184 static const char readme_msg[] =
2185         "tracing mini-HOWTO:\n\n"
2186         "# mkdir /debug\n"
2187         "# mount -t debugfs nodev /debug\n\n"
2188         "# cat /debug/tracing/available_tracers\n"
2189         "wakeup preemptirqsoff preemptoff irqsoff ftrace sched_switch none\n\n"
2190         "# cat /debug/tracing/current_tracer\n"
2191         "none\n"
2192         "# echo sched_switch > /debug/tracing/current_tracer\n"
2193         "# cat /debug/tracing/current_tracer\n"
2194         "sched_switch\n"
2195         "# cat /debug/tracing/trace_options\n"
2196         "noprint-parent nosym-offset nosym-addr noverbose\n"
2197         "# echo print-parent > /debug/tracing/trace_options\n"
2198         "# echo 1 > /debug/tracing/tracing_enabled\n"
2199         "# cat /debug/tracing/trace > /tmp/trace.txt\n"
2200         "echo 0 > /debug/tracing/tracing_enabled\n"
2201 ;
2202
2203 static ssize_t
2204 tracing_readme_read(struct file *filp, char __user *ubuf,
2205                        size_t cnt, loff_t *ppos)
2206 {
2207         return simple_read_from_buffer(ubuf, cnt, ppos,
2208                                         readme_msg, strlen(readme_msg));
2209 }
2210
2211 static struct file_operations tracing_readme_fops = {
2212         .open           = tracing_open_generic,
2213         .read           = tracing_readme_read,
2214 };
2215
2216 static ssize_t
2217 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2218                   size_t cnt, loff_t *ppos)
2219 {
2220         char buf[64];
2221         int r;
2222
2223         r = sprintf(buf, "%u\n", tracer_enabled);
2224         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2225 }
2226
2227 static ssize_t
2228 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2229                    size_t cnt, loff_t *ppos)
2230 {
2231         struct trace_array *tr = filp->private_data;
2232         char buf[64];
2233         unsigned long val;
2234         int ret;
2235
2236         if (cnt >= sizeof(buf))
2237                 return -EINVAL;
2238
2239         if (copy_from_user(&buf, ubuf, cnt))
2240                 return -EFAULT;
2241
2242         buf[cnt] = 0;
2243
2244         ret = strict_strtoul(buf, 10, &val);
2245         if (ret < 0)
2246                 return ret;
2247
2248         val = !!val;
2249
2250         mutex_lock(&trace_types_lock);
2251         if (tracer_enabled ^ val) {
2252                 if (val) {
2253                         tracer_enabled = 1;
2254                         if (current_trace->start)
2255                                 current_trace->start(tr);
2256                         tracing_start();
2257                 } else {
2258                         tracer_enabled = 0;
2259                         tracing_stop();
2260                         if (current_trace->stop)
2261                                 current_trace->stop(tr);
2262                 }
2263         }
2264         mutex_unlock(&trace_types_lock);
2265
2266         filp->f_pos += cnt;
2267
2268         return cnt;
2269 }
2270
2271 static ssize_t
2272 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2273                        size_t cnt, loff_t *ppos)
2274 {
2275         char buf[max_tracer_type_len+2];
2276         int r;
2277
2278         mutex_lock(&trace_types_lock);
2279         if (current_trace)
2280                 r = sprintf(buf, "%s\n", current_trace->name);
2281         else
2282                 r = sprintf(buf, "\n");
2283         mutex_unlock(&trace_types_lock);
2284
2285         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2286 }
2287
2288 int tracer_init(struct tracer *t, struct trace_array *tr)
2289 {
2290         tracing_reset_online_cpus(tr);
2291         return t->init(tr);
2292 }
2293
2294 struct trace_option_dentry;
2295
2296 static struct trace_option_dentry *
2297 create_trace_option_files(struct tracer *tracer);
2298
2299 static void
2300 destroy_trace_option_files(struct trace_option_dentry *topts);
2301
2302 static int tracing_set_tracer(const char *buf)
2303 {
2304         static struct trace_option_dentry *topts;
2305         struct trace_array *tr = &global_trace;
2306         struct tracer *t;
2307         int ret = 0;
2308
2309         mutex_lock(&trace_types_lock);
2310         for (t = trace_types; t; t = t->next) {
2311                 if (strcmp(t->name, buf) == 0)
2312                         break;
2313         }
2314         if (!t) {
2315                 ret = -EINVAL;
2316                 goto out;
2317         }
2318         if (t == current_trace)
2319                 goto out;
2320
2321         trace_branch_disable();
2322         if (current_trace && current_trace->reset)
2323                 current_trace->reset(tr);
2324
2325         destroy_trace_option_files(topts);
2326
2327         current_trace = t;
2328
2329         topts = create_trace_option_files(current_trace);
2330
2331         if (t->init) {
2332                 ret = tracer_init(t, tr);
2333                 if (ret)
2334                         goto out;
2335         }
2336
2337         trace_branch_enable(tr);
2338  out:
2339         mutex_unlock(&trace_types_lock);
2340
2341         return ret;
2342 }
2343
2344 static ssize_t
2345 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2346                         size_t cnt, loff_t *ppos)
2347 {
2348         char buf[max_tracer_type_len+1];
2349         int i;
2350         size_t ret;
2351         int err;
2352
2353         ret = cnt;
2354
2355         if (cnt > max_tracer_type_len)
2356                 cnt = max_tracer_type_len;
2357
2358         if (copy_from_user(&buf, ubuf, cnt))
2359                 return -EFAULT;
2360
2361         buf[cnt] = 0;
2362
2363         /* strip ending whitespace. */
2364         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2365                 buf[i] = 0;
2366
2367         err = tracing_set_tracer(buf);
2368         if (err)
2369                 return err;
2370
2371         filp->f_pos += ret;
2372
2373         return ret;
2374 }
2375
2376 static ssize_t
2377 tracing_max_lat_read(struct file *filp, char __user *ubuf,
2378                      size_t cnt, loff_t *ppos)
2379 {
2380         unsigned long *ptr = filp->private_data;
2381         char buf[64];
2382         int r;
2383
2384         r = snprintf(buf, sizeof(buf), "%ld\n",
2385                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2386         if (r > sizeof(buf))
2387                 r = sizeof(buf);
2388         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2389 }
2390
2391 static ssize_t
2392 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2393                       size_t cnt, loff_t *ppos)
2394 {
2395         unsigned long *ptr = filp->private_data;
2396         char buf[64];
2397         unsigned long val;
2398         int ret;
2399
2400         if (cnt >= sizeof(buf))
2401                 return -EINVAL;
2402
2403         if (copy_from_user(&buf, ubuf, cnt))
2404                 return -EFAULT;
2405
2406         buf[cnt] = 0;
2407
2408         ret = strict_strtoul(buf, 10, &val);
2409         if (ret < 0)
2410                 return ret;
2411
2412         *ptr = val * 1000;
2413
2414         return cnt;
2415 }
2416
2417 static int tracing_open_pipe(struct inode *inode, struct file *filp)
2418 {
2419         long cpu_file = (long) inode->i_private;
2420         struct trace_iterator *iter;
2421         int ret = 0;
2422
2423         if (tracing_disabled)
2424                 return -ENODEV;
2425
2426         mutex_lock(&trace_types_lock);
2427
2428         /* We only allow one reader per cpu */
2429         if (cpu_file == TRACE_PIPE_ALL_CPU) {
2430                 if (!cpumask_empty(tracing_reader_cpumask)) {
2431                         ret = -EBUSY;
2432                         goto out;
2433                 }
2434                 cpumask_setall(tracing_reader_cpumask);
2435         } else {
2436                 if (!cpumask_test_cpu(cpu_file, tracing_reader_cpumask))
2437                         cpumask_set_cpu(cpu_file, tracing_reader_cpumask);
2438                 else {
2439                         ret = -EBUSY;
2440                         goto out;
2441                 }
2442         }
2443
2444         /* create a buffer to store the information to pass to userspace */
2445         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2446         if (!iter) {
2447                 ret = -ENOMEM;
2448                 goto out;
2449         }
2450
2451         /*
2452          * We make a copy of the current tracer to avoid concurrent
2453          * changes on it while we are reading.
2454          */
2455         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
2456         if (!iter->trace) {
2457                 ret = -ENOMEM;
2458                 goto fail;
2459         }
2460         if (current_trace)
2461                 *iter->trace = *current_trace;
2462
2463         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
2464                 ret = -ENOMEM;
2465                 goto fail;
2466         }
2467
2468         /* trace pipe does not show start of buffer */
2469         cpumask_setall(iter->started);
2470
2471         iter->cpu_file = cpu_file;
2472         iter->tr = &global_trace;
2473         mutex_init(&iter->mutex);
2474         filp->private_data = iter;
2475
2476         if (iter->trace->pipe_open)
2477                 iter->trace->pipe_open(iter);
2478
2479 out:
2480         mutex_unlock(&trace_types_lock);
2481         return ret;
2482
2483 fail:
2484         kfree(iter->trace);
2485         kfree(iter);
2486         mutex_unlock(&trace_types_lock);
2487         return ret;
2488 }
2489
2490 static int tracing_release_pipe(struct inode *inode, struct file *file)
2491 {
2492         struct trace_iterator *iter = file->private_data;
2493
2494         mutex_lock(&trace_types_lock);
2495
2496         if (iter->cpu_file == TRACE_PIPE_ALL_CPU)
2497                 cpumask_clear(tracing_reader_cpumask);
2498         else
2499                 cpumask_clear_cpu(iter->cpu_file, tracing_reader_cpumask);
2500
2501         mutex_unlock(&trace_types_lock);
2502
2503         free_cpumask_var(iter->started);
2504         mutex_destroy(&iter->mutex);
2505         kfree(iter->trace);
2506         kfree(iter);
2507
2508         return 0;
2509 }
2510
2511 static unsigned int
2512 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2513 {
2514         struct trace_iterator *iter = filp->private_data;
2515
2516         if (trace_flags & TRACE_ITER_BLOCK) {
2517                 /*
2518                  * Always select as readable when in blocking mode
2519                  */
2520                 return POLLIN | POLLRDNORM;
2521         } else {
2522                 if (!trace_empty(iter))
2523                         return POLLIN | POLLRDNORM;
2524                 poll_wait(filp, &trace_wait, poll_table);
2525                 if (!trace_empty(iter))
2526                         return POLLIN | POLLRDNORM;
2527
2528                 return 0;
2529         }
2530 }
2531
2532
2533 void default_wait_pipe(struct trace_iterator *iter)
2534 {
2535         DEFINE_WAIT(wait);
2536
2537         prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
2538
2539         if (trace_empty(iter))
2540                 schedule();
2541
2542         finish_wait(&trace_wait, &wait);
2543 }
2544
2545 /*
2546  * This is a make-shift waitqueue.
2547  * A tracer might use this callback on some rare cases:
2548  *
2549  *  1) the current tracer might hold the runqueue lock when it wakes up
2550  *     a reader, hence a deadlock (sched, function, and function graph tracers)
2551  *  2) the function tracers, trace all functions, we don't want
2552  *     the overhead of calling wake_up and friends
2553  *     (and tracing them too)
2554  *
2555  *     Anyway, this is really very primitive wakeup.
2556  */
2557 void poll_wait_pipe(struct trace_iterator *iter)
2558 {
2559         set_current_state(TASK_INTERRUPTIBLE);
2560         /* sleep for 100 msecs, and try again. */
2561         schedule_timeout(HZ / 10);
2562 }
2563
2564 /* Must be called with trace_types_lock mutex held. */
2565 static int tracing_wait_pipe(struct file *filp)
2566 {
2567         struct trace_iterator *iter = filp->private_data;
2568
2569         while (trace_empty(iter)) {
2570
2571                 if ((filp->f_flags & O_NONBLOCK)) {
2572                         return -EAGAIN;
2573                 }
2574
2575                 mutex_unlock(&iter->mutex);
2576
2577                 iter->trace->wait_pipe(iter);
2578
2579                 mutex_lock(&iter->mutex);
2580
2581                 if (signal_pending(current))
2582                         return -EINTR;
2583
2584                 /*
2585                  * We block until we read something and tracing is disabled.
2586                  * We still block if tracing is disabled, but we have never
2587                  * read anything. This allows a user to cat this file, and
2588                  * then enable tracing. But after we have read something,
2589                  * we give an EOF when tracing is again disabled.
2590                  *
2591                  * iter->pos will be 0 if we haven't read anything.
2592                  */
2593                 if (!tracer_enabled && iter->pos)
2594                         break;
2595         }
2596
2597         return 1;
2598 }
2599
2600 /*
2601  * Consumer reader.
2602  */
2603 static ssize_t
2604 tracing_read_pipe(struct file *filp, char __user *ubuf,
2605                   size_t cnt, loff_t *ppos)
2606 {
2607         struct trace_iterator *iter = filp->private_data;
2608         static struct tracer *old_tracer;
2609         ssize_t sret;
2610
2611         /* return any leftover data */
2612         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2613         if (sret != -EBUSY)
2614                 return sret;
2615
2616         trace_seq_init(&iter->seq);
2617
2618         /* copy the tracer to avoid using a global lock all around */
2619         mutex_lock(&trace_types_lock);
2620         if (unlikely(old_tracer != current_trace && current_trace)) {
2621                 old_tracer = current_trace;
2622                 *iter->trace = *current_trace;
2623         }
2624         mutex_unlock(&trace_types_lock);
2625
2626         /*
2627          * Avoid more than one consumer on a single file descriptor
2628          * This is just a matter of traces coherency, the ring buffer itself
2629          * is protected.
2630          */
2631         mutex_lock(&iter->mutex);
2632         if (iter->trace->read) {
2633                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2634                 if (sret)
2635                         goto out;
2636         }
2637
2638 waitagain:
2639         sret = tracing_wait_pipe(filp);
2640         if (sret <= 0)
2641                 goto out;
2642
2643         /* stop when tracing is finished */
2644         if (trace_empty(iter)) {
2645                 sret = 0;
2646                 goto out;
2647         }
2648
2649         if (cnt >= PAGE_SIZE)
2650                 cnt = PAGE_SIZE - 1;
2651
2652         /* reset all but tr, trace, and overruns */
2653         memset(&iter->seq, 0,
2654                sizeof(struct trace_iterator) -
2655                offsetof(struct trace_iterator, seq));
2656         iter->pos = -1;
2657
2658         while (find_next_entry_inc(iter) != NULL) {
2659                 enum print_line_t ret;
2660                 int len = iter->seq.len;
2661
2662                 ret = print_trace_line(iter);
2663                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2664                         /* don't print partial lines */
2665                         iter->seq.len = len;
2666                         break;
2667                 }
2668                 if (ret != TRACE_TYPE_NO_CONSUME)
2669                         trace_consume(iter);
2670
2671                 if (iter->seq.len >= cnt)
2672                         break;
2673         }
2674
2675         /* Now copy what we have to the user */
2676         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2677         if (iter->seq.readpos >= iter->seq.len)
2678                 trace_seq_init(&iter->seq);
2679
2680         /*
2681          * If there was nothing to send to user, inspite of consuming trace
2682          * entries, go back to wait for more entries.
2683          */
2684         if (sret == -EBUSY)
2685                 goto waitagain;
2686
2687 out:
2688         mutex_unlock(&iter->mutex);
2689
2690         return sret;
2691 }
2692
2693 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
2694                                      struct pipe_buffer *buf)
2695 {
2696         __free_page(buf->page);
2697 }
2698
2699 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
2700                                      unsigned int idx)
2701 {
2702         __free_page(spd->pages[idx]);
2703 }
2704
2705 static struct pipe_buf_operations tracing_pipe_buf_ops = {
2706         .can_merge              = 0,
2707         .map                    = generic_pipe_buf_map,
2708         .unmap                  = generic_pipe_buf_unmap,
2709         .confirm                = generic_pipe_buf_confirm,
2710         .release                = tracing_pipe_buf_release,
2711         .steal                  = generic_pipe_buf_steal,
2712         .get                    = generic_pipe_buf_get,
2713 };
2714
2715 static size_t
2716 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
2717 {
2718         size_t count;
2719         int ret;
2720
2721         /* Seq buffer is page-sized, exactly what we need. */
2722         for (;;) {
2723                 count = iter->seq.len;
2724                 ret = print_trace_line(iter);
2725                 count = iter->seq.len - count;
2726                 if (rem < count) {
2727                         rem = 0;
2728                         iter->seq.len -= count;
2729                         break;
2730                 }
2731                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2732                         iter->seq.len -= count;
2733                         break;
2734                 }
2735
2736                 trace_consume(iter);
2737                 rem -= count;
2738                 if (!find_next_entry_inc(iter)) {
2739                         rem = 0;
2740                         iter->ent = NULL;
2741                         break;
2742                 }
2743         }
2744
2745         return rem;
2746 }
2747
2748 static ssize_t tracing_splice_read_pipe(struct file *filp,
2749                                         loff_t *ppos,
2750                                         struct pipe_inode_info *pipe,
2751                                         size_t len,
2752                                         unsigned int flags)
2753 {
2754         struct page *pages[PIPE_BUFFERS];
2755         struct partial_page partial[PIPE_BUFFERS];
2756         struct trace_iterator *iter = filp->private_data;
2757         struct splice_pipe_desc spd = {
2758                 .pages          = pages,
2759                 .partial        = partial,
2760                 .nr_pages       = 0, /* This gets updated below. */
2761                 .flags          = flags,
2762                 .ops            = &tracing_pipe_buf_ops,
2763                 .spd_release    = tracing_spd_release_pipe,
2764         };
2765         static struct tracer *old_tracer;
2766         ssize_t ret;
2767         size_t rem;
2768         unsigned int i;
2769
2770         /* copy the tracer to avoid using a global lock all around */
2771         mutex_lock(&trace_types_lock);
2772         if (unlikely(old_tracer != current_trace && current_trace)) {
2773                 old_tracer = current_trace;
2774                 *iter->trace = *current_trace;
2775         }
2776         mutex_unlock(&trace_types_lock);
2777
2778         mutex_lock(&iter->mutex);
2779
2780         if (iter->trace->splice_read) {
2781                 ret = iter->trace->splice_read(iter, filp,
2782                                                ppos, pipe, len, flags);
2783                 if (ret)
2784                         goto out_err;
2785         }
2786
2787         ret = tracing_wait_pipe(filp);
2788         if (ret <= 0)
2789                 goto out_err;
2790
2791         if (!iter->ent && !find_next_entry_inc(iter)) {
2792                 ret = -EFAULT;
2793                 goto out_err;
2794         }
2795
2796         /* Fill as many pages as possible. */
2797         for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
2798                 pages[i] = alloc_page(GFP_KERNEL);
2799                 if (!pages[i])
2800                         break;
2801
2802                 rem = tracing_fill_pipe_page(rem, iter);
2803
2804                 /* Copy the data into the page, so we can start over. */
2805                 ret = trace_seq_to_buffer(&iter->seq,
2806                                           page_address(pages[i]),
2807                                           iter->seq.len);
2808                 if (ret < 0) {
2809                         __free_page(pages[i]);
2810                         break;
2811                 }
2812                 partial[i].offset = 0;
2813                 partial[i].len = iter->seq.len;
2814
2815                 trace_seq_init(&iter->seq);
2816         }
2817
2818         mutex_unlock(&iter->mutex);
2819
2820         spd.nr_pages = i;
2821
2822         return splice_to_pipe(pipe, &spd);
2823
2824 out_err:
2825         mutex_unlock(&iter->mutex);
2826
2827         return ret;
2828 }
2829
2830 static ssize_t
2831 tracing_entries_read(struct file *filp, char __user *ubuf,
2832                      size_t cnt, loff_t *ppos)
2833 {
2834         struct trace_array *tr = filp->private_data;
2835         char buf[64];
2836         int r;
2837
2838         r = sprintf(buf, "%lu\n", tr->entries >> 10);
2839         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2840 }
2841
2842 static ssize_t
2843 tracing_entries_write(struct file *filp, const char __user *ubuf,
2844                       size_t cnt, loff_t *ppos)
2845 {
2846         unsigned long val;
2847         char buf[64];
2848         int ret, cpu;
2849
2850         if (cnt >= sizeof(buf))
2851                 return -EINVAL;
2852
2853         if (copy_from_user(&buf, ubuf, cnt))
2854                 return -EFAULT;
2855
2856         buf[cnt] = 0;
2857
2858         ret = strict_strtoul(buf, 10, &val);
2859         if (ret < 0)
2860                 return ret;
2861
2862         /* must have at least 1 entry */
2863         if (!val)
2864                 return -EINVAL;
2865
2866         mutex_lock(&trace_types_lock);
2867
2868         tracing_stop();
2869
2870         /* disable all cpu buffers */
2871         for_each_tracing_cpu(cpu) {
2872                 if (global_trace.data[cpu])
2873                         atomic_inc(&global_trace.data[cpu]->disabled);
2874                 if (max_tr.data[cpu])
2875                         atomic_inc(&max_tr.data[cpu]->disabled);
2876         }
2877
2878         /* value is in KB */
2879         val <<= 10;
2880
2881         if (val != global_trace.entries) {
2882                 ret = ring_buffer_resize(global_trace.buffer, val);
2883                 if (ret < 0) {
2884                         cnt = ret;
2885                         goto out;
2886                 }
2887
2888                 ret = ring_buffer_resize(max_tr.buffer, val);
2889                 if (ret < 0) {
2890                         int r;
2891                         cnt = ret;
2892                         r = ring_buffer_resize(global_trace.buffer,
2893                                                global_trace.entries);
2894                         if (r < 0) {
2895                                 /* AARGH! We are left with different
2896                                  * size max buffer!!!! */
2897                                 WARN_ON(1);
2898                                 tracing_disabled = 1;
2899                         }
2900                         goto out;
2901                 }
2902
2903                 global_trace.entries = val;
2904         }
2905
2906         filp->f_pos += cnt;
2907
2908         /* If check pages failed, return ENOMEM */
2909         if (tracing_disabled)
2910                 cnt = -ENOMEM;
2911  out:
2912         for_each_tracing_cpu(cpu) {
2913                 if (global_trace.data[cpu])
2914                         atomic_dec(&global_trace.data[cpu]->disabled);
2915                 if (max_tr.data[cpu])
2916                         atomic_dec(&max_tr.data[cpu]->disabled);
2917         }
2918
2919         tracing_start();
2920         max_tr.entries = global_trace.entries;
2921         mutex_unlock(&trace_types_lock);
2922
2923         return cnt;
2924 }
2925
2926 static int mark_printk(const char *fmt, ...)
2927 {
2928         int ret;
2929         va_list args;
2930         va_start(args, fmt);
2931         ret = trace_vprintk(0, -1, fmt, args);
2932         va_end(args);
2933         return ret;
2934 }
2935
2936 static ssize_t
2937 tracing_mark_write(struct file *filp, const char __user *ubuf,
2938                                         size_t cnt, loff_t *fpos)
2939 {
2940         char *buf;
2941         char *end;
2942
2943         if (tracing_disabled)
2944                 return -EINVAL;
2945
2946         if (cnt > TRACE_BUF_SIZE)
2947                 cnt = TRACE_BUF_SIZE;
2948
2949         buf = kmalloc(cnt + 1, GFP_KERNEL);
2950         if (buf == NULL)
2951                 return -ENOMEM;
2952
2953         if (copy_from_user(buf, ubuf, cnt)) {
2954                 kfree(buf);
2955                 return -EFAULT;
2956         }
2957
2958         /* Cut from the first nil or newline. */
2959         buf[cnt] = '\0';
2960         end = strchr(buf, '\n');
2961         if (end)
2962                 *end = '\0';
2963
2964         cnt = mark_printk("%s\n", buf);
2965         kfree(buf);
2966         *fpos += cnt;
2967
2968         return cnt;
2969 }
2970
2971 static struct file_operations tracing_max_lat_fops = {
2972         .open           = tracing_open_generic,
2973         .read           = tracing_max_lat_read,
2974         .write          = tracing_max_lat_write,
2975 };
2976
2977 static struct file_operations tracing_ctrl_fops = {
2978         .open           = tracing_open_generic,
2979         .read           = tracing_ctrl_read,
2980         .write          = tracing_ctrl_write,
2981 };
2982
2983 static struct file_operations set_tracer_fops = {
2984         .open           = tracing_open_generic,
2985         .read           = tracing_set_trace_read,
2986         .write          = tracing_set_trace_write,
2987 };
2988
2989 static struct file_operations tracing_pipe_fops = {
2990         .open           = tracing_open_pipe,
2991         .poll           = tracing_poll_pipe,
2992         .read           = tracing_read_pipe,
2993         .splice_read    = tracing_splice_read_pipe,
2994         .release        = tracing_release_pipe,
2995 };
2996
2997 static struct file_operations tracing_entries_fops = {
2998         .open           = tracing_open_generic,
2999         .read           = tracing_entries_read,
3000         .write          = tracing_entries_write,
3001 };
3002
3003 static struct file_operations tracing_mark_fops = {
3004         .open           = tracing_open_generic,
3005         .write          = tracing_mark_write,
3006 };
3007
3008 #ifdef CONFIG_DYNAMIC_FTRACE
3009
3010 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3011 {
3012         return 0;
3013 }
3014
3015 static ssize_t
3016 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
3017                   size_t cnt, loff_t *ppos)
3018 {
3019         static char ftrace_dyn_info_buffer[1024];
3020         static DEFINE_MUTEX(dyn_info_mutex);
3021         unsigned long *p = filp->private_data;
3022         char *buf = ftrace_dyn_info_buffer;
3023         int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
3024         int r;
3025
3026         mutex_lock(&dyn_info_mutex);
3027         r = sprintf(buf, "%ld ", *p);
3028
3029         r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
3030         buf[r++] = '\n';
3031
3032         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3033
3034         mutex_unlock(&dyn_info_mutex);
3035
3036         return r;
3037 }
3038
3039 static struct file_operations tracing_dyn_info_fops = {
3040         .open           = tracing_open_generic,
3041         .read           = tracing_read_dyn_info,
3042 };
3043 #endif
3044
3045 static struct dentry *d_tracer;
3046
3047 struct dentry *tracing_init_dentry(void)
3048 {
3049         static int once;
3050
3051         if (d_tracer)
3052                 return d_tracer;
3053
3054         d_tracer = debugfs_create_dir("tracing", NULL);
3055
3056         if (!d_tracer && !once) {
3057                 once = 1;
3058                 pr_warning("Could not create debugfs directory 'tracing'\n");
3059                 return NULL;
3060         }
3061
3062         return d_tracer;
3063 }
3064
3065 static struct dentry *d_percpu;
3066
3067 struct dentry *tracing_dentry_percpu(void)
3068 {
3069         static int once;
3070         struct dentry *d_tracer;
3071
3072         if (d_percpu)
3073                 return d_percpu;
3074
3075         d_tracer = tracing_init_dentry();
3076
3077         if (!d_tracer)
3078                 return NULL;
3079
3080         d_percpu = debugfs_create_dir("per_cpu", d_tracer);
3081
3082         if (!d_percpu && !once) {
3083                 once = 1;
3084                 pr_warning("Could not create debugfs directory 'per_cpu'\n");
3085                 return NULL;
3086         }
3087
3088         return d_percpu;
3089 }
3090
3091 static void tracing_init_debugfs_percpu(long cpu)
3092 {
3093         struct dentry *d_percpu = tracing_dentry_percpu();
3094         struct dentry *entry, *d_cpu;
3095         /* strlen(cpu) + MAX(log10(cpu)) + '\0' */
3096         char cpu_dir[7];
3097
3098         if (cpu > 999 || cpu < 0)
3099                 return;
3100
3101         sprintf(cpu_dir, "cpu%ld", cpu);
3102         d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
3103         if (!d_cpu) {
3104                 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
3105                 return;
3106         }
3107
3108         /* per cpu trace_pipe */
3109         entry = debugfs_create_file("trace_pipe", 0444, d_cpu,
3110                                 (void *) cpu, &tracing_pipe_fops);
3111         if (!entry)
3112                 pr_warning("Could not create debugfs 'trace_pipe' entry\n");
3113
3114         /* per cpu trace */
3115         entry = debugfs_create_file("trace", 0444, d_cpu,
3116                                 (void *) cpu, &tracing_fops);
3117         if (!entry)
3118                 pr_warning("Could not create debugfs 'trace' entry\n");
3119 }
3120
3121 #ifdef CONFIG_FTRACE_SELFTEST
3122 /* Let selftest have access to static functions in this file */
3123 #include "trace_selftest.c"
3124 #endif
3125
3126 struct trace_option_dentry {
3127         struct tracer_opt               *opt;
3128         struct tracer_flags             *flags;
3129         struct dentry                   *entry;
3130 };
3131
3132 static ssize_t
3133 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
3134                         loff_t *ppos)
3135 {
3136         struct trace_option_dentry *topt = filp->private_data;
3137         char *buf;
3138
3139         if (topt->flags->val & topt->opt->bit)
3140                 buf = "1\n";
3141         else
3142                 buf = "0\n";
3143
3144         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3145 }
3146
3147 static ssize_t
3148 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
3149                          loff_t *ppos)
3150 {
3151         struct trace_option_dentry *topt = filp->private_data;
3152         unsigned long val;
3153         char buf[64];
3154         int ret;
3155
3156         if (cnt >= sizeof(buf))
3157                 return -EINVAL;
3158
3159         if (copy_from_user(&buf, ubuf, cnt))
3160                 return -EFAULT;
3161
3162         buf[cnt] = 0;
3163
3164         ret = strict_strtoul(buf, 10, &val);
3165         if (ret < 0)
3166                 return ret;
3167
3168         ret = 0;
3169         switch (val) {
3170         case 0:
3171                 /* do nothing if already cleared */
3172                 if (!(topt->flags->val & topt->opt->bit))
3173                         break;
3174
3175                 mutex_lock(&trace_types_lock);
3176                 if (current_trace->set_flag)
3177                         ret = current_trace->set_flag(topt->flags->val,
3178                                                       topt->opt->bit, 0);
3179                 mutex_unlock(&trace_types_lock);
3180                 if (ret)
3181                         return ret;
3182                 topt->flags->val &= ~topt->opt->bit;
3183                 break;
3184         case 1:
3185                 /* do nothing if already set */
3186                 if (topt->flags->val & topt->opt->bit)
3187                         break;
3188
3189                 mutex_lock(&trace_types_lock);
3190                 if (current_trace->set_flag)
3191                         ret = current_trace->set_flag(topt->flags->val,
3192                                                       topt->opt->bit, 1);
3193                 mutex_unlock(&trace_types_lock);
3194                 if (ret)
3195                         return ret;
3196                 topt->flags->val |= topt->opt->bit;
3197                 break;
3198
3199         default:
3200                 return -EINVAL;
3201         }
3202
3203         *ppos += cnt;
3204
3205         return cnt;
3206 }
3207
3208
3209 static const struct file_operations trace_options_fops = {
3210         .open = tracing_open_generic,
3211         .read = trace_options_read,
3212         .write = trace_options_write,
3213 };
3214
3215 static ssize_t
3216 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
3217                         loff_t *ppos)
3218 {
3219         long index = (long)filp->private_data;
3220         char *buf;
3221
3222         if (trace_flags & (1 << index))
3223                 buf = "1\n";
3224         else
3225                 buf = "0\n";
3226
3227         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3228 }
3229
3230 static ssize_t
3231 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
3232                          loff_t *ppos)
3233 {
3234         long index = (long)filp->private_data;
3235         char buf[64];
3236         unsigned long val;
3237         int ret;
3238
3239         if (cnt >= sizeof(buf))
3240                 return -EINVAL;
3241
3242         if (copy_from_user(&buf, ubuf, cnt))
3243                 return -EFAULT;
3244
3245         buf[cnt] = 0;
3246
3247         ret = strict_strtoul(buf, 10, &val);
3248         if (ret < 0)
3249                 return ret;
3250
3251         switch (val) {
3252         case 0:
3253                 trace_flags &= ~(1 << index);
3254                 break;
3255         case 1:
3256                 trace_flags |= 1 << index;
3257                 break;
3258
3259         default:
3260                 return -EINVAL;
3261         }
3262
3263         *ppos += cnt;
3264
3265         return cnt;
3266 }
3267
3268 static const struct file_operations trace_options_core_fops = {
3269         .open = tracing_open_generic,
3270         .read = trace_options_core_read,
3271         .write = trace_options_core_write,
3272 };
3273
3274 static struct dentry *trace_options_init_dentry(void)
3275 {
3276         struct dentry *d_tracer;
3277         static struct dentry *t_options;
3278
3279         if (t_options)
3280                 return t_options;
3281
3282         d_tracer = tracing_init_dentry();
3283         if (!d_tracer)
3284                 return NULL;
3285
3286         t_options = debugfs_create_dir("options", d_tracer);
3287         if (!t_options) {
3288                 pr_warning("Could not create debugfs directory 'options'\n");
3289                 return NULL;
3290         }
3291
3292         return t_options;
3293 }
3294
3295 static void
3296 create_trace_option_file(struct trace_option_dentry *topt,
3297                          struct tracer_flags *flags,
3298                          struct tracer_opt *opt)
3299 {
3300         struct dentry *t_options;
3301         struct dentry *entry;
3302
3303         t_options = trace_options_init_dentry();
3304         if (!t_options)
3305                 return;
3306
3307         topt->flags = flags;
3308         topt->opt = opt;
3309
3310         entry = debugfs_create_file(opt->name, 0644, t_options, topt,
3311                                     &trace_options_fops);
3312
3313         topt->entry = entry;
3314
3315 }
3316
3317 static struct trace_option_dentry *
3318 create_trace_option_files(struct tracer *tracer)
3319 {
3320         struct trace_option_dentry *topts;
3321         struct tracer_flags *flags;
3322         struct tracer_opt *opts;
3323         int cnt;
3324
3325         if (!tracer)
3326                 return NULL;
3327
3328         flags = tracer->flags;
3329
3330         if (!flags || !flags->opts)
3331                 return NULL;
3332
3333         opts = flags->opts;
3334
3335         for (cnt = 0; opts[cnt].name; cnt++)
3336                 ;
3337
3338         topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
3339         if (!topts)
3340                 return NULL;
3341
3342         for (cnt = 0; opts[cnt].name; cnt++)
3343                 create_trace_option_file(&topts[cnt], flags,
3344                                          &opts[cnt]);
3345
3346         return topts;
3347 }
3348
3349 static void
3350 destroy_trace_option_files(struct trace_option_dentry *topts)
3351 {
3352         int cnt;
3353
3354         if (!topts)
3355                 return;
3356
3357         for (cnt = 0; topts[cnt].opt; cnt++) {
3358                 if (topts[cnt].entry)
3359                         debugfs_remove(topts[cnt].entry);
3360         }
3361
3362         kfree(topts);
3363 }
3364
3365 static struct dentry *
3366 create_trace_option_core_file(const char *option, long index)
3367 {
3368         struct dentry *t_options;
3369         struct dentry *entry;
3370
3371         t_options = trace_options_init_dentry();
3372         if (!t_options)
3373                 return NULL;
3374
3375         entry = debugfs_create_file(option, 0644, t_options, (void *)index,
3376                                     &trace_options_core_fops);
3377
3378         return entry;
3379 }
3380
3381 static __init void create_trace_options_dir(void)
3382 {
3383         struct dentry *t_options;
3384         struct dentry *entry;
3385         int i;
3386
3387         t_options = trace_options_init_dentry();
3388         if (!t_options)
3389                 return;
3390
3391         for (i = 0; trace_options[i]; i++) {
3392                 entry = create_trace_option_core_file(trace_options[i], i);
3393                 if (!entry)
3394                         pr_warning("Could not create debugfs %s entry\n",
3395                                    trace_options[i]);
3396         }
3397 }
3398
3399 static __init int tracer_init_debugfs(void)
3400 {
3401         struct dentry *d_tracer;
3402         struct dentry *entry;
3403         int cpu;
3404
3405         d_tracer = tracing_init_dentry();
3406
3407         entry = debugfs_create_file("tracing_enabled", 0644, d_tracer,
3408                                     &global_trace, &tracing_ctrl_fops);
3409         if (!entry)
3410                 pr_warning("Could not create debugfs 'tracing_enabled' entry\n");
3411
3412         entry = debugfs_create_file("trace_options", 0644, d_tracer,
3413                                     NULL, &tracing_iter_fops);
3414         if (!entry)
3415                 pr_warning("Could not create debugfs 'trace_options' entry\n");
3416
3417         create_trace_options_dir();
3418
3419         entry = debugfs_create_file("tracing_cpumask", 0644, d_tracer,
3420                                     NULL, &tracing_cpumask_fops);
3421         if (!entry)
3422                 pr_warning("Could not create debugfs 'tracing_cpumask' entry\n");
3423
3424         entry = debugfs_create_file("trace", 0444, d_tracer,
3425                                  (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
3426         if (!entry)
3427                 pr_warning("Could not create debugfs 'trace' entry\n");
3428
3429         entry = debugfs_create_file("available_tracers", 0444, d_tracer,
3430                                     &global_trace, &show_traces_fops);
3431         if (!entry)
3432                 pr_warning("Could not create debugfs 'available_tracers' entry\n");
3433
3434         entry = debugfs_create_file("current_tracer", 0444, d_tracer,
3435                                     &global_trace, &set_tracer_fops);
3436         if (!entry)
3437                 pr_warning("Could not create debugfs 'current_tracer' entry\n");
3438
3439         entry = debugfs_create_file("tracing_max_latency", 0644, d_tracer,
3440                                     &tracing_max_latency,
3441                                     &tracing_max_lat_fops);
3442         if (!entry)
3443                 pr_warning("Could not create debugfs "
3444                            "'tracing_max_latency' entry\n");
3445
3446         entry = debugfs_create_file("tracing_thresh", 0644, d_tracer,
3447                                     &tracing_thresh, &tracing_max_lat_fops);
3448         if (!entry)
3449                 pr_warning("Could not create debugfs "
3450                            "'tracing_thresh' entry\n");
3451         entry = debugfs_create_file("README", 0644, d_tracer,
3452                                     NULL, &tracing_readme_fops);
3453         if (!entry)
3454                 pr_warning("Could not create debugfs 'README' entry\n");
3455
3456         entry = debugfs_create_file("trace_pipe", 0444, d_tracer,
3457                         (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
3458         if (!entry)
3459                 pr_warning("Could not create debugfs "
3460                            "'trace_pipe' entry\n");
3461
3462         entry = debugfs_create_file("buffer_size_kb", 0644, d_tracer,
3463                                     &global_trace, &tracing_entries_fops);
3464         if (!entry)
3465                 pr_warning("Could not create debugfs "
3466                            "'buffer_size_kb' entry\n");
3467
3468         entry = debugfs_create_file("trace_marker", 0220, d_tracer,
3469                                     NULL, &tracing_mark_fops);
3470         if (!entry)
3471                 pr_warning("Could not create debugfs "
3472                            "'trace_marker' entry\n");
3473
3474 #ifdef CONFIG_DYNAMIC_FTRACE
3475         entry = debugfs_create_file("dyn_ftrace_total_info", 0444, d_tracer,
3476                                     &ftrace_update_tot_cnt,
3477                                     &tracing_dyn_info_fops);
3478         if (!entry)
3479                 pr_warning("Could not create debugfs "
3480                            "'dyn_ftrace_total_info' entry\n");
3481 #endif
3482 #ifdef CONFIG_SYSPROF_TRACER
3483         init_tracer_sysprof_debugfs(d_tracer);
3484 #endif
3485
3486         for_each_tracing_cpu(cpu)
3487                 tracing_init_debugfs_percpu(cpu);
3488
3489         return 0;
3490 }
3491
3492 int trace_vprintk(unsigned long ip, int depth, const char *fmt, va_list args)
3493 {
3494         static DEFINE_SPINLOCK(trace_buf_lock);
3495         static char trace_buf[TRACE_BUF_SIZE];
3496
3497         struct ring_buffer_event *event;
3498         struct trace_array *tr = &global_trace;
3499         struct trace_array_cpu *data;
3500         int cpu, len = 0, size, pc;
3501         struct print_entry *entry;
3502         unsigned long irq_flags;
3503
3504         if (tracing_disabled || tracing_selftest_running)
3505                 return 0;
3506
3507         pc = preempt_count();
3508         preempt_disable_notrace();
3509         cpu = raw_smp_processor_id();
3510         data = tr->data[cpu];
3511
3512         if (unlikely(atomic_read(&data->disabled)))
3513                 goto out;
3514
3515         pause_graph_tracing();
3516         spin_lock_irqsave(&trace_buf_lock, irq_flags);
3517         len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
3518
3519         len = min(len, TRACE_BUF_SIZE-1);
3520         trace_buf[len] = 0;
3521
3522         size = sizeof(*entry) + len + 1;
3523         event = trace_buffer_lock_reserve(tr, TRACE_PRINT, size, irq_flags, pc);
3524         if (!event)
3525                 goto out_unlock;
3526         entry = ring_buffer_event_data(event);
3527         entry->ip                       = ip;
3528         entry->depth                    = depth;
3529
3530         memcpy(&entry->buf, trace_buf, len);
3531         entry->buf[len] = 0;
3532         ring_buffer_unlock_commit(tr->buffer, event);
3533
3534  out_unlock:
3535         spin_unlock_irqrestore(&trace_buf_lock, irq_flags);
3536         unpause_graph_tracing();
3537  out:
3538         preempt_enable_notrace();
3539
3540         return len;
3541 }
3542 EXPORT_SYMBOL_GPL(trace_vprintk);
3543
3544 int __ftrace_printk(unsigned long ip, const char *fmt, ...)
3545 {
3546         int ret;
3547         va_list ap;
3548
3549         if (!(trace_flags & TRACE_ITER_PRINTK))
3550                 return 0;
3551
3552         va_start(ap, fmt);
3553         ret = trace_vprintk(ip, task_curr_ret_stack(current), fmt, ap);
3554         va_end(ap);
3555         return ret;
3556 }
3557 EXPORT_SYMBOL_GPL(__ftrace_printk);
3558
3559 int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap)
3560 {
3561         if (!(trace_flags & TRACE_ITER_PRINTK))
3562                 return 0;
3563
3564         return trace_vprintk(ip, task_curr_ret_stack(current), fmt, ap);
3565 }
3566 EXPORT_SYMBOL_GPL(__ftrace_vprintk);
3567
3568 static int trace_panic_handler(struct notifier_block *this,
3569                                unsigned long event, void *unused)
3570 {
3571         if (ftrace_dump_on_oops)
3572                 ftrace_dump();
3573         return NOTIFY_OK;
3574 }
3575
3576 static struct notifier_block trace_panic_notifier = {
3577         .notifier_call  = trace_panic_handler,
3578         .next           = NULL,
3579         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
3580 };
3581
3582 static int trace_die_handler(struct notifier_block *self,
3583                              unsigned long val,
3584                              void *data)
3585 {
3586         switch (val) {
3587         case DIE_OOPS:
3588                 if (ftrace_dump_on_oops)
3589                         ftrace_dump();
3590                 break;
3591         default:
3592                 break;
3593         }
3594         return NOTIFY_OK;
3595 }
3596
3597 static struct notifier_block trace_die_notifier = {
3598         .notifier_call = trace_die_handler,
3599         .priority = 200
3600 };
3601
3602 /*
3603  * printk is set to max of 1024, we really don't need it that big.
3604  * Nothing should be printing 1000 characters anyway.
3605  */
3606 #define TRACE_MAX_PRINT         1000
3607
3608 /*
3609  * Define here KERN_TRACE so that we have one place to modify
3610  * it if we decide to change what log level the ftrace dump
3611  * should be at.
3612  */
3613 #define KERN_TRACE              KERN_EMERG
3614
3615 static void
3616 trace_printk_seq(struct trace_seq *s)
3617 {
3618         /* Probably should print a warning here. */
3619         if (s->len >= 1000)
3620                 s->len = 1000;
3621
3622         /* should be zero ended, but we are paranoid. */
3623         s->buffer[s->len] = 0;
3624
3625         printk(KERN_TRACE "%s", s->buffer);
3626
3627         trace_seq_init(s);
3628 }
3629
3630 void ftrace_dump(void)
3631 {
3632         static DEFINE_SPINLOCK(ftrace_dump_lock);
3633         /* use static because iter can be a bit big for the stack */
3634         static struct trace_iterator iter;
3635         static int dump_ran;
3636         unsigned long flags;
3637         int cnt = 0, cpu;
3638
3639         /* only one dump */
3640         spin_lock_irqsave(&ftrace_dump_lock, flags);
3641         if (dump_ran)
3642                 goto out;
3643
3644         dump_ran = 1;
3645
3646         /* No turning back! */
3647         tracing_off();
3648         ftrace_kill();
3649
3650         for_each_tracing_cpu(cpu) {
3651                 atomic_inc(&global_trace.data[cpu]->disabled);
3652         }
3653
3654         /* don't look at user memory in panic mode */
3655         trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
3656
3657         printk(KERN_TRACE "Dumping ftrace buffer:\n");
3658
3659         iter.tr = &global_trace;
3660         iter.trace = current_trace;
3661
3662         /*
3663          * We need to stop all tracing on all CPUS to read the
3664          * the next buffer. This is a bit expensive, but is
3665          * not done often. We fill all what we can read,
3666          * and then release the locks again.
3667          */
3668
3669         while (!trace_empty(&iter)) {
3670
3671                 if (!cnt)
3672                         printk(KERN_TRACE "---------------------------------\n");
3673
3674                 cnt++;
3675
3676                 /* reset all but tr, trace, and overruns */
3677                 memset(&iter.seq, 0,
3678                        sizeof(struct trace_iterator) -
3679                        offsetof(struct trace_iterator, seq));
3680                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
3681                 iter.pos = -1;
3682
3683                 if (find_next_entry_inc(&iter) != NULL) {
3684                         print_trace_line(&iter);
3685                         trace_consume(&iter);
3686                 }
3687
3688                 trace_printk_seq(&iter.seq);
3689         }
3690
3691         if (!cnt)
3692                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
3693         else
3694                 printk(KERN_TRACE "---------------------------------\n");
3695
3696  out:
3697         spin_unlock_irqrestore(&ftrace_dump_lock, flags);
3698 }
3699
3700 __init static int tracer_alloc_buffers(void)
3701 {
3702         struct trace_array_cpu *data;
3703         int i;
3704         int ret = -ENOMEM;
3705
3706         if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
3707                 goto out;
3708
3709         if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
3710                 goto out_free_buffer_mask;
3711
3712         if (!alloc_cpumask_var(&tracing_reader_cpumask, GFP_KERNEL))
3713                 goto out_free_tracing_cpumask;
3714
3715         cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
3716         cpumask_copy(tracing_cpumask, cpu_all_mask);
3717         cpumask_clear(tracing_reader_cpumask);
3718
3719         /* TODO: make the number of buffers hot pluggable with CPUS */
3720         global_trace.buffer = ring_buffer_alloc(trace_buf_size,
3721                                                    TRACE_BUFFER_FLAGS);
3722         if (!global_trace.buffer) {
3723                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
3724                 WARN_ON(1);
3725                 goto out_free_cpumask;
3726         }
3727         global_trace.entries = ring_buffer_size(global_trace.buffer);
3728
3729
3730 #ifdef CONFIG_TRACER_MAX_TRACE
3731         max_tr.buffer = ring_buffer_alloc(trace_buf_size,
3732                                              TRACE_BUFFER_FLAGS);
3733         if (!max_tr.buffer) {
3734                 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
3735                 WARN_ON(1);
3736                 ring_buffer_free(global_trace.buffer);
3737                 goto out_free_cpumask;
3738         }
3739         max_tr.entries = ring_buffer_size(max_tr.buffer);
3740         WARN_ON(max_tr.entries != global_trace.entries);
3741 #endif
3742
3743         /* Allocate the first page for all buffers */
3744         for_each_tracing_cpu(i) {
3745                 data = global_trace.data[i] = &per_cpu(global_trace_cpu, i);
3746                 max_tr.data[i] = &per_cpu(max_data, i);
3747         }
3748
3749         trace_init_cmdlines();
3750
3751         register_tracer(&nop_trace);
3752         current_trace = &nop_trace;
3753 #ifdef CONFIG_BOOT_TRACER
3754         register_tracer(&boot_tracer);
3755 #endif
3756         /* All seems OK, enable tracing */
3757         tracing_disabled = 0;
3758
3759         atomic_notifier_chain_register(&panic_notifier_list,
3760                                        &trace_panic_notifier);
3761
3762         register_die_notifier(&trace_die_notifier);
3763         ret = 0;
3764
3765 out_free_cpumask:
3766         free_cpumask_var(tracing_reader_cpumask);
3767 out_free_tracing_cpumask:
3768         free_cpumask_var(tracing_cpumask);
3769 out_free_buffer_mask:
3770         free_cpumask_var(tracing_buffer_mask);
3771 out:
3772         return ret;
3773 }
3774
3775 __init static int clear_boot_tracer(void)
3776 {
3777         /*
3778          * The default tracer at boot buffer is an init section.
3779          * This function is called in lateinit. If we did not
3780          * find the boot tracer, then clear it out, to prevent
3781          * later registration from accessing the buffer that is
3782          * about to be freed.
3783          */
3784         if (!default_bootup_tracer)
3785                 return 0;
3786
3787         printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
3788                default_bootup_tracer);
3789         default_bootup_tracer = NULL;
3790
3791         return 0;
3792 }
3793
3794 early_initcall(tracer_alloc_buffers);
3795 fs_initcall(tracer_init_debugfs);
3796 late_initcall(clear_boot_tracer);