ftrace: Move setting of clock-source out of options
[linux-2.6.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 William Lee Irwin III
13  */
14 #include <linux/ring_buffer.h>
15 #include <linux/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/smp_lock.h>
21 #include <linux/notifier.h>
22 #include <linux/irqflags.h>
23 #include <linux/debugfs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/ctype.h>
36 #include <linux/init.h>
37 #include <linux/poll.h>
38 #include <linux/gfp.h>
39 #include <linux/fs.h>
40
41 #include "trace.h"
42 #include "trace_output.h"
43
44 #define TRACE_BUFFER_FLAGS      (RB_FL_OVERWRITE)
45
46 unsigned long __read_mostly     tracing_max_latency;
47 unsigned long __read_mostly     tracing_thresh;
48
49 /*
50  * On boot up, the ring buffer is set to the minimum size, so that
51  * we do not waste memory on systems that are not using tracing.
52  */
53 int ring_buffer_expanded;
54
55 /*
56  * We need to change this state when a selftest is running.
57  * A selftest will lurk into the ring-buffer to count the
58  * entries inserted during the selftest although some concurrent
59  * insertions into the ring-buffer such as trace_printk could occurred
60  * at the same time, giving false positive or negative results.
61  */
62 static bool __read_mostly tracing_selftest_running;
63
64 /*
65  * If a tracer is running, we do not want to run SELFTEST.
66  */
67 bool __read_mostly tracing_selftest_disabled;
68
69 /* For tracers that don't implement custom flags */
70 static struct tracer_opt dummy_tracer_opt[] = {
71         { }
72 };
73
74 static struct tracer_flags dummy_tracer_flags = {
75         .val = 0,
76         .opts = dummy_tracer_opt
77 };
78
79 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
80 {
81         return 0;
82 }
83
84 /*
85  * Kill all tracing for good (never come back).
86  * It is initialized to 1 but will turn to zero if the initialization
87  * of the tracer is successful. But that is the only place that sets
88  * this back to zero.
89  */
90 static int tracing_disabled = 1;
91
92 DEFINE_PER_CPU(local_t, ftrace_cpu_disabled);
93
94 static inline void ftrace_disable_cpu(void)
95 {
96         preempt_disable();
97         local_inc(&__get_cpu_var(ftrace_cpu_disabled));
98 }
99
100 static inline void ftrace_enable_cpu(void)
101 {
102         local_dec(&__get_cpu_var(ftrace_cpu_disabled));
103         preempt_enable();
104 }
105
106 static cpumask_var_t __read_mostly      tracing_buffer_mask;
107
108 /* Define which cpu buffers are currently read in trace_pipe */
109 static cpumask_var_t                    tracing_reader_cpumask;
110
111 #define for_each_tracing_cpu(cpu)       \
112         for_each_cpu(cpu, tracing_buffer_mask)
113
114 /*
115  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
116  *
117  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
118  * is set, then ftrace_dump is called. This will output the contents
119  * of the ftrace buffers to the console.  This is very useful for
120  * capturing traces that lead to crashes and outputing it to a
121  * serial console.
122  *
123  * It is default off, but you can enable it with either specifying
124  * "ftrace_dump_on_oops" in the kernel command line, or setting
125  * /proc/sys/kernel/ftrace_dump_on_oops to true.
126  */
127 int ftrace_dump_on_oops;
128
129 static int tracing_set_tracer(const char *buf);
130
131 #define BOOTUP_TRACER_SIZE              100
132 static char bootup_tracer_buf[BOOTUP_TRACER_SIZE] __initdata;
133 static char *default_bootup_tracer;
134
135 static int __init set_ftrace(char *str)
136 {
137         strncpy(bootup_tracer_buf, str, BOOTUP_TRACER_SIZE);
138         default_bootup_tracer = bootup_tracer_buf;
139         /* We are using ftrace early, expand it */
140         ring_buffer_expanded = 1;
141         return 1;
142 }
143 __setup("ftrace=", set_ftrace);
144
145 static int __init set_ftrace_dump_on_oops(char *str)
146 {
147         ftrace_dump_on_oops = 1;
148         return 1;
149 }
150 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
151
152 unsigned long long ns2usecs(cycle_t nsec)
153 {
154         nsec += 500;
155         do_div(nsec, 1000);
156         return nsec;
157 }
158
159 /*
160  * The global_trace is the descriptor that holds the tracing
161  * buffers for the live tracing. For each CPU, it contains
162  * a link list of pages that will store trace entries. The
163  * page descriptor of the pages in the memory is used to hold
164  * the link list by linking the lru item in the page descriptor
165  * to each of the pages in the buffer per CPU.
166  *
167  * For each active CPU there is a data field that holds the
168  * pages for the buffer for that CPU. Each CPU has the same number
169  * of pages allocated for its buffer.
170  */
171 static struct trace_array       global_trace;
172
173 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
174
175 int filter_current_check_discard(struct ftrace_event_call *call, void *rec,
176                                  struct ring_buffer_event *event)
177 {
178         return filter_check_discard(call, rec, global_trace.buffer, event);
179 }
180 EXPORT_SYMBOL_GPL(filter_current_check_discard);
181
182 cycle_t ftrace_now(int cpu)
183 {
184         u64 ts;
185
186         /* Early boot up does not have a buffer yet */
187         if (!global_trace.buffer)
188                 return trace_clock_local();
189
190         ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
191         ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
192
193         return ts;
194 }
195
196 /*
197  * The max_tr is used to snapshot the global_trace when a maximum
198  * latency is reached. Some tracers will use this to store a maximum
199  * trace while it continues examining live traces.
200  *
201  * The buffers for the max_tr are set up the same as the global_trace.
202  * When a snapshot is taken, the link list of the max_tr is swapped
203  * with the link list of the global_trace and the buffers are reset for
204  * the global_trace so the tracing can continue.
205  */
206 static struct trace_array       max_tr;
207
208 static DEFINE_PER_CPU(struct trace_array_cpu, max_data);
209
210 /* tracer_enabled is used to toggle activation of a tracer */
211 static int                      tracer_enabled = 1;
212
213 /**
214  * tracing_is_enabled - return tracer_enabled status
215  *
216  * This function is used by other tracers to know the status
217  * of the tracer_enabled flag.  Tracers may use this function
218  * to know if it should enable their features when starting
219  * up. See irqsoff tracer for an example (start_irqsoff_tracer).
220  */
221 int tracing_is_enabled(void)
222 {
223         return tracer_enabled;
224 }
225
226 /*
227  * trace_buf_size is the size in bytes that is allocated
228  * for a buffer. Note, the number of bytes is always rounded
229  * to page size.
230  *
231  * This number is purposely set to a low number of 16384.
232  * If the dump on oops happens, it will be much appreciated
233  * to not have to wait for all that output. Anyway this can be
234  * boot time and run time configurable.
235  */
236 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
237
238 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
239
240 /* trace_types holds a link list of available tracers. */
241 static struct tracer            *trace_types __read_mostly;
242
243 /* current_trace points to the tracer that is currently active */
244 static struct tracer            *current_trace __read_mostly;
245
246 /*
247  * max_tracer_type_len is used to simplify the allocating of
248  * buffers to read userspace tracer names. We keep track of
249  * the longest tracer name registered.
250  */
251 static int                      max_tracer_type_len;
252
253 /*
254  * trace_types_lock is used to protect the trace_types list.
255  * This lock is also used to keep user access serialized.
256  * Accesses from userspace will grab this lock while userspace
257  * activities happen inside the kernel.
258  */
259 static DEFINE_MUTEX(trace_types_lock);
260
261 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
262 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
263
264 /* trace_flags holds trace_options default values */
265 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
266         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
267         TRACE_ITER_GRAPH_TIME;
268
269 /**
270  * trace_wake_up - wake up tasks waiting for trace input
271  *
272  * Simply wakes up any task that is blocked on the trace_wait
273  * queue. These is used with trace_poll for tasks polling the trace.
274  */
275 void trace_wake_up(void)
276 {
277         /*
278          * The runqueue_is_locked() can fail, but this is the best we
279          * have for now:
280          */
281         if (!(trace_flags & TRACE_ITER_BLOCK) && !runqueue_is_locked())
282                 wake_up(&trace_wait);
283 }
284
285 static int __init set_buf_size(char *str)
286 {
287         unsigned long buf_size;
288
289         if (!str)
290                 return 0;
291         buf_size = memparse(str, &str);
292         /* nr_entries can not be zero */
293         if (buf_size == 0)
294                 return 0;
295         trace_buf_size = buf_size;
296         return 1;
297 }
298 __setup("trace_buf_size=", set_buf_size);
299
300 unsigned long nsecs_to_usecs(unsigned long nsecs)
301 {
302         return nsecs / 1000;
303 }
304
305 /* These must match the bit postions in trace_iterator_flags */
306 static const char *trace_options[] = {
307         "print-parent",
308         "sym-offset",
309         "sym-addr",
310         "verbose",
311         "raw",
312         "hex",
313         "bin",
314         "block",
315         "stacktrace",
316         "sched-tree",
317         "trace_printk",
318         "ftrace_preempt",
319         "branch",
320         "annotate",
321         "userstacktrace",
322         "sym-userobj",
323         "printk-msg-only",
324         "context-info",
325         "latency-format",
326         "sleep-time",
327         "graph-time",
328         NULL
329 };
330
331 static struct {
332         u64 (*func)(void);
333         const char *name;
334 } trace_clocks[] = {
335         { trace_clock_local,    "local" },
336         { trace_clock_global,   "global" },
337 };
338
339 int trace_clock_id;
340
341 /*
342  * ftrace_max_lock is used to protect the swapping of buffers
343  * when taking a max snapshot. The buffers themselves are
344  * protected by per_cpu spinlocks. But the action of the swap
345  * needs its own lock.
346  *
347  * This is defined as a raw_spinlock_t in order to help
348  * with performance when lockdep debugging is enabled.
349  */
350 static raw_spinlock_t ftrace_max_lock =
351         (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
352
353 /*
354  * Copy the new maximum trace into the separate maximum-trace
355  * structure. (this way the maximum trace is permanently saved,
356  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
357  */
358 static void
359 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
360 {
361         struct trace_array_cpu *data = tr->data[cpu];
362
363         max_tr.cpu = cpu;
364         max_tr.time_start = data->preempt_timestamp;
365
366         data = max_tr.data[cpu];
367         data->saved_latency = tracing_max_latency;
368
369         memcpy(data->comm, tsk->comm, TASK_COMM_LEN);
370         data->pid = tsk->pid;
371         data->uid = task_uid(tsk);
372         data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
373         data->policy = tsk->policy;
374         data->rt_priority = tsk->rt_priority;
375
376         /* record this tasks comm */
377         tracing_record_cmdline(tsk);
378 }
379
380 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
381 {
382         int len;
383         int ret;
384
385         if (!cnt)
386                 return 0;
387
388         if (s->len <= s->readpos)
389                 return -EBUSY;
390
391         len = s->len - s->readpos;
392         if (cnt > len)
393                 cnt = len;
394         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
395         if (ret == cnt)
396                 return -EFAULT;
397
398         cnt -= ret;
399
400         s->readpos += cnt;
401         return cnt;
402 }
403
404 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
405 {
406         int len;
407         void *ret;
408
409         if (s->len <= s->readpos)
410                 return -EBUSY;
411
412         len = s->len - s->readpos;
413         if (cnt > len)
414                 cnt = len;
415         ret = memcpy(buf, s->buffer + s->readpos, cnt);
416         if (!ret)
417                 return -EFAULT;
418
419         s->readpos += cnt;
420         return cnt;
421 }
422
423 /**
424  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
425  * @tr: tracer
426  * @tsk: the task with the latency
427  * @cpu: The cpu that initiated the trace.
428  *
429  * Flip the buffers between the @tr and the max_tr and record information
430  * about which task was the cause of this latency.
431  */
432 void
433 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
434 {
435         struct ring_buffer *buf = tr->buffer;
436
437         WARN_ON_ONCE(!irqs_disabled());
438         __raw_spin_lock(&ftrace_max_lock);
439
440         tr->buffer = max_tr.buffer;
441         max_tr.buffer = buf;
442
443         ftrace_disable_cpu();
444         ring_buffer_reset(tr->buffer);
445         ftrace_enable_cpu();
446
447         __update_max_tr(tr, tsk, cpu);
448         __raw_spin_unlock(&ftrace_max_lock);
449 }
450
451 /**
452  * update_max_tr_single - only copy one trace over, and reset the rest
453  * @tr - tracer
454  * @tsk - task with the latency
455  * @cpu - the cpu of the buffer to copy.
456  *
457  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
458  */
459 void
460 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
461 {
462         int ret;
463
464         WARN_ON_ONCE(!irqs_disabled());
465         __raw_spin_lock(&ftrace_max_lock);
466
467         ftrace_disable_cpu();
468
469         ring_buffer_reset(max_tr.buffer);
470         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
471
472         ftrace_enable_cpu();
473
474         WARN_ON_ONCE(ret && ret != -EAGAIN);
475
476         __update_max_tr(tr, tsk, cpu);
477         __raw_spin_unlock(&ftrace_max_lock);
478 }
479
480 /**
481  * register_tracer - register a tracer with the ftrace system.
482  * @type - the plugin for the tracer
483  *
484  * Register a new plugin tracer.
485  */
486 int register_tracer(struct tracer *type)
487 __releases(kernel_lock)
488 __acquires(kernel_lock)
489 {
490         struct tracer *t;
491         int len;
492         int ret = 0;
493
494         if (!type->name) {
495                 pr_info("Tracer must have a name\n");
496                 return -1;
497         }
498
499         /*
500          * When this gets called we hold the BKL which means that
501          * preemption is disabled. Various trace selftests however
502          * need to disable and enable preemption for successful tests.
503          * So we drop the BKL here and grab it after the tests again.
504          */
505         unlock_kernel();
506         mutex_lock(&trace_types_lock);
507
508         tracing_selftest_running = true;
509
510         for (t = trace_types; t; t = t->next) {
511                 if (strcmp(type->name, t->name) == 0) {
512                         /* already found */
513                         pr_info("Trace %s already registered\n",
514                                 type->name);
515                         ret = -1;
516                         goto out;
517                 }
518         }
519
520         if (!type->set_flag)
521                 type->set_flag = &dummy_set_flag;
522         if (!type->flags)
523                 type->flags = &dummy_tracer_flags;
524         else
525                 if (!type->flags->opts)
526                         type->flags->opts = dummy_tracer_opt;
527         if (!type->wait_pipe)
528                 type->wait_pipe = default_wait_pipe;
529
530
531 #ifdef CONFIG_FTRACE_STARTUP_TEST
532         if (type->selftest && !tracing_selftest_disabled) {
533                 struct tracer *saved_tracer = current_trace;
534                 struct trace_array *tr = &global_trace;
535                 int i;
536
537                 /*
538                  * Run a selftest on this tracer.
539                  * Here we reset the trace buffer, and set the current
540                  * tracer to be this tracer. The tracer can then run some
541                  * internal tracing to verify that everything is in order.
542                  * If we fail, we do not register this tracer.
543                  */
544                 for_each_tracing_cpu(i)
545                         tracing_reset(tr, i);
546
547                 current_trace = type;
548                 /* the test is responsible for initializing and enabling */
549                 pr_info("Testing tracer %s: ", type->name);
550                 ret = type->selftest(type, tr);
551                 /* the test is responsible for resetting too */
552                 current_trace = saved_tracer;
553                 if (ret) {
554                         printk(KERN_CONT "FAILED!\n");
555                         goto out;
556                 }
557                 /* Only reset on passing, to avoid touching corrupted buffers */
558                 for_each_tracing_cpu(i)
559                         tracing_reset(tr, i);
560
561                 printk(KERN_CONT "PASSED\n");
562         }
563 #endif
564
565         type->next = trace_types;
566         trace_types = type;
567         len = strlen(type->name);
568         if (len > max_tracer_type_len)
569                 max_tracer_type_len = len;
570
571  out:
572         tracing_selftest_running = false;
573         mutex_unlock(&trace_types_lock);
574
575         if (ret || !default_bootup_tracer)
576                 goto out_unlock;
577
578         if (strncmp(default_bootup_tracer, type->name, BOOTUP_TRACER_SIZE))
579                 goto out_unlock;
580
581         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
582         /* Do we want this tracer to start on bootup? */
583         tracing_set_tracer(type->name);
584         default_bootup_tracer = NULL;
585         /* disable other selftests, since this will break it. */
586         tracing_selftest_disabled = 1;
587 #ifdef CONFIG_FTRACE_STARTUP_TEST
588         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
589                type->name);
590 #endif
591
592  out_unlock:
593         lock_kernel();
594         return ret;
595 }
596
597 void unregister_tracer(struct tracer *type)
598 {
599         struct tracer **t;
600         int len;
601
602         mutex_lock(&trace_types_lock);
603         for (t = &trace_types; *t; t = &(*t)->next) {
604                 if (*t == type)
605                         goto found;
606         }
607         pr_info("Trace %s not registered\n", type->name);
608         goto out;
609
610  found:
611         *t = (*t)->next;
612
613         if (type == current_trace && tracer_enabled) {
614                 tracer_enabled = 0;
615                 tracing_stop();
616                 if (current_trace->stop)
617                         current_trace->stop(&global_trace);
618                 current_trace = &nop_trace;
619         }
620
621         if (strlen(type->name) != max_tracer_type_len)
622                 goto out;
623
624         max_tracer_type_len = 0;
625         for (t = &trace_types; *t; t = &(*t)->next) {
626                 len = strlen((*t)->name);
627                 if (len > max_tracer_type_len)
628                         max_tracer_type_len = len;
629         }
630  out:
631         mutex_unlock(&trace_types_lock);
632 }
633
634 void tracing_reset(struct trace_array *tr, int cpu)
635 {
636         ftrace_disable_cpu();
637         ring_buffer_reset_cpu(tr->buffer, cpu);
638         ftrace_enable_cpu();
639 }
640
641 void tracing_reset_online_cpus(struct trace_array *tr)
642 {
643         int cpu;
644
645         tr->time_start = ftrace_now(tr->cpu);
646
647         for_each_online_cpu(cpu)
648                 tracing_reset(tr, cpu);
649 }
650
651 void tracing_reset_current(int cpu)
652 {
653         tracing_reset(&global_trace, cpu);
654 }
655
656 void tracing_reset_current_online_cpus(void)
657 {
658         tracing_reset_online_cpus(&global_trace);
659 }
660
661 #define SAVED_CMDLINES 128
662 #define NO_CMDLINE_MAP UINT_MAX
663 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
664 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
665 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
666 static int cmdline_idx;
667 static raw_spinlock_t trace_cmdline_lock = __RAW_SPIN_LOCK_UNLOCKED;
668
669 /* temporary disable recording */
670 static atomic_t trace_record_cmdline_disabled __read_mostly;
671
672 static void trace_init_cmdlines(void)
673 {
674         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
675         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
676         cmdline_idx = 0;
677 }
678
679 static int trace_stop_count;
680 static DEFINE_SPINLOCK(tracing_start_lock);
681
682 /**
683  * ftrace_off_permanent - disable all ftrace code permanently
684  *
685  * This should only be called when a serious anomally has
686  * been detected.  This will turn off the function tracing,
687  * ring buffers, and other tracing utilites. It takes no
688  * locks and can be called from any context.
689  */
690 void ftrace_off_permanent(void)
691 {
692         tracing_disabled = 1;
693         ftrace_stop();
694         tracing_off_permanent();
695 }
696
697 /**
698  * tracing_start - quick start of the tracer
699  *
700  * If tracing is enabled but was stopped by tracing_stop,
701  * this will start the tracer back up.
702  */
703 void tracing_start(void)
704 {
705         struct ring_buffer *buffer;
706         unsigned long flags;
707
708         if (tracing_disabled)
709                 return;
710
711         spin_lock_irqsave(&tracing_start_lock, flags);
712         if (--trace_stop_count) {
713                 if (trace_stop_count < 0) {
714                         /* Someone screwed up their debugging */
715                         WARN_ON_ONCE(1);
716                         trace_stop_count = 0;
717                 }
718                 goto out;
719         }
720
721
722         buffer = global_trace.buffer;
723         if (buffer)
724                 ring_buffer_record_enable(buffer);
725
726         buffer = max_tr.buffer;
727         if (buffer)
728                 ring_buffer_record_enable(buffer);
729
730         ftrace_start();
731  out:
732         spin_unlock_irqrestore(&tracing_start_lock, flags);
733 }
734
735 /**
736  * tracing_stop - quick stop of the tracer
737  *
738  * Light weight way to stop tracing. Use in conjunction with
739  * tracing_start.
740  */
741 void tracing_stop(void)
742 {
743         struct ring_buffer *buffer;
744         unsigned long flags;
745
746         ftrace_stop();
747         spin_lock_irqsave(&tracing_start_lock, flags);
748         if (trace_stop_count++)
749                 goto out;
750
751         buffer = global_trace.buffer;
752         if (buffer)
753                 ring_buffer_record_disable(buffer);
754
755         buffer = max_tr.buffer;
756         if (buffer)
757                 ring_buffer_record_disable(buffer);
758
759  out:
760         spin_unlock_irqrestore(&tracing_start_lock, flags);
761 }
762
763 void trace_stop_cmdline_recording(void);
764
765 static void trace_save_cmdline(struct task_struct *tsk)
766 {
767         unsigned pid, idx;
768
769         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
770                 return;
771
772         /*
773          * It's not the end of the world if we don't get
774          * the lock, but we also don't want to spin
775          * nor do we want to disable interrupts,
776          * so if we miss here, then better luck next time.
777          */
778         if (!__raw_spin_trylock(&trace_cmdline_lock))
779                 return;
780
781         idx = map_pid_to_cmdline[tsk->pid];
782         if (idx == NO_CMDLINE_MAP) {
783                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
784
785                 /*
786                  * Check whether the cmdline buffer at idx has a pid
787                  * mapped. We are going to overwrite that entry so we
788                  * need to clear the map_pid_to_cmdline. Otherwise we
789                  * would read the new comm for the old pid.
790                  */
791                 pid = map_cmdline_to_pid[idx];
792                 if (pid != NO_CMDLINE_MAP)
793                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
794
795                 map_cmdline_to_pid[idx] = tsk->pid;
796                 map_pid_to_cmdline[tsk->pid] = idx;
797
798                 cmdline_idx = idx;
799         }
800
801         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
802
803         __raw_spin_unlock(&trace_cmdline_lock);
804 }
805
806 void trace_find_cmdline(int pid, char comm[])
807 {
808         unsigned map;
809
810         if (!pid) {
811                 strcpy(comm, "<idle>");
812                 return;
813         }
814
815         if (pid > PID_MAX_DEFAULT) {
816                 strcpy(comm, "<...>");
817                 return;
818         }
819
820         preempt_disable();
821         __raw_spin_lock(&trace_cmdline_lock);
822         map = map_pid_to_cmdline[pid];
823         if (map != NO_CMDLINE_MAP)
824                 strcpy(comm, saved_cmdlines[map]);
825         else
826                 strcpy(comm, "<...>");
827
828         __raw_spin_unlock(&trace_cmdline_lock);
829         preempt_enable();
830 }
831
832 void tracing_record_cmdline(struct task_struct *tsk)
833 {
834         if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
835             !tracing_is_on())
836                 return;
837
838         trace_save_cmdline(tsk);
839 }
840
841 void
842 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
843                              int pc)
844 {
845         struct task_struct *tsk = current;
846
847         entry->preempt_count            = pc & 0xff;
848         entry->pid                      = (tsk) ? tsk->pid : 0;
849         entry->tgid                     = (tsk) ? tsk->tgid : 0;
850         entry->flags =
851 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
852                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
853 #else
854                 TRACE_FLAG_IRQS_NOSUPPORT |
855 #endif
856                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
857                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
858                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
859 }
860 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
861
862 struct ring_buffer_event *trace_buffer_lock_reserve(struct trace_array *tr,
863                                                     int type,
864                                                     unsigned long len,
865                                                     unsigned long flags, int pc)
866 {
867         struct ring_buffer_event *event;
868
869         event = ring_buffer_lock_reserve(tr->buffer, len);
870         if (event != NULL) {
871                 struct trace_entry *ent = ring_buffer_event_data(event);
872
873                 tracing_generic_entry_update(ent, flags, pc);
874                 ent->type = type;
875         }
876
877         return event;
878 }
879
880 static inline void __trace_buffer_unlock_commit(struct trace_array *tr,
881                                         struct ring_buffer_event *event,
882                                         unsigned long flags, int pc,
883                                         int wake)
884 {
885         ring_buffer_unlock_commit(tr->buffer, event);
886
887         ftrace_trace_stack(tr, flags, 6, pc);
888         ftrace_trace_userstack(tr, flags, pc);
889
890         if (wake)
891                 trace_wake_up();
892 }
893
894 void trace_buffer_unlock_commit(struct trace_array *tr,
895                                         struct ring_buffer_event *event,
896                                         unsigned long flags, int pc)
897 {
898         __trace_buffer_unlock_commit(tr, event, flags, pc, 1);
899 }
900
901 struct ring_buffer_event *
902 trace_current_buffer_lock_reserve(int type, unsigned long len,
903                                   unsigned long flags, int pc)
904 {
905         return trace_buffer_lock_reserve(&global_trace,
906                                          type, len, flags, pc);
907 }
908 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
909
910 void trace_current_buffer_unlock_commit(struct ring_buffer_event *event,
911                                         unsigned long flags, int pc)
912 {
913         __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 1);
914 }
915 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
916
917 void trace_nowake_buffer_unlock_commit(struct ring_buffer_event *event,
918                                         unsigned long flags, int pc)
919 {
920         __trace_buffer_unlock_commit(&global_trace, event, flags, pc, 0);
921 }
922 EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
923
924 void trace_current_buffer_discard_commit(struct ring_buffer_event *event)
925 {
926         ring_buffer_discard_commit(global_trace.buffer, event);
927 }
928 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
929
930 void
931 trace_function(struct trace_array *tr,
932                unsigned long ip, unsigned long parent_ip, unsigned long flags,
933                int pc)
934 {
935         struct ftrace_event_call *call = &event_function;
936         struct ring_buffer_event *event;
937         struct ftrace_entry *entry;
938
939         /* If we are reading the ring buffer, don't trace */
940         if (unlikely(local_read(&__get_cpu_var(ftrace_cpu_disabled))))
941                 return;
942
943         event = trace_buffer_lock_reserve(tr, TRACE_FN, sizeof(*entry),
944                                           flags, pc);
945         if (!event)
946                 return;
947         entry   = ring_buffer_event_data(event);
948         entry->ip                       = ip;
949         entry->parent_ip                = parent_ip;
950
951         if (!filter_check_discard(call, entry, tr->buffer, event))
952                 ring_buffer_unlock_commit(tr->buffer, event);
953 }
954
955 void
956 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
957        unsigned long ip, unsigned long parent_ip, unsigned long flags,
958        int pc)
959 {
960         if (likely(!atomic_read(&data->disabled)))
961                 trace_function(tr, ip, parent_ip, flags, pc);
962 }
963
964 #ifdef CONFIG_STACKTRACE
965 static void __ftrace_trace_stack(struct trace_array *tr,
966                                  unsigned long flags,
967                                  int skip, int pc)
968 {
969         struct ftrace_event_call *call = &event_kernel_stack;
970         struct ring_buffer_event *event;
971         struct stack_entry *entry;
972         struct stack_trace trace;
973
974         event = trace_buffer_lock_reserve(tr, TRACE_STACK,
975                                           sizeof(*entry), flags, pc);
976         if (!event)
977                 return;
978         entry   = ring_buffer_event_data(event);
979         memset(&entry->caller, 0, sizeof(entry->caller));
980
981         trace.nr_entries        = 0;
982         trace.max_entries       = FTRACE_STACK_ENTRIES;
983         trace.skip              = skip;
984         trace.entries           = entry->caller;
985
986         save_stack_trace(&trace);
987         if (!filter_check_discard(call, entry, tr->buffer, event))
988                 ring_buffer_unlock_commit(tr->buffer, event);
989 }
990
991 void ftrace_trace_stack(struct trace_array *tr, unsigned long flags, int skip,
992                         int pc)
993 {
994         if (!(trace_flags & TRACE_ITER_STACKTRACE))
995                 return;
996
997         __ftrace_trace_stack(tr, flags, skip, pc);
998 }
999
1000 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1001                    int pc)
1002 {
1003         __ftrace_trace_stack(tr, flags, skip, pc);
1004 }
1005
1006 void ftrace_trace_userstack(struct trace_array *tr, unsigned long flags, int pc)
1007 {
1008         struct ftrace_event_call *call = &event_user_stack;
1009         struct ring_buffer_event *event;
1010         struct userstack_entry *entry;
1011         struct stack_trace trace;
1012
1013         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1014                 return;
1015
1016         event = trace_buffer_lock_reserve(tr, TRACE_USER_STACK,
1017                                           sizeof(*entry), flags, pc);
1018         if (!event)
1019                 return;
1020         entry   = ring_buffer_event_data(event);
1021
1022         memset(&entry->caller, 0, sizeof(entry->caller));
1023
1024         trace.nr_entries        = 0;
1025         trace.max_entries       = FTRACE_STACK_ENTRIES;
1026         trace.skip              = 0;
1027         trace.entries           = entry->caller;
1028
1029         save_stack_trace_user(&trace);
1030         if (!filter_check_discard(call, entry, tr->buffer, event))
1031                 ring_buffer_unlock_commit(tr->buffer, event);
1032 }
1033
1034 #ifdef UNUSED
1035 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1036 {
1037         ftrace_trace_userstack(tr, flags, preempt_count());
1038 }
1039 #endif /* UNUSED */
1040
1041 #endif /* CONFIG_STACKTRACE */
1042
1043 static void
1044 ftrace_trace_special(void *__tr,
1045                      unsigned long arg1, unsigned long arg2, unsigned long arg3,
1046                      int pc)
1047 {
1048         struct ring_buffer_event *event;
1049         struct trace_array *tr = __tr;
1050         struct special_entry *entry;
1051
1052         event = trace_buffer_lock_reserve(tr, TRACE_SPECIAL,
1053                                           sizeof(*entry), 0, pc);
1054         if (!event)
1055                 return;
1056         entry   = ring_buffer_event_data(event);
1057         entry->arg1                     = arg1;
1058         entry->arg2                     = arg2;
1059         entry->arg3                     = arg3;
1060         trace_buffer_unlock_commit(tr, event, 0, pc);
1061 }
1062
1063 void
1064 __trace_special(void *__tr, void *__data,
1065                 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1066 {
1067         ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
1068 }
1069
1070 void
1071 ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1072 {
1073         struct trace_array *tr = &global_trace;
1074         struct trace_array_cpu *data;
1075         unsigned long flags;
1076         int cpu;
1077         int pc;
1078
1079         if (tracing_disabled)
1080                 return;
1081
1082         pc = preempt_count();
1083         local_irq_save(flags);
1084         cpu = raw_smp_processor_id();
1085         data = tr->data[cpu];
1086
1087         if (likely(atomic_inc_return(&data->disabled) == 1))
1088                 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
1089
1090         atomic_dec(&data->disabled);
1091         local_irq_restore(flags);
1092 }
1093
1094 /**
1095  * trace_vbprintk - write binary msg to tracing buffer
1096  *
1097  */
1098 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1099 {
1100         static raw_spinlock_t trace_buf_lock =
1101                 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
1102         static u32 trace_buf[TRACE_BUF_SIZE];
1103
1104         struct ftrace_event_call *call = &event_bprint;
1105         struct ring_buffer_event *event;
1106         struct trace_array *tr = &global_trace;
1107         struct trace_array_cpu *data;
1108         struct bprint_entry *entry;
1109         unsigned long flags;
1110         int disable;
1111         int resched;
1112         int cpu, len = 0, size, pc;
1113
1114         if (unlikely(tracing_selftest_running || tracing_disabled))
1115                 return 0;
1116
1117         /* Don't pollute graph traces with trace_vprintk internals */
1118         pause_graph_tracing();
1119
1120         pc = preempt_count();
1121         resched = ftrace_preempt_disable();
1122         cpu = raw_smp_processor_id();
1123         data = tr->data[cpu];
1124
1125         disable = atomic_inc_return(&data->disabled);
1126         if (unlikely(disable != 1))
1127                 goto out;
1128
1129         /* Lockdep uses trace_printk for lock tracing */
1130         local_irq_save(flags);
1131         __raw_spin_lock(&trace_buf_lock);
1132         len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1133
1134         if (len > TRACE_BUF_SIZE || len < 0)
1135                 goto out_unlock;
1136
1137         size = sizeof(*entry) + sizeof(u32) * len;
1138         event = trace_buffer_lock_reserve(tr, TRACE_BPRINT, size, flags, pc);
1139         if (!event)
1140                 goto out_unlock;
1141         entry = ring_buffer_event_data(event);
1142         entry->ip                       = ip;
1143         entry->fmt                      = fmt;
1144
1145         memcpy(entry->buf, trace_buf, sizeof(u32) * len);
1146         if (!filter_check_discard(call, entry, tr->buffer, event))
1147                 ring_buffer_unlock_commit(tr->buffer, event);
1148
1149 out_unlock:
1150         __raw_spin_unlock(&trace_buf_lock);
1151         local_irq_restore(flags);
1152
1153 out:
1154         atomic_dec_return(&data->disabled);
1155         ftrace_preempt_enable(resched);
1156         unpause_graph_tracing();
1157
1158         return len;
1159 }
1160 EXPORT_SYMBOL_GPL(trace_vbprintk);
1161
1162 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1163 {
1164         static raw_spinlock_t trace_buf_lock = __RAW_SPIN_LOCK_UNLOCKED;
1165         static char trace_buf[TRACE_BUF_SIZE];
1166
1167         struct ftrace_event_call *call = &event_print;
1168         struct ring_buffer_event *event;
1169         struct trace_array *tr = &global_trace;
1170         struct trace_array_cpu *data;
1171         int cpu, len = 0, size, pc;
1172         struct print_entry *entry;
1173         unsigned long irq_flags;
1174         int disable;
1175
1176         if (tracing_disabled || tracing_selftest_running)
1177                 return 0;
1178
1179         pc = preempt_count();
1180         preempt_disable_notrace();
1181         cpu = raw_smp_processor_id();
1182         data = tr->data[cpu];
1183
1184         disable = atomic_inc_return(&data->disabled);
1185         if (unlikely(disable != 1))
1186                 goto out;
1187
1188         pause_graph_tracing();
1189         raw_local_irq_save(irq_flags);
1190         __raw_spin_lock(&trace_buf_lock);
1191         len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1192
1193         len = min(len, TRACE_BUF_SIZE-1);
1194         trace_buf[len] = 0;
1195
1196         size = sizeof(*entry) + len + 1;
1197         event = trace_buffer_lock_reserve(tr, TRACE_PRINT, size, irq_flags, pc);
1198         if (!event)
1199                 goto out_unlock;
1200         entry = ring_buffer_event_data(event);
1201         entry->ip                       = ip;
1202
1203         memcpy(&entry->buf, trace_buf, len);
1204         entry->buf[len] = 0;
1205         if (!filter_check_discard(call, entry, tr->buffer, event))
1206                 ring_buffer_unlock_commit(tr->buffer, event);
1207
1208  out_unlock:
1209         __raw_spin_unlock(&trace_buf_lock);
1210         raw_local_irq_restore(irq_flags);
1211         unpause_graph_tracing();
1212  out:
1213         atomic_dec_return(&data->disabled);
1214         preempt_enable_notrace();
1215
1216         return len;
1217 }
1218 EXPORT_SYMBOL_GPL(trace_vprintk);
1219
1220 enum trace_file_type {
1221         TRACE_FILE_LAT_FMT      = 1,
1222         TRACE_FILE_ANNOTATE     = 2,
1223 };
1224
1225 static void trace_iterator_increment(struct trace_iterator *iter)
1226 {
1227         /* Don't allow ftrace to trace into the ring buffers */
1228         ftrace_disable_cpu();
1229
1230         iter->idx++;
1231         if (iter->buffer_iter[iter->cpu])
1232                 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1233
1234         ftrace_enable_cpu();
1235 }
1236
1237 static struct trace_entry *
1238 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts)
1239 {
1240         struct ring_buffer_event *event;
1241         struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
1242
1243         /* Don't allow ftrace to trace into the ring buffers */
1244         ftrace_disable_cpu();
1245
1246         if (buf_iter)
1247                 event = ring_buffer_iter_peek(buf_iter, ts);
1248         else
1249                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts);
1250
1251         ftrace_enable_cpu();
1252
1253         return event ? ring_buffer_event_data(event) : NULL;
1254 }
1255
1256 static struct trace_entry *
1257 __find_next_entry(struct trace_iterator *iter, int *ent_cpu, u64 *ent_ts)
1258 {
1259         struct ring_buffer *buffer = iter->tr->buffer;
1260         struct trace_entry *ent, *next = NULL;
1261         int cpu_file = iter->cpu_file;
1262         u64 next_ts = 0, ts;
1263         int next_cpu = -1;
1264         int cpu;
1265
1266         /*
1267          * If we are in a per_cpu trace file, don't bother by iterating over
1268          * all cpu and peek directly.
1269          */
1270         if (cpu_file > TRACE_PIPE_ALL_CPU) {
1271                 if (ring_buffer_empty_cpu(buffer, cpu_file))
1272                         return NULL;
1273                 ent = peek_next_entry(iter, cpu_file, ent_ts);
1274                 if (ent_cpu)
1275                         *ent_cpu = cpu_file;
1276
1277                 return ent;
1278         }
1279
1280         for_each_tracing_cpu(cpu) {
1281
1282                 if (ring_buffer_empty_cpu(buffer, cpu))
1283                         continue;
1284
1285                 ent = peek_next_entry(iter, cpu, &ts);
1286
1287                 /*
1288                  * Pick the entry with the smallest timestamp:
1289                  */
1290                 if (ent && (!next || ts < next_ts)) {
1291                         next = ent;
1292                         next_cpu = cpu;
1293                         next_ts = ts;
1294                 }
1295         }
1296
1297         if (ent_cpu)
1298                 *ent_cpu = next_cpu;
1299
1300         if (ent_ts)
1301                 *ent_ts = next_ts;
1302
1303         return next;
1304 }
1305
1306 /* Find the next real entry, without updating the iterator itself */
1307 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1308                                           int *ent_cpu, u64 *ent_ts)
1309 {
1310         return __find_next_entry(iter, ent_cpu, ent_ts);
1311 }
1312
1313 /* Find the next real entry, and increment the iterator to the next entry */
1314 static void *find_next_entry_inc(struct trace_iterator *iter)
1315 {
1316         iter->ent = __find_next_entry(iter, &iter->cpu, &iter->ts);
1317
1318         if (iter->ent)
1319                 trace_iterator_increment(iter);
1320
1321         return iter->ent ? iter : NULL;
1322 }
1323
1324 static void trace_consume(struct trace_iterator *iter)
1325 {
1326         /* Don't allow ftrace to trace into the ring buffers */
1327         ftrace_disable_cpu();
1328         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts);
1329         ftrace_enable_cpu();
1330 }
1331
1332 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1333 {
1334         struct trace_iterator *iter = m->private;
1335         int i = (int)*pos;
1336         void *ent;
1337
1338         (*pos)++;
1339
1340         /* can't go backwards */
1341         if (iter->idx > i)
1342                 return NULL;
1343
1344         if (iter->idx < 0)
1345                 ent = find_next_entry_inc(iter);
1346         else
1347                 ent = iter;
1348
1349         while (ent && iter->idx < i)
1350                 ent = find_next_entry_inc(iter);
1351
1352         iter->pos = *pos;
1353
1354         return ent;
1355 }
1356
1357 /*
1358  * No necessary locking here. The worst thing which can
1359  * happen is loosing events consumed at the same time
1360  * by a trace_pipe reader.
1361  * Other than that, we don't risk to crash the ring buffer
1362  * because it serializes the readers.
1363  *
1364  * The current tracer is copied to avoid a global locking
1365  * all around.
1366  */
1367 static void *s_start(struct seq_file *m, loff_t *pos)
1368 {
1369         struct trace_iterator *iter = m->private;
1370         static struct tracer *old_tracer;
1371         int cpu_file = iter->cpu_file;
1372         void *p = NULL;
1373         loff_t l = 0;
1374         int cpu;
1375
1376         /* copy the tracer to avoid using a global lock all around */
1377         mutex_lock(&trace_types_lock);
1378         if (unlikely(old_tracer != current_trace && current_trace)) {
1379                 old_tracer = current_trace;
1380                 *iter->trace = *current_trace;
1381         }
1382         mutex_unlock(&trace_types_lock);
1383
1384         atomic_inc(&trace_record_cmdline_disabled);
1385
1386         if (*pos != iter->pos) {
1387                 iter->ent = NULL;
1388                 iter->cpu = 0;
1389                 iter->idx = -1;
1390
1391                 ftrace_disable_cpu();
1392
1393                 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1394                         for_each_tracing_cpu(cpu)
1395                                 ring_buffer_iter_reset(iter->buffer_iter[cpu]);
1396                 } else
1397                         ring_buffer_iter_reset(iter->buffer_iter[cpu_file]);
1398
1399
1400                 ftrace_enable_cpu();
1401
1402                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1403                         ;
1404
1405         } else {
1406                 l = *pos - 1;
1407                 p = s_next(m, p, &l);
1408         }
1409
1410         trace_event_read_lock();
1411         return p;
1412 }
1413
1414 static void s_stop(struct seq_file *m, void *p)
1415 {
1416         atomic_dec(&trace_record_cmdline_disabled);
1417         trace_event_read_unlock();
1418 }
1419
1420 static void print_lat_help_header(struct seq_file *m)
1421 {
1422         seq_puts(m, "#                  _------=> CPU#            \n");
1423         seq_puts(m, "#                 / _-----=> irqs-off        \n");
1424         seq_puts(m, "#                | / _----=> need-resched    \n");
1425         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
1426         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
1427         seq_puts(m, "#                |||| /                      \n");
1428         seq_puts(m, "#                |||||     delay             \n");
1429         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
1430         seq_puts(m, "#     \\   /      |||||   \\   |   /           \n");
1431 }
1432
1433 static void print_func_help_header(struct seq_file *m)
1434 {
1435         seq_puts(m, "#           TASK-PID    CPU#    TIMESTAMP  FUNCTION\n");
1436         seq_puts(m, "#              | |       |          |         |\n");
1437 }
1438
1439
1440 static void
1441 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1442 {
1443         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1444         struct trace_array *tr = iter->tr;
1445         struct trace_array_cpu *data = tr->data[tr->cpu];
1446         struct tracer *type = current_trace;
1447         unsigned long total;
1448         unsigned long entries;
1449         const char *name = "preemption";
1450
1451         if (type)
1452                 name = type->name;
1453
1454         entries = ring_buffer_entries(iter->tr->buffer);
1455         total = entries +
1456                 ring_buffer_overruns(iter->tr->buffer);
1457
1458         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
1459                    name, UTS_RELEASE);
1460         seq_puts(m, "# -----------------------------------"
1461                  "---------------------------------\n");
1462         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
1463                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
1464                    nsecs_to_usecs(data->saved_latency),
1465                    entries,
1466                    total,
1467                    tr->cpu,
1468 #if defined(CONFIG_PREEMPT_NONE)
1469                    "server",
1470 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
1471                    "desktop",
1472 #elif defined(CONFIG_PREEMPT)
1473                    "preempt",
1474 #else
1475                    "unknown",
1476 #endif
1477                    /* These are reserved for later use */
1478                    0, 0, 0, 0);
1479 #ifdef CONFIG_SMP
1480         seq_printf(m, " #P:%d)\n", num_online_cpus());
1481 #else
1482         seq_puts(m, ")\n");
1483 #endif
1484         seq_puts(m, "#    -----------------\n");
1485         seq_printf(m, "#    | task: %.16s-%d "
1486                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1487                    data->comm, data->pid, data->uid, data->nice,
1488                    data->policy, data->rt_priority);
1489         seq_puts(m, "#    -----------------\n");
1490
1491         if (data->critical_start) {
1492                 seq_puts(m, "#  => started at: ");
1493                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1494                 trace_print_seq(m, &iter->seq);
1495                 seq_puts(m, "\n#  => ended at:   ");
1496                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1497                 trace_print_seq(m, &iter->seq);
1498                 seq_puts(m, "#\n");
1499         }
1500
1501         seq_puts(m, "#\n");
1502 }
1503
1504 static void test_cpu_buff_start(struct trace_iterator *iter)
1505 {
1506         struct trace_seq *s = &iter->seq;
1507
1508         if (!(trace_flags & TRACE_ITER_ANNOTATE))
1509                 return;
1510
1511         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1512                 return;
1513
1514         if (cpumask_test_cpu(iter->cpu, iter->started))
1515                 return;
1516
1517         cpumask_set_cpu(iter->cpu, iter->started);
1518
1519         /* Don't print started cpu buffer for the first entry of the trace */
1520         if (iter->idx > 1)
1521                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1522                                 iter->cpu);
1523 }
1524
1525 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
1526 {
1527         struct trace_seq *s = &iter->seq;
1528         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1529         struct trace_entry *entry;
1530         struct trace_event *event;
1531
1532         entry = iter->ent;
1533
1534         test_cpu_buff_start(iter);
1535
1536         event = ftrace_find_event(entry->type);
1537
1538         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1539                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1540                         if (!trace_print_lat_context(iter))
1541                                 goto partial;
1542                 } else {
1543                         if (!trace_print_context(iter))
1544                                 goto partial;
1545                 }
1546         }
1547
1548         if (event)
1549                 return event->trace(iter, sym_flags);
1550
1551         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1552                 goto partial;
1553
1554         return TRACE_TYPE_HANDLED;
1555 partial:
1556         return TRACE_TYPE_PARTIAL_LINE;
1557 }
1558
1559 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
1560 {
1561         struct trace_seq *s = &iter->seq;
1562         struct trace_entry *entry;
1563         struct trace_event *event;
1564
1565         entry = iter->ent;
1566
1567         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1568                 if (!trace_seq_printf(s, "%d %d %llu ",
1569                                       entry->pid, iter->cpu, iter->ts))
1570                         goto partial;
1571         }
1572
1573         event = ftrace_find_event(entry->type);
1574         if (event)
1575                 return event->raw(iter, 0);
1576
1577         if (!trace_seq_printf(s, "%d ?\n", entry->type))
1578                 goto partial;
1579
1580         return TRACE_TYPE_HANDLED;
1581 partial:
1582         return TRACE_TYPE_PARTIAL_LINE;
1583 }
1584
1585 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
1586 {
1587         struct trace_seq *s = &iter->seq;
1588         unsigned char newline = '\n';
1589         struct trace_entry *entry;
1590         struct trace_event *event;
1591
1592         entry = iter->ent;
1593
1594         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1595                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1596                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1597                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1598         }
1599
1600         event = ftrace_find_event(entry->type);
1601         if (event) {
1602                 enum print_line_t ret = event->hex(iter, 0);
1603                 if (ret != TRACE_TYPE_HANDLED)
1604                         return ret;
1605         }
1606
1607         SEQ_PUT_FIELD_RET(s, newline);
1608
1609         return TRACE_TYPE_HANDLED;
1610 }
1611
1612 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
1613 {
1614         struct trace_seq *s = &iter->seq;
1615         struct trace_entry *entry;
1616         struct trace_event *event;
1617
1618         entry = iter->ent;
1619
1620         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1621                 SEQ_PUT_FIELD_RET(s, entry->pid);
1622                 SEQ_PUT_FIELD_RET(s, iter->cpu);
1623                 SEQ_PUT_FIELD_RET(s, iter->ts);
1624         }
1625
1626         event = ftrace_find_event(entry->type);
1627         return event ? event->binary(iter, 0) : TRACE_TYPE_HANDLED;
1628 }
1629
1630 static int trace_empty(struct trace_iterator *iter)
1631 {
1632         int cpu;
1633
1634         /* If we are looking at one CPU buffer, only check that one */
1635         if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
1636                 cpu = iter->cpu_file;
1637                 if (iter->buffer_iter[cpu]) {
1638                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1639                                 return 0;
1640                 } else {
1641                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1642                                 return 0;
1643                 }
1644                 return 1;
1645         }
1646
1647         for_each_tracing_cpu(cpu) {
1648                 if (iter->buffer_iter[cpu]) {
1649                         if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
1650                                 return 0;
1651                 } else {
1652                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
1653                                 return 0;
1654                 }
1655         }
1656
1657         return 1;
1658 }
1659
1660 /*  Called with trace_event_read_lock() held. */
1661 static enum print_line_t print_trace_line(struct trace_iterator *iter)
1662 {
1663         enum print_line_t ret;
1664
1665         if (iter->trace && iter->trace->print_line) {
1666                 ret = iter->trace->print_line(iter);
1667                 if (ret != TRACE_TYPE_UNHANDLED)
1668                         return ret;
1669         }
1670
1671         if (iter->ent->type == TRACE_BPRINT &&
1672                         trace_flags & TRACE_ITER_PRINTK &&
1673                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
1674                 return trace_print_bprintk_msg_only(iter);
1675
1676         if (iter->ent->type == TRACE_PRINT &&
1677                         trace_flags & TRACE_ITER_PRINTK &&
1678                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
1679                 return trace_print_printk_msg_only(iter);
1680
1681         if (trace_flags & TRACE_ITER_BIN)
1682                 return print_bin_fmt(iter);
1683
1684         if (trace_flags & TRACE_ITER_HEX)
1685                 return print_hex_fmt(iter);
1686
1687         if (trace_flags & TRACE_ITER_RAW)
1688                 return print_raw_fmt(iter);
1689
1690         return print_trace_fmt(iter);
1691 }
1692
1693 static int s_show(struct seq_file *m, void *v)
1694 {
1695         struct trace_iterator *iter = v;
1696
1697         if (iter->ent == NULL) {
1698                 if (iter->tr) {
1699                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
1700                         seq_puts(m, "#\n");
1701                 }
1702                 if (iter->trace && iter->trace->print_header)
1703                         iter->trace->print_header(m);
1704                 else if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1705                         /* print nothing if the buffers are empty */
1706                         if (trace_empty(iter))
1707                                 return 0;
1708                         print_trace_header(m, iter);
1709                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1710                                 print_lat_help_header(m);
1711                 } else {
1712                         if (!(trace_flags & TRACE_ITER_VERBOSE))
1713                                 print_func_help_header(m);
1714                 }
1715         } else {
1716                 print_trace_line(iter);
1717                 trace_print_seq(m, &iter->seq);
1718         }
1719
1720         return 0;
1721 }
1722
1723 static struct seq_operations tracer_seq_ops = {
1724         .start          = s_start,
1725         .next           = s_next,
1726         .stop           = s_stop,
1727         .show           = s_show,
1728 };
1729
1730 static struct trace_iterator *
1731 __tracing_open(struct inode *inode, struct file *file)
1732 {
1733         long cpu_file = (long) inode->i_private;
1734         void *fail_ret = ERR_PTR(-ENOMEM);
1735         struct trace_iterator *iter;
1736         struct seq_file *m;
1737         int cpu, ret;
1738
1739         if (tracing_disabled)
1740                 return ERR_PTR(-ENODEV);
1741
1742         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1743         if (!iter)
1744                 return ERR_PTR(-ENOMEM);
1745
1746         /*
1747          * We make a copy of the current tracer to avoid concurrent
1748          * changes on it while we are reading.
1749          */
1750         mutex_lock(&trace_types_lock);
1751         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
1752         if (!iter->trace)
1753                 goto fail;
1754
1755         if (current_trace)
1756                 *iter->trace = *current_trace;
1757
1758         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL))
1759                 goto fail;
1760
1761         cpumask_clear(iter->started);
1762
1763         if (current_trace && current_trace->print_max)
1764                 iter->tr = &max_tr;
1765         else
1766                 iter->tr = &global_trace;
1767         iter->pos = -1;
1768         mutex_init(&iter->mutex);
1769         iter->cpu_file = cpu_file;
1770
1771         /* Notify the tracer early; before we stop tracing. */
1772         if (iter->trace && iter->trace->open)
1773                 iter->trace->open(iter);
1774
1775         /* Annotate start of buffers if we had overruns */
1776         if (ring_buffer_overruns(iter->tr->buffer))
1777                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
1778
1779         if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
1780                 for_each_tracing_cpu(cpu) {
1781
1782                         iter->buffer_iter[cpu] =
1783                                 ring_buffer_read_start(iter->tr->buffer, cpu);
1784                 }
1785         } else {
1786                 cpu = iter->cpu_file;
1787                 iter->buffer_iter[cpu] =
1788                                 ring_buffer_read_start(iter->tr->buffer, cpu);
1789         }
1790
1791         /* TODO stop tracer */
1792         ret = seq_open(file, &tracer_seq_ops);
1793         if (ret < 0) {
1794                 fail_ret = ERR_PTR(ret);
1795                 goto fail_buffer;
1796         }
1797
1798         m = file->private_data;
1799         m->private = iter;
1800
1801         /* stop the trace while dumping */
1802         tracing_stop();
1803
1804         mutex_unlock(&trace_types_lock);
1805
1806         return iter;
1807
1808  fail_buffer:
1809         for_each_tracing_cpu(cpu) {
1810                 if (iter->buffer_iter[cpu])
1811                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1812         }
1813         free_cpumask_var(iter->started);
1814  fail:
1815         mutex_unlock(&trace_types_lock);
1816         kfree(iter->trace);
1817         kfree(iter);
1818
1819         return fail_ret;
1820 }
1821
1822 int tracing_open_generic(struct inode *inode, struct file *filp)
1823 {
1824         if (tracing_disabled)
1825                 return -ENODEV;
1826
1827         filp->private_data = inode->i_private;
1828         return 0;
1829 }
1830
1831 static int tracing_release(struct inode *inode, struct file *file)
1832 {
1833         struct seq_file *m = (struct seq_file *)file->private_data;
1834         struct trace_iterator *iter;
1835         int cpu;
1836
1837         if (!(file->f_mode & FMODE_READ))
1838                 return 0;
1839
1840         iter = m->private;
1841
1842         mutex_lock(&trace_types_lock);
1843         for_each_tracing_cpu(cpu) {
1844                 if (iter->buffer_iter[cpu])
1845                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
1846         }
1847
1848         if (iter->trace && iter->trace->close)
1849                 iter->trace->close(iter);
1850
1851         /* reenable tracing if it was previously enabled */
1852         tracing_start();
1853         mutex_unlock(&trace_types_lock);
1854
1855         seq_release(inode, file);
1856         mutex_destroy(&iter->mutex);
1857         free_cpumask_var(iter->started);
1858         kfree(iter->trace);
1859         kfree(iter);
1860         return 0;
1861 }
1862
1863 static int tracing_open(struct inode *inode, struct file *file)
1864 {
1865         struct trace_iterator *iter;
1866         int ret = 0;
1867
1868         /* If this file was open for write, then erase contents */
1869         if ((file->f_mode & FMODE_WRITE) &&
1870             (file->f_flags & O_TRUNC)) {
1871                 long cpu = (long) inode->i_private;
1872
1873                 if (cpu == TRACE_PIPE_ALL_CPU)
1874                         tracing_reset_online_cpus(&global_trace);
1875                 else
1876                         tracing_reset(&global_trace, cpu);
1877         }
1878
1879         if (file->f_mode & FMODE_READ) {
1880                 iter = __tracing_open(inode, file);
1881                 if (IS_ERR(iter))
1882                         ret = PTR_ERR(iter);
1883                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
1884                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
1885         }
1886         return ret;
1887 }
1888
1889 static void *
1890 t_next(struct seq_file *m, void *v, loff_t *pos)
1891 {
1892         struct tracer *t = v;
1893
1894         (*pos)++;
1895
1896         if (t)
1897                 t = t->next;
1898
1899         return t;
1900 }
1901
1902 static void *t_start(struct seq_file *m, loff_t *pos)
1903 {
1904         struct tracer *t;
1905         loff_t l = 0;
1906
1907         mutex_lock(&trace_types_lock);
1908         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
1909                 ;
1910
1911         return t;
1912 }
1913
1914 static void t_stop(struct seq_file *m, void *p)
1915 {
1916         mutex_unlock(&trace_types_lock);
1917 }
1918
1919 static int t_show(struct seq_file *m, void *v)
1920 {
1921         struct tracer *t = v;
1922
1923         if (!t)
1924                 return 0;
1925
1926         seq_printf(m, "%s", t->name);
1927         if (t->next)
1928                 seq_putc(m, ' ');
1929         else
1930                 seq_putc(m, '\n');
1931
1932         return 0;
1933 }
1934
1935 static struct seq_operations show_traces_seq_ops = {
1936         .start          = t_start,
1937         .next           = t_next,
1938         .stop           = t_stop,
1939         .show           = t_show,
1940 };
1941
1942 static int show_traces_open(struct inode *inode, struct file *file)
1943 {
1944         if (tracing_disabled)
1945                 return -ENODEV;
1946
1947         return seq_open(file, &show_traces_seq_ops);
1948 }
1949
1950 static ssize_t
1951 tracing_write_stub(struct file *filp, const char __user *ubuf,
1952                    size_t count, loff_t *ppos)
1953 {
1954         return count;
1955 }
1956
1957 static const struct file_operations tracing_fops = {
1958         .open           = tracing_open,
1959         .read           = seq_read,
1960         .write          = tracing_write_stub,
1961         .llseek         = seq_lseek,
1962         .release        = tracing_release,
1963 };
1964
1965 static const struct file_operations show_traces_fops = {
1966         .open           = show_traces_open,
1967         .read           = seq_read,
1968         .release        = seq_release,
1969 };
1970
1971 /*
1972  * Only trace on a CPU if the bitmask is set:
1973  */
1974 static cpumask_var_t tracing_cpumask;
1975
1976 /*
1977  * The tracer itself will not take this lock, but still we want
1978  * to provide a consistent cpumask to user-space:
1979  */
1980 static DEFINE_MUTEX(tracing_cpumask_update_lock);
1981
1982 /*
1983  * Temporary storage for the character representation of the
1984  * CPU bitmask (and one more byte for the newline):
1985  */
1986 static char mask_str[NR_CPUS + 1];
1987
1988 static ssize_t
1989 tracing_cpumask_read(struct file *filp, char __user *ubuf,
1990                      size_t count, loff_t *ppos)
1991 {
1992         int len;
1993
1994         mutex_lock(&tracing_cpumask_update_lock);
1995
1996         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
1997         if (count - len < 2) {
1998                 count = -EINVAL;
1999                 goto out_err;
2000         }
2001         len += sprintf(mask_str + len, "\n");
2002         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2003
2004 out_err:
2005         mutex_unlock(&tracing_cpumask_update_lock);
2006
2007         return count;
2008 }
2009
2010 static ssize_t
2011 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2012                       size_t count, loff_t *ppos)
2013 {
2014         int err, cpu;
2015         cpumask_var_t tracing_cpumask_new;
2016
2017         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2018                 return -ENOMEM;
2019
2020         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2021         if (err)
2022                 goto err_unlock;
2023
2024         mutex_lock(&tracing_cpumask_update_lock);
2025
2026         local_irq_disable();
2027         __raw_spin_lock(&ftrace_max_lock);
2028         for_each_tracing_cpu(cpu) {
2029                 /*
2030                  * Increase/decrease the disabled counter if we are
2031                  * about to flip a bit in the cpumask:
2032                  */
2033                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2034                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2035                         atomic_inc(&global_trace.data[cpu]->disabled);
2036                 }
2037                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2038                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2039                         atomic_dec(&global_trace.data[cpu]->disabled);
2040                 }
2041         }
2042         __raw_spin_unlock(&ftrace_max_lock);
2043         local_irq_enable();
2044
2045         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2046
2047         mutex_unlock(&tracing_cpumask_update_lock);
2048         free_cpumask_var(tracing_cpumask_new);
2049
2050         return count;
2051
2052 err_unlock:
2053         free_cpumask_var(tracing_cpumask_new);
2054
2055         return err;
2056 }
2057
2058 static const struct file_operations tracing_cpumask_fops = {
2059         .open           = tracing_open_generic,
2060         .read           = tracing_cpumask_read,
2061         .write          = tracing_cpumask_write,
2062 };
2063
2064 static ssize_t
2065 tracing_trace_options_read(struct file *filp, char __user *ubuf,
2066                        size_t cnt, loff_t *ppos)
2067 {
2068         struct tracer_opt *trace_opts;
2069         u32 tracer_flags;
2070         int len = 0;
2071         char *buf;
2072         int r = 0;
2073         int i;
2074
2075
2076         /* calculate max size */
2077         for (i = 0; trace_options[i]; i++) {
2078                 len += strlen(trace_options[i]);
2079                 len += 3; /* "no" and newline */
2080         }
2081
2082         mutex_lock(&trace_types_lock);
2083         tracer_flags = current_trace->flags->val;
2084         trace_opts = current_trace->flags->opts;
2085
2086         /*
2087          * Increase the size with names of options specific
2088          * of the current tracer.
2089          */
2090         for (i = 0; trace_opts[i].name; i++) {
2091                 len += strlen(trace_opts[i].name);
2092                 len += 3; /* "no" and newline */
2093         }
2094
2095         /* +1 for \0 */
2096         buf = kmalloc(len + 1, GFP_KERNEL);
2097         if (!buf) {
2098                 mutex_unlock(&trace_types_lock);
2099                 return -ENOMEM;
2100         }
2101
2102         for (i = 0; trace_options[i]; i++) {
2103                 if (trace_flags & (1 << i))
2104                         r += sprintf(buf + r, "%s\n", trace_options[i]);
2105                 else
2106                         r += sprintf(buf + r, "no%s\n", trace_options[i]);
2107         }
2108
2109         for (i = 0; trace_opts[i].name; i++) {
2110                 if (tracer_flags & trace_opts[i].bit)
2111                         r += sprintf(buf + r, "%s\n",
2112                                 trace_opts[i].name);
2113                 else
2114                         r += sprintf(buf + r, "no%s\n",
2115                                 trace_opts[i].name);
2116         }
2117         mutex_unlock(&trace_types_lock);
2118
2119         WARN_ON(r >= len + 1);
2120
2121         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2122
2123         kfree(buf);
2124         return r;
2125 }
2126
2127 /* Try to assign a tracer specific option */
2128 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2129 {
2130         struct tracer_flags *tracer_flags = trace->flags;
2131         struct tracer_opt *opts = NULL;
2132         int ret = 0, i = 0;
2133         int len;
2134
2135         for (i = 0; tracer_flags->opts[i].name; i++) {
2136                 opts = &tracer_flags->opts[i];
2137                 len = strlen(opts->name);
2138
2139                 if (strncmp(cmp, opts->name, len) == 0) {
2140                         ret = trace->set_flag(tracer_flags->val,
2141                                 opts->bit, !neg);
2142                         break;
2143                 }
2144         }
2145         /* Not found */
2146         if (!tracer_flags->opts[i].name)
2147                 return -EINVAL;
2148
2149         /* Refused to handle */
2150         if (ret)
2151                 return ret;
2152
2153         if (neg)
2154                 tracer_flags->val &= ~opts->bit;
2155         else
2156                 tracer_flags->val |= opts->bit;
2157
2158         return 0;
2159 }
2160
2161 static void set_tracer_flags(unsigned int mask, int enabled)
2162 {
2163         /* do nothing if flag is already set */
2164         if (!!(trace_flags & mask) == !!enabled)
2165                 return;
2166
2167         if (enabled)
2168                 trace_flags |= mask;
2169         else
2170                 trace_flags &= ~mask;
2171 }
2172
2173 static ssize_t
2174 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2175                         size_t cnt, loff_t *ppos)
2176 {
2177         char buf[64];
2178         char *cmp = buf;
2179         int neg = 0;
2180         int ret;
2181         int i;
2182
2183         if (cnt >= sizeof(buf))
2184                 return -EINVAL;
2185
2186         if (copy_from_user(&buf, ubuf, cnt))
2187                 return -EFAULT;
2188
2189         buf[cnt] = 0;
2190
2191         if (strncmp(buf, "no", 2) == 0) {
2192                 neg = 1;
2193                 cmp += 2;
2194         }
2195
2196         for (i = 0; trace_options[i]; i++) {
2197                 int len = strlen(trace_options[i]);
2198
2199                 if (strncmp(cmp, trace_options[i], len) == 0) {
2200                         set_tracer_flags(1 << i, !neg);
2201                         break;
2202                 }
2203         }
2204
2205         /* If no option could be set, test the specific tracer options */
2206         if (!trace_options[i]) {
2207                 mutex_lock(&trace_types_lock);
2208                 ret = set_tracer_option(current_trace, cmp, neg);
2209                 mutex_unlock(&trace_types_lock);
2210                 if (ret)
2211                         return ret;
2212         }
2213
2214         filp->f_pos += cnt;
2215
2216         return cnt;
2217 }
2218
2219 static const struct file_operations tracing_iter_fops = {
2220         .open           = tracing_open_generic,
2221         .read           = tracing_trace_options_read,
2222         .write          = tracing_trace_options_write,
2223 };
2224
2225 static const char readme_msg[] =
2226         "tracing mini-HOWTO:\n\n"
2227         "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2228         "# cat /sys/kernel/debug/tracing/available_tracers\n"
2229         "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
2230         "# cat /sys/kernel/debug/tracing/current_tracer\n"
2231         "nop\n"
2232         "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2233         "# cat /sys/kernel/debug/tracing/current_tracer\n"
2234         "sched_switch\n"
2235         "# cat /sys/kernel/debug/tracing/trace_options\n"
2236         "noprint-parent nosym-offset nosym-addr noverbose\n"
2237         "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2238         "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
2239         "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2240         "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
2241 ;
2242
2243 static ssize_t
2244 tracing_readme_read(struct file *filp, char __user *ubuf,
2245                        size_t cnt, loff_t *ppos)
2246 {
2247         return simple_read_from_buffer(ubuf, cnt, ppos,
2248                                         readme_msg, strlen(readme_msg));
2249 }
2250
2251 static const struct file_operations tracing_readme_fops = {
2252         .open           = tracing_open_generic,
2253         .read           = tracing_readme_read,
2254 };
2255
2256 static ssize_t
2257 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2258                                 size_t cnt, loff_t *ppos)
2259 {
2260         char *buf_comm;
2261         char *file_buf;
2262         char *buf;
2263         int len = 0;
2264         int pid;
2265         int i;
2266
2267         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2268         if (!file_buf)
2269                 return -ENOMEM;
2270
2271         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2272         if (!buf_comm) {
2273                 kfree(file_buf);
2274                 return -ENOMEM;
2275         }
2276
2277         buf = file_buf;
2278
2279         for (i = 0; i < SAVED_CMDLINES; i++) {
2280                 int r;
2281
2282                 pid = map_cmdline_to_pid[i];
2283                 if (pid == -1 || pid == NO_CMDLINE_MAP)
2284                         continue;
2285
2286                 trace_find_cmdline(pid, buf_comm);
2287                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2288                 buf += r;
2289                 len += r;
2290         }
2291
2292         len = simple_read_from_buffer(ubuf, cnt, ppos,
2293                                       file_buf, len);
2294
2295         kfree(file_buf);
2296         kfree(buf_comm);
2297
2298         return len;
2299 }
2300
2301 static const struct file_operations tracing_saved_cmdlines_fops = {
2302     .open       = tracing_open_generic,
2303     .read       = tracing_saved_cmdlines_read,
2304 };
2305
2306 static ssize_t
2307 tracing_ctrl_read(struct file *filp, char __user *ubuf,
2308                   size_t cnt, loff_t *ppos)
2309 {
2310         char buf[64];
2311         int r;
2312
2313         r = sprintf(buf, "%u\n", tracer_enabled);
2314         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2315 }
2316
2317 static ssize_t
2318 tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2319                    size_t cnt, loff_t *ppos)
2320 {
2321         struct trace_array *tr = filp->private_data;
2322         char buf[64];
2323         unsigned long val;
2324         int ret;
2325
2326         if (cnt >= sizeof(buf))
2327                 return -EINVAL;
2328
2329         if (copy_from_user(&buf, ubuf, cnt))
2330                 return -EFAULT;
2331
2332         buf[cnt] = 0;
2333
2334         ret = strict_strtoul(buf, 10, &val);
2335         if (ret < 0)
2336                 return ret;
2337
2338         val = !!val;
2339
2340         mutex_lock(&trace_types_lock);
2341         if (tracer_enabled ^ val) {
2342                 if (val) {
2343                         tracer_enabled = 1;
2344                         if (current_trace->start)
2345                                 current_trace->start(tr);
2346                         tracing_start();
2347                 } else {
2348                         tracer_enabled = 0;
2349                         tracing_stop();
2350                         if (current_trace->stop)
2351                                 current_trace->stop(tr);
2352                 }
2353         }
2354         mutex_unlock(&trace_types_lock);
2355
2356         filp->f_pos += cnt;
2357
2358         return cnt;
2359 }
2360
2361 static ssize_t
2362 tracing_set_trace_read(struct file *filp, char __user *ubuf,
2363                        size_t cnt, loff_t *ppos)
2364 {
2365         char buf[max_tracer_type_len+2];
2366         int r;
2367
2368         mutex_lock(&trace_types_lock);
2369         if (current_trace)
2370                 r = sprintf(buf, "%s\n", current_trace->name);
2371         else
2372                 r = sprintf(buf, "\n");
2373         mutex_unlock(&trace_types_lock);
2374
2375         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2376 }
2377
2378 int tracer_init(struct tracer *t, struct trace_array *tr)
2379 {
2380         tracing_reset_online_cpus(tr);
2381         return t->init(tr);
2382 }
2383
2384 static int tracing_resize_ring_buffer(unsigned long size)
2385 {
2386         int ret;
2387
2388         /*
2389          * If kernel or user changes the size of the ring buffer
2390          * we use the size that was given, and we can forget about
2391          * expanding it later.
2392          */
2393         ring_buffer_expanded = 1;
2394
2395         ret = ring_buffer_resize(global_trace.buffer, size);
2396         if (ret < 0)
2397                 return ret;
2398
2399         ret = ring_buffer_resize(max_tr.buffer, size);
2400         if (ret < 0) {
2401                 int r;
2402
2403                 r = ring_buffer_resize(global_trace.buffer,
2404                                        global_trace.entries);
2405                 if (r < 0) {
2406                         /*
2407                          * AARGH! We are left with different
2408                          * size max buffer!!!!
2409                          * The max buffer is our "snapshot" buffer.
2410                          * When a tracer needs a snapshot (one of the
2411                          * latency tracers), it swaps the max buffer
2412                          * with the saved snap shot. We succeeded to
2413                          * update the size of the main buffer, but failed to
2414                          * update the size of the max buffer. But when we tried
2415                          * to reset the main buffer to the original size, we
2416                          * failed there too. This is very unlikely to
2417                          * happen, but if it does, warn and kill all
2418                          * tracing.
2419                          */
2420                         WARN_ON(1);
2421                         tracing_disabled = 1;
2422                 }
2423                 return ret;
2424         }
2425
2426         global_trace.entries = size;
2427
2428         return ret;
2429 }
2430
2431 /**
2432  * tracing_update_buffers - used by tracing facility to expand ring buffers
2433  *
2434  * To save on memory when the tracing is never used on a system with it
2435  * configured in. The ring buffers are set to a minimum size. But once
2436  * a user starts to use the tracing facility, then they need to grow
2437  * to their default size.
2438  *
2439  * This function is to be called when a tracer is about to be used.
2440  */
2441 int tracing_update_buffers(void)
2442 {
2443         int ret = 0;
2444
2445         mutex_lock(&trace_types_lock);
2446         if (!ring_buffer_expanded)
2447                 ret = tracing_resize_ring_buffer(trace_buf_size);
2448         mutex_unlock(&trace_types_lock);
2449
2450         return ret;
2451 }
2452
2453 struct trace_option_dentry;
2454
2455 static struct trace_option_dentry *
2456 create_trace_option_files(struct tracer *tracer);
2457
2458 static void
2459 destroy_trace_option_files(struct trace_option_dentry *topts);
2460
2461 static int tracing_set_tracer(const char *buf)
2462 {
2463         static struct trace_option_dentry *topts;
2464         struct trace_array *tr = &global_trace;
2465         struct tracer *t;
2466         int ret = 0;
2467
2468         mutex_lock(&trace_types_lock);
2469
2470         if (!ring_buffer_expanded) {
2471                 ret = tracing_resize_ring_buffer(trace_buf_size);
2472                 if (ret < 0)
2473                         goto out;
2474                 ret = 0;
2475         }
2476
2477         for (t = trace_types; t; t = t->next) {
2478                 if (strcmp(t->name, buf) == 0)
2479                         break;
2480         }
2481         if (!t) {
2482                 ret = -EINVAL;
2483                 goto out;
2484         }
2485         if (t == current_trace)
2486                 goto out;
2487
2488         trace_branch_disable();
2489         if (current_trace && current_trace->reset)
2490                 current_trace->reset(tr);
2491
2492         destroy_trace_option_files(topts);
2493
2494         current_trace = t;
2495
2496         topts = create_trace_option_files(current_trace);
2497
2498         if (t->init) {
2499                 ret = tracer_init(t, tr);
2500                 if (ret)
2501                         goto out;
2502         }
2503
2504         trace_branch_enable(tr);
2505  out:
2506         mutex_unlock(&trace_types_lock);
2507
2508         return ret;
2509 }
2510
2511 static ssize_t
2512 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2513                         size_t cnt, loff_t *ppos)
2514 {
2515         char buf[max_tracer_type_len+1];
2516         int i;
2517         size_t ret;
2518         int err;
2519
2520         ret = cnt;
2521
2522         if (cnt > max_tracer_type_len)
2523                 cnt = max_tracer_type_len;
2524
2525         if (copy_from_user(&buf, ubuf, cnt))
2526                 return -EFAULT;
2527
2528         buf[cnt] = 0;
2529
2530         /* strip ending whitespace. */
2531         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2532                 buf[i] = 0;
2533
2534         err = tracing_set_tracer(buf);
2535         if (err)
2536                 return err;
2537
2538         filp->f_pos += ret;
2539
2540         return ret;
2541 }
2542
2543 static ssize_t
2544 tracing_max_lat_read(struct file *filp, char __user *ubuf,
2545                      size_t cnt, loff_t *ppos)
2546 {
2547         unsigned long *ptr = filp->private_data;
2548         char buf[64];
2549         int r;
2550
2551         r = snprintf(buf, sizeof(buf), "%ld\n",
2552                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
2553         if (r > sizeof(buf))
2554                 r = sizeof(buf);
2555         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2556 }
2557
2558 static ssize_t
2559 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2560                       size_t cnt, loff_t *ppos)
2561 {
2562         unsigned long *ptr = filp->private_data;
2563         char buf[64];
2564         unsigned long val;
2565         int ret;
2566
2567         if (cnt >= sizeof(buf))
2568                 return -EINVAL;
2569
2570         if (copy_from_user(&buf, ubuf, cnt))
2571                 return -EFAULT;
2572
2573         buf[cnt] = 0;
2574
2575         ret = strict_strtoul(buf, 10, &val);
2576         if (ret < 0)
2577                 return ret;
2578
2579         *ptr = val * 1000;
2580
2581         return cnt;
2582 }
2583
2584 static int tracing_open_pipe(struct inode *inode, struct file *filp)
2585 {
2586         long cpu_file = (long) inode->i_private;
2587         struct trace_iterator *iter;
2588         int ret = 0;
2589
2590         if (tracing_disabled)
2591                 return -ENODEV;
2592
2593         mutex_lock(&trace_types_lock);
2594
2595         /* We only allow one reader per cpu */
2596         if (cpu_file == TRACE_PIPE_ALL_CPU) {
2597                 if (!cpumask_empty(tracing_reader_cpumask)) {
2598                         ret = -EBUSY;
2599                         goto out;
2600                 }
2601                 cpumask_setall(tracing_reader_cpumask);
2602         } else {
2603                 if (!cpumask_test_cpu(cpu_file, tracing_reader_cpumask))
2604                         cpumask_set_cpu(cpu_file, tracing_reader_cpumask);
2605                 else {
2606                         ret = -EBUSY;
2607                         goto out;
2608                 }
2609         }
2610
2611         /* create a buffer to store the information to pass to userspace */
2612         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
2613         if (!iter) {
2614                 ret = -ENOMEM;
2615                 goto out;
2616         }
2617
2618         /*
2619          * We make a copy of the current tracer to avoid concurrent
2620          * changes on it while we are reading.
2621          */
2622         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
2623         if (!iter->trace) {
2624                 ret = -ENOMEM;
2625                 goto fail;
2626         }
2627         if (current_trace)
2628                 *iter->trace = *current_trace;
2629
2630         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
2631                 ret = -ENOMEM;
2632                 goto fail;
2633         }
2634
2635         /* trace pipe does not show start of buffer */
2636         cpumask_setall(iter->started);
2637
2638         if (trace_flags & TRACE_ITER_LATENCY_FMT)
2639                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2640
2641         iter->cpu_file = cpu_file;
2642         iter->tr = &global_trace;
2643         mutex_init(&iter->mutex);
2644         filp->private_data = iter;
2645
2646         if (iter->trace->pipe_open)
2647                 iter->trace->pipe_open(iter);
2648
2649 out:
2650         mutex_unlock(&trace_types_lock);
2651         return ret;
2652
2653 fail:
2654         kfree(iter->trace);
2655         kfree(iter);
2656         mutex_unlock(&trace_types_lock);
2657         return ret;
2658 }
2659
2660 static int tracing_release_pipe(struct inode *inode, struct file *file)
2661 {
2662         struct trace_iterator *iter = file->private_data;
2663
2664         mutex_lock(&trace_types_lock);
2665
2666         if (iter->cpu_file == TRACE_PIPE_ALL_CPU)
2667                 cpumask_clear(tracing_reader_cpumask);
2668         else
2669                 cpumask_clear_cpu(iter->cpu_file, tracing_reader_cpumask);
2670
2671         mutex_unlock(&trace_types_lock);
2672
2673         free_cpumask_var(iter->started);
2674         mutex_destroy(&iter->mutex);
2675         kfree(iter->trace);
2676         kfree(iter);
2677
2678         return 0;
2679 }
2680
2681 static unsigned int
2682 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
2683 {
2684         struct trace_iterator *iter = filp->private_data;
2685
2686         if (trace_flags & TRACE_ITER_BLOCK) {
2687                 /*
2688                  * Always select as readable when in blocking mode
2689                  */
2690                 return POLLIN | POLLRDNORM;
2691         } else {
2692                 if (!trace_empty(iter))
2693                         return POLLIN | POLLRDNORM;
2694                 poll_wait(filp, &trace_wait, poll_table);
2695                 if (!trace_empty(iter))
2696                         return POLLIN | POLLRDNORM;
2697
2698                 return 0;
2699         }
2700 }
2701
2702
2703 void default_wait_pipe(struct trace_iterator *iter)
2704 {
2705         DEFINE_WAIT(wait);
2706
2707         prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
2708
2709         if (trace_empty(iter))
2710                 schedule();
2711
2712         finish_wait(&trace_wait, &wait);
2713 }
2714
2715 /*
2716  * This is a make-shift waitqueue.
2717  * A tracer might use this callback on some rare cases:
2718  *
2719  *  1) the current tracer might hold the runqueue lock when it wakes up
2720  *     a reader, hence a deadlock (sched, function, and function graph tracers)
2721  *  2) the function tracers, trace all functions, we don't want
2722  *     the overhead of calling wake_up and friends
2723  *     (and tracing them too)
2724  *
2725  *     Anyway, this is really very primitive wakeup.
2726  */
2727 void poll_wait_pipe(struct trace_iterator *iter)
2728 {
2729         set_current_state(TASK_INTERRUPTIBLE);
2730         /* sleep for 100 msecs, and try again. */
2731         schedule_timeout(HZ / 10);
2732 }
2733
2734 /* Must be called with trace_types_lock mutex held. */
2735 static int tracing_wait_pipe(struct file *filp)
2736 {
2737         struct trace_iterator *iter = filp->private_data;
2738
2739         while (trace_empty(iter)) {
2740
2741                 if ((filp->f_flags & O_NONBLOCK)) {
2742                         return -EAGAIN;
2743                 }
2744
2745                 mutex_unlock(&iter->mutex);
2746
2747                 iter->trace->wait_pipe(iter);
2748
2749                 mutex_lock(&iter->mutex);
2750
2751                 if (signal_pending(current))
2752                         return -EINTR;
2753
2754                 /*
2755                  * We block until we read something and tracing is disabled.
2756                  * We still block if tracing is disabled, but we have never
2757                  * read anything. This allows a user to cat this file, and
2758                  * then enable tracing. But after we have read something,
2759                  * we give an EOF when tracing is again disabled.
2760                  *
2761                  * iter->pos will be 0 if we haven't read anything.
2762                  */
2763                 if (!tracer_enabled && iter->pos)
2764                         break;
2765         }
2766
2767         return 1;
2768 }
2769
2770 /*
2771  * Consumer reader.
2772  */
2773 static ssize_t
2774 tracing_read_pipe(struct file *filp, char __user *ubuf,
2775                   size_t cnt, loff_t *ppos)
2776 {
2777         struct trace_iterator *iter = filp->private_data;
2778         static struct tracer *old_tracer;
2779         ssize_t sret;
2780
2781         /* return any leftover data */
2782         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2783         if (sret != -EBUSY)
2784                 return sret;
2785
2786         trace_seq_init(&iter->seq);
2787
2788         /* copy the tracer to avoid using a global lock all around */
2789         mutex_lock(&trace_types_lock);
2790         if (unlikely(old_tracer != current_trace && current_trace)) {
2791                 old_tracer = current_trace;
2792                 *iter->trace = *current_trace;
2793         }
2794         mutex_unlock(&trace_types_lock);
2795
2796         /*
2797          * Avoid more than one consumer on a single file descriptor
2798          * This is just a matter of traces coherency, the ring buffer itself
2799          * is protected.
2800          */
2801         mutex_lock(&iter->mutex);
2802         if (iter->trace->read) {
2803                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
2804                 if (sret)
2805                         goto out;
2806         }
2807
2808 waitagain:
2809         sret = tracing_wait_pipe(filp);
2810         if (sret <= 0)
2811                 goto out;
2812
2813         /* stop when tracing is finished */
2814         if (trace_empty(iter)) {
2815                 sret = 0;
2816                 goto out;
2817         }
2818
2819         if (cnt >= PAGE_SIZE)
2820                 cnt = PAGE_SIZE - 1;
2821
2822         /* reset all but tr, trace, and overruns */
2823         memset(&iter->seq, 0,
2824                sizeof(struct trace_iterator) -
2825                offsetof(struct trace_iterator, seq));
2826         iter->pos = -1;
2827
2828         trace_event_read_lock();
2829         while (find_next_entry_inc(iter) != NULL) {
2830                 enum print_line_t ret;
2831                 int len = iter->seq.len;
2832
2833                 ret = print_trace_line(iter);
2834                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2835                         /* don't print partial lines */
2836                         iter->seq.len = len;
2837                         break;
2838                 }
2839                 if (ret != TRACE_TYPE_NO_CONSUME)
2840                         trace_consume(iter);
2841
2842                 if (iter->seq.len >= cnt)
2843                         break;
2844         }
2845         trace_event_read_unlock();
2846
2847         /* Now copy what we have to the user */
2848         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
2849         if (iter->seq.readpos >= iter->seq.len)
2850                 trace_seq_init(&iter->seq);
2851
2852         /*
2853          * If there was nothing to send to user, inspite of consuming trace
2854          * entries, go back to wait for more entries.
2855          */
2856         if (sret == -EBUSY)
2857                 goto waitagain;
2858
2859 out:
2860         mutex_unlock(&iter->mutex);
2861
2862         return sret;
2863 }
2864
2865 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
2866                                      struct pipe_buffer *buf)
2867 {
2868         __free_page(buf->page);
2869 }
2870
2871 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
2872                                      unsigned int idx)
2873 {
2874         __free_page(spd->pages[idx]);
2875 }
2876
2877 static struct pipe_buf_operations tracing_pipe_buf_ops = {
2878         .can_merge              = 0,
2879         .map                    = generic_pipe_buf_map,
2880         .unmap                  = generic_pipe_buf_unmap,
2881         .confirm                = generic_pipe_buf_confirm,
2882         .release                = tracing_pipe_buf_release,
2883         .steal                  = generic_pipe_buf_steal,
2884         .get                    = generic_pipe_buf_get,
2885 };
2886
2887 static size_t
2888 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
2889 {
2890         size_t count;
2891         int ret;
2892
2893         /* Seq buffer is page-sized, exactly what we need. */
2894         for (;;) {
2895                 count = iter->seq.len;
2896                 ret = print_trace_line(iter);
2897                 count = iter->seq.len - count;
2898                 if (rem < count) {
2899                         rem = 0;
2900                         iter->seq.len -= count;
2901                         break;
2902                 }
2903                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
2904                         iter->seq.len -= count;
2905                         break;
2906                 }
2907
2908                 if (ret != TRACE_TYPE_NO_CONSUME)
2909                         trace_consume(iter);
2910                 rem -= count;
2911                 if (!find_next_entry_inc(iter)) {
2912                         rem = 0;
2913                         iter->ent = NULL;
2914                         break;
2915                 }
2916         }
2917
2918         return rem;
2919 }
2920
2921 static ssize_t tracing_splice_read_pipe(struct file *filp,
2922                                         loff_t *ppos,
2923                                         struct pipe_inode_info *pipe,
2924                                         size_t len,
2925                                         unsigned int flags)
2926 {
2927         struct page *pages[PIPE_BUFFERS];
2928         struct partial_page partial[PIPE_BUFFERS];
2929         struct trace_iterator *iter = filp->private_data;
2930         struct splice_pipe_desc spd = {
2931                 .pages          = pages,
2932                 .partial        = partial,
2933                 .nr_pages       = 0, /* This gets updated below. */
2934                 .flags          = flags,
2935                 .ops            = &tracing_pipe_buf_ops,
2936                 .spd_release    = tracing_spd_release_pipe,
2937         };
2938         static struct tracer *old_tracer;
2939         ssize_t ret;
2940         size_t rem;
2941         unsigned int i;
2942
2943         /* copy the tracer to avoid using a global lock all around */
2944         mutex_lock(&trace_types_lock);
2945         if (unlikely(old_tracer != current_trace && current_trace)) {
2946                 old_tracer = current_trace;
2947                 *iter->trace = *current_trace;
2948         }
2949         mutex_unlock(&trace_types_lock);
2950
2951         mutex_lock(&iter->mutex);
2952
2953         if (iter->trace->splice_read) {
2954                 ret = iter->trace->splice_read(iter, filp,
2955                                                ppos, pipe, len, flags);
2956                 if (ret)
2957                         goto out_err;
2958         }
2959
2960         ret = tracing_wait_pipe(filp);
2961         if (ret <= 0)
2962                 goto out_err;
2963
2964         if (!iter->ent && !find_next_entry_inc(iter)) {
2965                 ret = -EFAULT;
2966                 goto out_err;
2967         }
2968
2969         trace_event_read_lock();
2970
2971         /* Fill as many pages as possible. */
2972         for (i = 0, rem = len; i < PIPE_BUFFERS && rem; i++) {
2973                 pages[i] = alloc_page(GFP_KERNEL);
2974                 if (!pages[i])
2975                         break;
2976
2977                 rem = tracing_fill_pipe_page(rem, iter);
2978
2979                 /* Copy the data into the page, so we can start over. */
2980                 ret = trace_seq_to_buffer(&iter->seq,
2981                                           page_address(pages[i]),
2982                                           iter->seq.len);
2983                 if (ret < 0) {
2984                         __free_page(pages[i]);
2985                         break;
2986                 }
2987                 partial[i].offset = 0;
2988                 partial[i].len = iter->seq.len;
2989
2990                 trace_seq_init(&iter->seq);
2991         }
2992
2993         trace_event_read_unlock();
2994         mutex_unlock(&iter->mutex);
2995
2996         spd.nr_pages = i;
2997
2998         return splice_to_pipe(pipe, &spd);
2999
3000 out_err:
3001         mutex_unlock(&iter->mutex);
3002
3003         return ret;
3004 }
3005
3006 static ssize_t
3007 tracing_entries_read(struct file *filp, char __user *ubuf,
3008                      size_t cnt, loff_t *ppos)
3009 {
3010         struct trace_array *tr = filp->private_data;
3011         char buf[96];
3012         int r;
3013
3014         mutex_lock(&trace_types_lock);
3015         if (!ring_buffer_expanded)
3016                 r = sprintf(buf, "%lu (expanded: %lu)\n",
3017                             tr->entries >> 10,
3018                             trace_buf_size >> 10);
3019         else
3020                 r = sprintf(buf, "%lu\n", tr->entries >> 10);
3021         mutex_unlock(&trace_types_lock);
3022
3023         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3024 }
3025
3026 static ssize_t
3027 tracing_entries_write(struct file *filp, const char __user *ubuf,
3028                       size_t cnt, loff_t *ppos)
3029 {
3030         unsigned long val;
3031         char buf[64];
3032         int ret, cpu;
3033
3034         if (cnt >= sizeof(buf))
3035                 return -EINVAL;
3036
3037         if (copy_from_user(&buf, ubuf, cnt))
3038                 return -EFAULT;
3039
3040         buf[cnt] = 0;
3041
3042         ret = strict_strtoul(buf, 10, &val);
3043         if (ret < 0)
3044                 return ret;
3045
3046         /* must have at least 1 entry */
3047         if (!val)
3048                 return -EINVAL;
3049
3050         mutex_lock(&trace_types_lock);
3051
3052         tracing_stop();
3053
3054         /* disable all cpu buffers */
3055         for_each_tracing_cpu(cpu) {
3056                 if (global_trace.data[cpu])
3057                         atomic_inc(&global_trace.data[cpu]->disabled);
3058                 if (max_tr.data[cpu])
3059                         atomic_inc(&max_tr.data[cpu]->disabled);
3060         }
3061
3062         /* value is in KB */
3063         val <<= 10;
3064
3065         if (val != global_trace.entries) {
3066                 ret = tracing_resize_ring_buffer(val);
3067                 if (ret < 0) {
3068                         cnt = ret;
3069                         goto out;
3070                 }
3071         }
3072
3073         filp->f_pos += cnt;
3074
3075         /* If check pages failed, return ENOMEM */
3076         if (tracing_disabled)
3077                 cnt = -ENOMEM;
3078  out:
3079         for_each_tracing_cpu(cpu) {
3080                 if (global_trace.data[cpu])
3081                         atomic_dec(&global_trace.data[cpu]->disabled);
3082                 if (max_tr.data[cpu])
3083                         atomic_dec(&max_tr.data[cpu]->disabled);
3084         }
3085
3086         tracing_start();
3087         max_tr.entries = global_trace.entries;
3088         mutex_unlock(&trace_types_lock);
3089
3090         return cnt;
3091 }
3092
3093 static int mark_printk(const char *fmt, ...)
3094 {
3095         int ret;
3096         va_list args;
3097         va_start(args, fmt);
3098         ret = trace_vprintk(0, fmt, args);
3099         va_end(args);
3100         return ret;
3101 }
3102
3103 static ssize_t
3104 tracing_mark_write(struct file *filp, const char __user *ubuf,
3105                                         size_t cnt, loff_t *fpos)
3106 {
3107         char *buf;
3108         char *end;
3109
3110         if (tracing_disabled)
3111                 return -EINVAL;
3112
3113         if (cnt > TRACE_BUF_SIZE)
3114                 cnt = TRACE_BUF_SIZE;
3115
3116         buf = kmalloc(cnt + 1, GFP_KERNEL);
3117         if (buf == NULL)
3118                 return -ENOMEM;
3119
3120         if (copy_from_user(buf, ubuf, cnt)) {
3121                 kfree(buf);
3122                 return -EFAULT;
3123         }
3124
3125         /* Cut from the first nil or newline. */
3126         buf[cnt] = '\0';
3127         end = strchr(buf, '\n');
3128         if (end)
3129                 *end = '\0';
3130
3131         cnt = mark_printk("%s\n", buf);
3132         kfree(buf);
3133         *fpos += cnt;
3134
3135         return cnt;
3136 }
3137
3138 static ssize_t tracing_clock_read(struct file *filp, char __user *ubuf,
3139                                   size_t cnt, loff_t *ppos)
3140 {
3141         char buf[64];
3142         int bufiter = 0;
3143         int i;
3144
3145         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
3146                 bufiter += snprintf(buf + bufiter, sizeof(buf) - bufiter,
3147                         "%s%s%s%s", i ? " " : "",
3148                         i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3149                         i == trace_clock_id ? "]" : "");
3150         bufiter += snprintf(buf + bufiter, sizeof(buf) - bufiter, "\n");
3151
3152         return simple_read_from_buffer(ubuf, cnt, ppos, buf, bufiter);
3153 }
3154
3155 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3156                                    size_t cnt, loff_t *fpos)
3157 {
3158         char buf[64];
3159         const char *clockstr;
3160         int i;
3161
3162         if (cnt >= sizeof(buf))
3163                 return -EINVAL;
3164
3165         if (copy_from_user(&buf, ubuf, cnt))
3166                 return -EFAULT;
3167
3168         buf[cnt] = 0;
3169
3170         clockstr = strstrip(buf);
3171
3172         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3173                 if (strcmp(trace_clocks[i].name, clockstr) == 0)
3174                         break;
3175         }
3176         if (i == ARRAY_SIZE(trace_clocks))
3177                 return -EINVAL;
3178
3179         trace_clock_id = i;
3180
3181         mutex_lock(&trace_types_lock);
3182
3183         ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3184         if (max_tr.buffer)
3185                 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3186
3187         mutex_unlock(&trace_types_lock);
3188
3189         *fpos += cnt;
3190
3191         return cnt;
3192 }
3193
3194 static const struct file_operations tracing_max_lat_fops = {
3195         .open           = tracing_open_generic,
3196         .read           = tracing_max_lat_read,
3197         .write          = tracing_max_lat_write,
3198 };
3199
3200 static const struct file_operations tracing_ctrl_fops = {
3201         .open           = tracing_open_generic,
3202         .read           = tracing_ctrl_read,
3203         .write          = tracing_ctrl_write,
3204 };
3205
3206 static const struct file_operations set_tracer_fops = {
3207         .open           = tracing_open_generic,
3208         .read           = tracing_set_trace_read,
3209         .write          = tracing_set_trace_write,
3210 };
3211
3212 static const struct file_operations tracing_pipe_fops = {
3213         .open           = tracing_open_pipe,
3214         .poll           = tracing_poll_pipe,
3215         .read           = tracing_read_pipe,
3216         .splice_read    = tracing_splice_read_pipe,
3217         .release        = tracing_release_pipe,
3218 };
3219
3220 static const struct file_operations tracing_entries_fops = {
3221         .open           = tracing_open_generic,
3222         .read           = tracing_entries_read,
3223         .write          = tracing_entries_write,
3224 };
3225
3226 static const struct file_operations tracing_mark_fops = {
3227         .open           = tracing_open_generic,
3228         .write          = tracing_mark_write,
3229 };
3230
3231 static const struct file_operations trace_clock_fops = {
3232         .open           = tracing_open_generic,
3233         .read           = tracing_clock_read,
3234         .write          = tracing_clock_write,
3235 };
3236
3237 struct ftrace_buffer_info {
3238         struct trace_array      *tr;
3239         void                    *spare;
3240         int                     cpu;
3241         unsigned int            read;
3242 };
3243
3244 static int tracing_buffers_open(struct inode *inode, struct file *filp)
3245 {
3246         int cpu = (int)(long)inode->i_private;
3247         struct ftrace_buffer_info *info;
3248
3249         if (tracing_disabled)
3250                 return -ENODEV;
3251
3252         info = kzalloc(sizeof(*info), GFP_KERNEL);
3253         if (!info)
3254                 return -ENOMEM;
3255
3256         info->tr        = &global_trace;
3257         info->cpu       = cpu;
3258         info->spare     = NULL;
3259         /* Force reading ring buffer for first read */
3260         info->read      = (unsigned int)-1;
3261
3262         filp->private_data = info;
3263
3264         return nonseekable_open(inode, filp);
3265 }
3266
3267 static ssize_t
3268 tracing_buffers_read(struct file *filp, char __user *ubuf,
3269                      size_t count, loff_t *ppos)
3270 {
3271         struct ftrace_buffer_info *info = filp->private_data;
3272         unsigned int pos;
3273         ssize_t ret;
3274         size_t size;
3275
3276         if (!count)
3277                 return 0;
3278
3279         if (!info->spare)
3280                 info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3281         if (!info->spare)
3282                 return -ENOMEM;
3283
3284         /* Do we have previous read data to read? */
3285         if (info->read < PAGE_SIZE)
3286                 goto read;
3287
3288         info->read = 0;
3289
3290         ret = ring_buffer_read_page(info->tr->buffer,
3291                                     &info->spare,
3292                                     count,
3293                                     info->cpu, 0);
3294         if (ret < 0)
3295                 return 0;
3296
3297         pos = ring_buffer_page_len(info->spare);
3298
3299         if (pos < PAGE_SIZE)
3300                 memset(info->spare + pos, 0, PAGE_SIZE - pos);
3301
3302 read:
3303         size = PAGE_SIZE - info->read;
3304         if (size > count)
3305                 size = count;
3306
3307         ret = copy_to_user(ubuf, info->spare + info->read, size);
3308         if (ret == size)
3309                 return -EFAULT;
3310         size -= ret;
3311
3312         *ppos += size;
3313         info->read += size;
3314
3315         return size;
3316 }
3317
3318 static int tracing_buffers_release(struct inode *inode, struct file *file)
3319 {
3320         struct ftrace_buffer_info *info = file->private_data;
3321
3322         if (info->spare)
3323                 ring_buffer_free_read_page(info->tr->buffer, info->spare);
3324         kfree(info);
3325
3326         return 0;
3327 }
3328
3329 struct buffer_ref {
3330         struct ring_buffer      *buffer;
3331         void                    *page;
3332         int                     ref;
3333 };
3334
3335 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3336                                     struct pipe_buffer *buf)
3337 {
3338         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3339
3340         if (--ref->ref)
3341                 return;
3342
3343         ring_buffer_free_read_page(ref->buffer, ref->page);
3344         kfree(ref);
3345         buf->private = 0;
3346 }
3347
3348 static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3349                                  struct pipe_buffer *buf)
3350 {
3351         return 1;
3352 }
3353
3354 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3355                                 struct pipe_buffer *buf)
3356 {
3357         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3358
3359         ref->ref++;
3360 }
3361
3362 /* Pipe buffer operations for a buffer. */
3363 static struct pipe_buf_operations buffer_pipe_buf_ops = {
3364         .can_merge              = 0,
3365         .map                    = generic_pipe_buf_map,
3366         .unmap                  = generic_pipe_buf_unmap,
3367         .confirm                = generic_pipe_buf_confirm,
3368         .release                = buffer_pipe_buf_release,
3369         .steal                  = buffer_pipe_buf_steal,
3370         .get                    = buffer_pipe_buf_get,
3371 };
3372
3373 /*
3374  * Callback from splice_to_pipe(), if we need to release some pages
3375  * at the end of the spd in case we error'ed out in filling the pipe.
3376  */
3377 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3378 {
3379         struct buffer_ref *ref =
3380                 (struct buffer_ref *)spd->partial[i].private;
3381
3382         if (--ref->ref)
3383                 return;
3384
3385         ring_buffer_free_read_page(ref->buffer, ref->page);
3386         kfree(ref);
3387         spd->partial[i].private = 0;
3388 }
3389
3390 static ssize_t
3391 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
3392                             struct pipe_inode_info *pipe, size_t len,
3393                             unsigned int flags)
3394 {
3395         struct ftrace_buffer_info *info = file->private_data;
3396         struct partial_page partial[PIPE_BUFFERS];
3397         struct page *pages[PIPE_BUFFERS];
3398         struct splice_pipe_desc spd = {
3399                 .pages          = pages,
3400                 .partial        = partial,
3401                 .flags          = flags,
3402                 .ops            = &buffer_pipe_buf_ops,
3403                 .spd_release    = buffer_spd_release,
3404         };
3405         struct buffer_ref *ref;
3406         int entries, size, i;
3407         size_t ret;
3408
3409         if (*ppos & (PAGE_SIZE - 1)) {
3410                 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
3411                 return -EINVAL;
3412         }
3413
3414         if (len & (PAGE_SIZE - 1)) {
3415                 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
3416                 if (len < PAGE_SIZE)
3417                         return -EINVAL;
3418                 len &= PAGE_MASK;
3419         }
3420
3421         entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3422
3423         for (i = 0; i < PIPE_BUFFERS && len && entries; i++, len -= PAGE_SIZE) {
3424                 struct page *page;
3425                 int r;
3426
3427                 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3428                 if (!ref)
3429                         break;
3430
3431                 ref->ref = 1;
3432                 ref->buffer = info->tr->buffer;
3433                 ref->page = ring_buffer_alloc_read_page(ref->buffer);
3434                 if (!ref->page) {
3435                         kfree(ref);
3436                         break;
3437                 }
3438
3439                 r = ring_buffer_read_page(ref->buffer, &ref->page,
3440                                           len, info->cpu, 1);
3441                 if (r < 0) {
3442                         ring_buffer_free_read_page(ref->buffer,
3443                                                    ref->page);
3444                         kfree(ref);
3445                         break;
3446                 }
3447
3448                 /*
3449                  * zero out any left over data, this is going to
3450                  * user land.
3451                  */
3452                 size = ring_buffer_page_len(ref->page);
3453                 if (size < PAGE_SIZE)
3454                         memset(ref->page + size, 0, PAGE_SIZE - size);
3455
3456                 page = virt_to_page(ref->page);
3457
3458                 spd.pages[i] = page;
3459                 spd.partial[i].len = PAGE_SIZE;
3460                 spd.partial[i].offset = 0;
3461                 spd.partial[i].private = (unsigned long)ref;
3462                 spd.nr_pages++;
3463                 *ppos += PAGE_SIZE;
3464
3465                 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3466         }
3467
3468         spd.nr_pages = i;
3469
3470         /* did we read anything? */
3471         if (!spd.nr_pages) {
3472                 if (flags & SPLICE_F_NONBLOCK)
3473                         ret = -EAGAIN;
3474                 else
3475                         ret = 0;
3476                 /* TODO: block */
3477                 return ret;
3478         }
3479
3480         ret = splice_to_pipe(pipe, &spd);
3481
3482         return ret;
3483 }
3484
3485 static const struct file_operations tracing_buffers_fops = {
3486         .open           = tracing_buffers_open,
3487         .read           = tracing_buffers_read,
3488         .release        = tracing_buffers_release,
3489         .splice_read    = tracing_buffers_splice_read,
3490         .llseek         = no_llseek,
3491 };
3492
3493 static ssize_t
3494 tracing_stats_read(struct file *filp, char __user *ubuf,
3495                    size_t count, loff_t *ppos)
3496 {
3497         unsigned long cpu = (unsigned long)filp->private_data;
3498         struct trace_array *tr = &global_trace;
3499         struct trace_seq *s;
3500         unsigned long cnt;
3501
3502         s = kmalloc(sizeof(*s), GFP_KERNEL);
3503         if (!s)
3504                 return ENOMEM;
3505
3506         trace_seq_init(s);
3507
3508         cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
3509         trace_seq_printf(s, "entries: %ld\n", cnt);
3510
3511         cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
3512         trace_seq_printf(s, "overrun: %ld\n", cnt);
3513
3514         cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
3515         trace_seq_printf(s, "commit overrun: %ld\n", cnt);
3516
3517         count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
3518
3519         kfree(s);
3520
3521         return count;
3522 }
3523
3524 static const struct file_operations tracing_stats_fops = {
3525         .open           = tracing_open_generic,
3526         .read           = tracing_stats_read,
3527 };
3528
3529 #ifdef CONFIG_DYNAMIC_FTRACE
3530
3531 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
3532 {
3533         return 0;
3534 }
3535
3536 static ssize_t
3537 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
3538                   size_t cnt, loff_t *ppos)
3539 {
3540         static char ftrace_dyn_info_buffer[1024];
3541         static DEFINE_MUTEX(dyn_info_mutex);
3542         unsigned long *p = filp->private_data;
3543         char *buf = ftrace_dyn_info_buffer;
3544         int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
3545         int r;
3546
3547         mutex_lock(&dyn_info_mutex);
3548         r = sprintf(buf, "%ld ", *p);
3549
3550         r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
3551         buf[r++] = '\n';
3552
3553         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3554
3555         mutex_unlock(&dyn_info_mutex);
3556
3557         return r;
3558 }
3559
3560 static const struct file_operations tracing_dyn_info_fops = {
3561         .open           = tracing_open_generic,
3562         .read           = tracing_read_dyn_info,
3563 };
3564 #endif
3565
3566 static struct dentry *d_tracer;
3567
3568 struct dentry *tracing_init_dentry(void)
3569 {
3570         static int once;
3571
3572         if (d_tracer)
3573                 return d_tracer;
3574
3575         if (!debugfs_initialized())
3576                 return NULL;
3577
3578         d_tracer = debugfs_create_dir("tracing", NULL);
3579
3580         if (!d_tracer && !once) {
3581                 once = 1;
3582                 pr_warning("Could not create debugfs directory 'tracing'\n");
3583                 return NULL;
3584         }
3585
3586         return d_tracer;
3587 }
3588
3589 static struct dentry *d_percpu;
3590
3591 struct dentry *tracing_dentry_percpu(void)
3592 {
3593         static int once;
3594         struct dentry *d_tracer;
3595
3596         if (d_percpu)
3597                 return d_percpu;
3598
3599         d_tracer = tracing_init_dentry();
3600
3601         if (!d_tracer)
3602                 return NULL;
3603
3604         d_percpu = debugfs_create_dir("per_cpu", d_tracer);
3605
3606         if (!d_percpu && !once) {
3607                 once = 1;
3608                 pr_warning("Could not create debugfs directory 'per_cpu'\n");
3609                 return NULL;
3610         }
3611
3612         return d_percpu;
3613 }
3614
3615 static void tracing_init_debugfs_percpu(long cpu)
3616 {
3617         struct dentry *d_percpu = tracing_dentry_percpu();
3618         struct dentry *d_cpu;
3619         /* strlen(cpu) + MAX(log10(cpu)) + '\0' */
3620         char cpu_dir[7];
3621
3622         if (cpu > 999 || cpu < 0)
3623                 return;
3624
3625         sprintf(cpu_dir, "cpu%ld", cpu);
3626         d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
3627         if (!d_cpu) {
3628                 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
3629                 return;
3630         }
3631
3632         /* per cpu trace_pipe */
3633         trace_create_file("trace_pipe", 0444, d_cpu,
3634                         (void *) cpu, &tracing_pipe_fops);
3635
3636         /* per cpu trace */
3637         trace_create_file("trace", 0644, d_cpu,
3638                         (void *) cpu, &tracing_fops);
3639
3640         trace_create_file("trace_pipe_raw", 0444, d_cpu,
3641                         (void *) cpu, &tracing_buffers_fops);
3642
3643         trace_create_file("stats", 0444, d_cpu,
3644                         (void *) cpu, &tracing_stats_fops);
3645 }
3646
3647 #ifdef CONFIG_FTRACE_SELFTEST
3648 /* Let selftest have access to static functions in this file */
3649 #include "trace_selftest.c"
3650 #endif
3651
3652 struct trace_option_dentry {
3653         struct tracer_opt               *opt;
3654         struct tracer_flags             *flags;
3655         struct dentry                   *entry;
3656 };
3657
3658 static ssize_t
3659 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
3660                         loff_t *ppos)
3661 {
3662         struct trace_option_dentry *topt = filp->private_data;
3663         char *buf;
3664
3665         if (topt->flags->val & topt->opt->bit)
3666                 buf = "1\n";
3667         else
3668                 buf = "0\n";
3669
3670         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3671 }
3672
3673 static ssize_t
3674 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
3675                          loff_t *ppos)
3676 {
3677         struct trace_option_dentry *topt = filp->private_data;
3678         unsigned long val;
3679         char buf[64];
3680         int ret;
3681
3682         if (cnt >= sizeof(buf))
3683                 return -EINVAL;
3684
3685         if (copy_from_user(&buf, ubuf, cnt))
3686                 return -EFAULT;
3687
3688         buf[cnt] = 0;
3689
3690         ret = strict_strtoul(buf, 10, &val);
3691         if (ret < 0)
3692                 return ret;
3693
3694         ret = 0;
3695         switch (val) {
3696         case 0:
3697                 /* do nothing if already cleared */
3698                 if (!(topt->flags->val & topt->opt->bit))
3699                         break;
3700
3701                 mutex_lock(&trace_types_lock);
3702                 if (current_trace->set_flag)
3703                         ret = current_trace->set_flag(topt->flags->val,
3704                                                       topt->opt->bit, 0);
3705                 mutex_unlock(&trace_types_lock);
3706                 if (ret)
3707                         return ret;
3708                 topt->flags->val &= ~topt->opt->bit;
3709                 break;
3710         case 1:
3711                 /* do nothing if already set */
3712                 if (topt->flags->val & topt->opt->bit)
3713                         break;
3714
3715                 mutex_lock(&trace_types_lock);
3716                 if (current_trace->set_flag)
3717                         ret = current_trace->set_flag(topt->flags->val,
3718                                                       topt->opt->bit, 1);
3719                 mutex_unlock(&trace_types_lock);
3720                 if (ret)
3721                         return ret;
3722                 topt->flags->val |= topt->opt->bit;
3723                 break;
3724
3725         default:
3726                 return -EINVAL;
3727         }
3728
3729         *ppos += cnt;
3730
3731         return cnt;
3732 }
3733
3734
3735 static const struct file_operations trace_options_fops = {
3736         .open = tracing_open_generic,
3737         .read = trace_options_read,
3738         .write = trace_options_write,
3739 };
3740
3741 static ssize_t
3742 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
3743                         loff_t *ppos)
3744 {
3745         long index = (long)filp->private_data;
3746         char *buf;
3747
3748         if (trace_flags & (1 << index))
3749                 buf = "1\n";
3750         else
3751                 buf = "0\n";
3752
3753         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
3754 }
3755
3756 static ssize_t
3757 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
3758                          loff_t *ppos)
3759 {
3760         long index = (long)filp->private_data;
3761         char buf[64];
3762         unsigned long val;
3763         int ret;
3764
3765         if (cnt >= sizeof(buf))
3766                 return -EINVAL;
3767
3768         if (copy_from_user(&buf, ubuf, cnt))
3769                 return -EFAULT;
3770
3771         buf[cnt] = 0;
3772
3773         ret = strict_strtoul(buf, 10, &val);
3774         if (ret < 0)
3775                 return ret;
3776
3777         switch (val) {
3778         case 0:
3779                 trace_flags &= ~(1 << index);
3780                 break;
3781         case 1:
3782                 trace_flags |= 1 << index;
3783                 break;
3784
3785         default:
3786                 return -EINVAL;
3787         }
3788
3789         *ppos += cnt;
3790
3791         return cnt;
3792 }
3793
3794 static const struct file_operations trace_options_core_fops = {
3795         .open = tracing_open_generic,
3796         .read = trace_options_core_read,
3797         .write = trace_options_core_write,
3798 };
3799
3800 struct dentry *trace_create_file(const char *name,
3801                                  mode_t mode,
3802                                  struct dentry *parent,
3803                                  void *data,
3804                                  const struct file_operations *fops)
3805 {
3806         struct dentry *ret;
3807
3808         ret = debugfs_create_file(name, mode, parent, data, fops);
3809         if (!ret)
3810                 pr_warning("Could not create debugfs '%s' entry\n", name);
3811
3812         return ret;
3813 }
3814
3815
3816 static struct dentry *trace_options_init_dentry(void)
3817 {
3818         struct dentry *d_tracer;
3819         static struct dentry *t_options;
3820
3821         if (t_options)
3822                 return t_options;
3823
3824         d_tracer = tracing_init_dentry();
3825         if (!d_tracer)
3826                 return NULL;
3827
3828         t_options = debugfs_create_dir("options", d_tracer);
3829         if (!t_options) {
3830                 pr_warning("Could not create debugfs directory 'options'\n");
3831                 return NULL;
3832         }
3833
3834         return t_options;
3835 }
3836
3837 static void
3838 create_trace_option_file(struct trace_option_dentry *topt,
3839                          struct tracer_flags *flags,
3840                          struct tracer_opt *opt)
3841 {
3842         struct dentry *t_options;
3843
3844         t_options = trace_options_init_dentry();
3845         if (!t_options)
3846                 return;
3847
3848         topt->flags = flags;
3849         topt->opt = opt;
3850
3851         topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
3852                                     &trace_options_fops);
3853
3854 }
3855
3856 static struct trace_option_dentry *
3857 create_trace_option_files(struct tracer *tracer)
3858 {
3859         struct trace_option_dentry *topts;
3860         struct tracer_flags *flags;
3861         struct tracer_opt *opts;
3862         int cnt;
3863
3864         if (!tracer)
3865                 return NULL;
3866
3867         flags = tracer->flags;
3868
3869         if (!flags || !flags->opts)
3870                 return NULL;
3871
3872         opts = flags->opts;
3873
3874         for (cnt = 0; opts[cnt].name; cnt++)
3875                 ;
3876
3877         topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
3878         if (!topts)
3879                 return NULL;
3880
3881         for (cnt = 0; opts[cnt].name; cnt++)
3882                 create_trace_option_file(&topts[cnt], flags,
3883                                          &opts[cnt]);
3884
3885         return topts;
3886 }
3887
3888 static void
3889 destroy_trace_option_files(struct trace_option_dentry *topts)
3890 {
3891         int cnt;
3892
3893         if (!topts)
3894                 return;
3895
3896         for (cnt = 0; topts[cnt].opt; cnt++) {
3897                 if (topts[cnt].entry)
3898                         debugfs_remove(topts[cnt].entry);
3899         }
3900
3901         kfree(topts);
3902 }
3903
3904 static struct dentry *
3905 create_trace_option_core_file(const char *option, long index)
3906 {
3907         struct dentry *t_options;
3908
3909         t_options = trace_options_init_dentry();
3910         if (!t_options)
3911                 return NULL;
3912
3913         return trace_create_file(option, 0644, t_options, (void *)index,
3914                                     &trace_options_core_fops);
3915 }
3916
3917 static __init void create_trace_options_dir(void)
3918 {
3919         struct dentry *t_options;
3920         int i;
3921
3922         t_options = trace_options_init_dentry();
3923         if (!t_options)
3924                 return;
3925
3926         for (i = 0; trace_options[i]; i++)
3927                 create_trace_option_core_file(trace_options[i], i);
3928 }
3929
3930 static __init int tracer_init_debugfs(void)
3931 {
3932         struct dentry *d_tracer;
3933         int cpu;
3934
3935         d_tracer = tracing_init_dentry();
3936
3937         trace_create_file("tracing_enabled", 0644, d_tracer,
3938                         &global_trace, &tracing_ctrl_fops);
3939
3940         trace_create_file("trace_options", 0644, d_tracer,
3941                         NULL, &tracing_iter_fops);
3942
3943         trace_create_file("tracing_cpumask", 0644, d_tracer,
3944                         NULL, &tracing_cpumask_fops);
3945
3946         trace_create_file("trace", 0644, d_tracer,
3947                         (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
3948
3949         trace_create_file("available_tracers", 0444, d_tracer,
3950                         &global_trace, &show_traces_fops);
3951
3952         trace_create_file("current_tracer", 0644, d_tracer,
3953                         &global_trace, &set_tracer_fops);
3954
3955         trace_create_file("tracing_max_latency", 0644, d_tracer,
3956                         &tracing_max_latency, &tracing_max_lat_fops);
3957
3958         trace_create_file("tracing_thresh", 0644, d_tracer,
3959                         &tracing_thresh, &tracing_max_lat_fops);
3960
3961         trace_create_file("README", 0444, d_tracer,
3962                         NULL, &tracing_readme_fops);
3963
3964         trace_create_file("trace_pipe", 0444, d_tracer,
3965                         (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
3966
3967         trace_create_file("buffer_size_kb", 0644, d_tracer,
3968                         &global_trace, &tracing_entries_fops);
3969
3970         trace_create_file("trace_marker", 0220, d_tracer,
3971                         NULL, &tracing_mark_fops);
3972
3973         trace_create_file("saved_cmdlines", 0444, d_tracer,
3974                         NULL, &tracing_saved_cmdlines_fops);
3975
3976         trace_create_file("trace_clock", 0644, d_tracer, NULL,
3977                           &trace_clock_fops);
3978
3979 #ifdef CONFIG_DYNAMIC_FTRACE
3980         trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
3981                         &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
3982 #endif
3983 #ifdef CONFIG_SYSPROF_TRACER
3984         init_tracer_sysprof_debugfs(d_tracer);
3985 #endif
3986
3987         create_trace_options_dir();
3988
3989         for_each_tracing_cpu(cpu)
3990                 tracing_init_debugfs_percpu(cpu);
3991
3992         return 0;
3993 }
3994
3995 static int trace_panic_handler(struct notifier_block *this,
3996                                unsigned long event, void *unused)
3997 {
3998         if (ftrace_dump_on_oops)
3999                 ftrace_dump();
4000         return NOTIFY_OK;
4001 }
4002
4003 static struct notifier_block trace_panic_notifier = {
4004         .notifier_call  = trace_panic_handler,
4005         .next           = NULL,
4006         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
4007 };
4008
4009 static int trace_die_handler(struct notifier_block *self,
4010                              unsigned long val,
4011                              void *data)
4012 {
4013         switch (val) {
4014         case DIE_OOPS:
4015                 if (ftrace_dump_on_oops)
4016                         ftrace_dump();
4017                 break;
4018         default:
4019                 break;
4020         }
4021         return NOTIFY_OK;
4022 }
4023
4024 static struct notifier_block trace_die_notifier = {
4025         .notifier_call = trace_die_handler,
4026         .priority = 200
4027 };
4028
4029 /*
4030  * printk is set to max of 1024, we really don't need it that big.
4031  * Nothing should be printing 1000 characters anyway.
4032  */
4033 #define TRACE_MAX_PRINT         1000
4034
4035 /*
4036  * Define here KERN_TRACE so that we have one place to modify
4037  * it if we decide to change what log level the ftrace dump
4038  * should be at.
4039  */
4040 #define KERN_TRACE              KERN_EMERG
4041
4042 static void
4043 trace_printk_seq(struct trace_seq *s)
4044 {
4045         /* Probably should print a warning here. */
4046         if (s->len >= 1000)
4047                 s->len = 1000;
4048
4049         /* should be zero ended, but we are paranoid. */
4050         s->buffer[s->len] = 0;
4051
4052         printk(KERN_TRACE "%s", s->buffer);
4053
4054         trace_seq_init(s);
4055 }
4056
4057 static void __ftrace_dump(bool disable_tracing)
4058 {
4059         static raw_spinlock_t ftrace_dump_lock =
4060                 (raw_spinlock_t)__RAW_SPIN_LOCK_UNLOCKED;
4061         /* use static because iter can be a bit big for the stack */
4062         static struct trace_iterator iter;
4063         unsigned int old_userobj;
4064         static int dump_ran;
4065         unsigned long flags;
4066         int cnt = 0, cpu;
4067
4068         /* only one dump */
4069         local_irq_save(flags);
4070         __raw_spin_lock(&ftrace_dump_lock);
4071         if (dump_ran)
4072                 goto out;
4073
4074         dump_ran = 1;
4075
4076         tracing_off();
4077
4078         if (disable_tracing)
4079                 ftrace_kill();
4080
4081         for_each_tracing_cpu(cpu) {
4082                 atomic_inc(&global_trace.data[cpu]->disabled);
4083         }
4084
4085         old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4086
4087         /* don't look at user memory in panic mode */
4088         trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4089
4090         printk(KERN_TRACE "Dumping ftrace buffer:\n");
4091
4092         /* Simulate the iterator */
4093         iter.tr = &global_trace;
4094         iter.trace = current_trace;
4095         iter.cpu_file = TRACE_PIPE_ALL_CPU;
4096
4097         /*
4098          * We need to stop all tracing on all CPUS to read the
4099          * the next buffer. This is a bit expensive, but is
4100          * not done often. We fill all what we can read,
4101          * and then release the locks again.
4102          */
4103
4104         while (!trace_empty(&iter)) {
4105
4106                 if (!cnt)
4107                         printk(KERN_TRACE "---------------------------------\n");
4108
4109                 cnt++;
4110
4111                 /* reset all but tr, trace, and overruns */
4112                 memset(&iter.seq, 0,
4113                        sizeof(struct trace_iterator) -
4114                        offsetof(struct trace_iterator, seq));
4115                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
4116                 iter.pos = -1;
4117
4118                 if (find_next_entry_inc(&iter) != NULL) {
4119                         int ret;
4120
4121                         ret = print_trace_line(&iter);
4122                         if (ret != TRACE_TYPE_NO_CONSUME)
4123                                 trace_consume(&iter);
4124                 }
4125
4126                 trace_printk_seq(&iter.seq);
4127         }
4128
4129         if (!cnt)
4130                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
4131         else
4132                 printk(KERN_TRACE "---------------------------------\n");
4133
4134         /* Re-enable tracing if requested */
4135         if (!disable_tracing) {
4136                 trace_flags |= old_userobj;
4137
4138                 for_each_tracing_cpu(cpu) {
4139                         atomic_dec(&global_trace.data[cpu]->disabled);
4140                 }
4141                 tracing_on();
4142         }
4143
4144  out:
4145         __raw_spin_unlock(&ftrace_dump_lock);
4146         local_irq_restore(flags);
4147 }
4148
4149 /* By default: disable tracing after the dump */
4150 void ftrace_dump(void)
4151 {
4152         __ftrace_dump(true);
4153 }
4154
4155 __init static int tracer_alloc_buffers(void)
4156 {
4157         int ring_buf_size;
4158         int i;
4159         int ret = -ENOMEM;
4160
4161         if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4162                 goto out;
4163
4164         if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4165                 goto out_free_buffer_mask;
4166
4167         if (!alloc_cpumask_var(&tracing_reader_cpumask, GFP_KERNEL))
4168                 goto out_free_tracing_cpumask;
4169
4170         /* To save memory, keep the ring buffer size to its minimum */
4171         if (ring_buffer_expanded)
4172                 ring_buf_size = trace_buf_size;
4173         else
4174                 ring_buf_size = 1;
4175
4176         cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4177         cpumask_copy(tracing_cpumask, cpu_all_mask);
4178         cpumask_clear(tracing_reader_cpumask);
4179
4180         /* TODO: make the number of buffers hot pluggable with CPUS */
4181         global_trace.buffer = ring_buffer_alloc(ring_buf_size,
4182                                                    TRACE_BUFFER_FLAGS);
4183         if (!global_trace.buffer) {
4184                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4185                 WARN_ON(1);
4186                 goto out_free_cpumask;
4187         }
4188         global_trace.entries = ring_buffer_size(global_trace.buffer);
4189
4190
4191 #ifdef CONFIG_TRACER_MAX_TRACE
4192         max_tr.buffer = ring_buffer_alloc(ring_buf_size,
4193                                              TRACE_BUFFER_FLAGS);
4194         if (!max_tr.buffer) {
4195                 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4196                 WARN_ON(1);
4197                 ring_buffer_free(global_trace.buffer);
4198                 goto out_free_cpumask;
4199         }
4200         max_tr.entries = ring_buffer_size(max_tr.buffer);
4201         WARN_ON(max_tr.entries != global_trace.entries);
4202 #endif
4203
4204         /* Allocate the first page for all buffers */
4205         for_each_tracing_cpu(i) {
4206                 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
4207                 max_tr.data[i] = &per_cpu(max_data, i);
4208         }
4209
4210         trace_init_cmdlines();
4211
4212         register_tracer(&nop_trace);
4213         current_trace = &nop_trace;
4214 #ifdef CONFIG_BOOT_TRACER
4215         register_tracer(&boot_tracer);
4216 #endif
4217         /* All seems OK, enable tracing */
4218         tracing_disabled = 0;
4219
4220         atomic_notifier_chain_register(&panic_notifier_list,
4221                                        &trace_panic_notifier);
4222
4223         register_die_notifier(&trace_die_notifier);
4224
4225         return 0;
4226
4227 out_free_cpumask:
4228         free_cpumask_var(tracing_reader_cpumask);
4229 out_free_tracing_cpumask:
4230         free_cpumask_var(tracing_cpumask);
4231 out_free_buffer_mask:
4232         free_cpumask_var(tracing_buffer_mask);
4233 out:
4234         return ret;
4235 }
4236
4237 __init static int clear_boot_tracer(void)
4238 {
4239         /*
4240          * The default tracer at boot buffer is an init section.
4241          * This function is called in lateinit. If we did not
4242          * find the boot tracer, then clear it out, to prevent
4243          * later registration from accessing the buffer that is
4244          * about to be freed.
4245          */
4246         if (!default_bootup_tracer)
4247                 return 0;
4248
4249         printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4250                default_bootup_tracer);
4251         default_bootup_tracer = NULL;
4252
4253         return 0;
4254 }
4255
4256 early_initcall(tracer_alloc_buffers);
4257 fs_initcall(tracer_init_debugfs);
4258 late_initcall(clear_boot_tracer);