perf: Fix strncpy() use, use strlcpy() instead of strncpy()
[linux-3.10.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 Nadia Yvette Chambers
13  */
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/irq_work.h>
23 #include <linux/debugfs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/rwsem.h>
36 #include <linux/slab.h>
37 #include <linux/ctype.h>
38 #include <linux/init.h>
39 #include <linux/poll.h>
40 #include <linux/nmi.h>
41 #include <linux/fs.h>
42 #include <linux/sched/rt.h>
43
44 #include "trace.h"
45 #include "trace_output.h"
46
47 /*
48  * On boot up, the ring buffer is set to the minimum size, so that
49  * we do not waste memory on systems that are not using tracing.
50  */
51 int ring_buffer_expanded;
52
53 /*
54  * We need to change this state when a selftest is running.
55  * A selftest will lurk into the ring-buffer to count the
56  * entries inserted during the selftest although some concurrent
57  * insertions into the ring-buffer such as trace_printk could occurred
58  * at the same time, giving false positive or negative results.
59  */
60 static bool __read_mostly tracing_selftest_running;
61
62 /*
63  * If a tracer is running, we do not want to run SELFTEST.
64  */
65 bool __read_mostly tracing_selftest_disabled;
66
67 /* For tracers that don't implement custom flags */
68 static struct tracer_opt dummy_tracer_opt[] = {
69         { }
70 };
71
72 static struct tracer_flags dummy_tracer_flags = {
73         .val = 0,
74         .opts = dummy_tracer_opt
75 };
76
77 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
78 {
79         return 0;
80 }
81
82 /*
83  * To prevent the comm cache from being overwritten when no
84  * tracing is active, only save the comm when a trace event
85  * occurred.
86  */
87 static DEFINE_PER_CPU(bool, trace_cmdline_save);
88
89 /*
90  * When a reader is waiting for data, then this variable is
91  * set to true.
92  */
93 static bool trace_wakeup_needed;
94
95 static struct irq_work trace_work_wakeup;
96
97 /*
98  * Kill all tracing for good (never come back).
99  * It is initialized to 1 but will turn to zero if the initialization
100  * of the tracer is successful. But that is the only place that sets
101  * this back to zero.
102  */
103 static int tracing_disabled = 1;
104
105 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
106
107 cpumask_var_t __read_mostly     tracing_buffer_mask;
108
109 /*
110  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
111  *
112  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
113  * is set, then ftrace_dump is called. This will output the contents
114  * of the ftrace buffers to the console.  This is very useful for
115  * capturing traces that lead to crashes and outputing it to a
116  * serial console.
117  *
118  * It is default off, but you can enable it with either specifying
119  * "ftrace_dump_on_oops" in the kernel command line, or setting
120  * /proc/sys/kernel/ftrace_dump_on_oops
121  * Set 1 if you want to dump buffers of all CPUs
122  * Set 2 if you want to dump the buffer of the CPU that triggered oops
123  */
124
125 enum ftrace_dump_mode ftrace_dump_on_oops;
126
127 static int tracing_set_tracer(const char *buf);
128
129 #define MAX_TRACER_SIZE         100
130 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
131 static char *default_bootup_tracer;
132
133 static int __init set_cmdline_ftrace(char *str)
134 {
135         strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
136         default_bootup_tracer = bootup_tracer_buf;
137         /* We are using ftrace early, expand it */
138         ring_buffer_expanded = 1;
139         return 1;
140 }
141 __setup("ftrace=", set_cmdline_ftrace);
142
143 static int __init set_ftrace_dump_on_oops(char *str)
144 {
145         if (*str++ != '=' || !*str) {
146                 ftrace_dump_on_oops = DUMP_ALL;
147                 return 1;
148         }
149
150         if (!strcmp("orig_cpu", str)) {
151                 ftrace_dump_on_oops = DUMP_ORIG;
152                 return 1;
153         }
154
155         return 0;
156 }
157 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
158
159
160 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
161 static char *trace_boot_options __initdata;
162
163 static int __init set_trace_boot_options(char *str)
164 {
165         strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
166         trace_boot_options = trace_boot_options_buf;
167         return 0;
168 }
169 __setup("trace_options=", set_trace_boot_options);
170
171 unsigned long long ns2usecs(cycle_t nsec)
172 {
173         nsec += 500;
174         do_div(nsec, 1000);
175         return nsec;
176 }
177
178 /*
179  * The global_trace is the descriptor that holds the tracing
180  * buffers for the live tracing. For each CPU, it contains
181  * a link list of pages that will store trace entries. The
182  * page descriptor of the pages in the memory is used to hold
183  * the link list by linking the lru item in the page descriptor
184  * to each of the pages in the buffer per CPU.
185  *
186  * For each active CPU there is a data field that holds the
187  * pages for the buffer for that CPU. Each CPU has the same number
188  * of pages allocated for its buffer.
189  */
190 static struct trace_array       global_trace;
191
192 static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
193
194 int filter_current_check_discard(struct ring_buffer *buffer,
195                                  struct ftrace_event_call *call, void *rec,
196                                  struct ring_buffer_event *event)
197 {
198         return filter_check_discard(call, rec, buffer, event);
199 }
200 EXPORT_SYMBOL_GPL(filter_current_check_discard);
201
202 cycle_t ftrace_now(int cpu)
203 {
204         u64 ts;
205
206         /* Early boot up does not have a buffer yet */
207         if (!global_trace.buffer)
208                 return trace_clock_local();
209
210         ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
211         ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
212
213         return ts;
214 }
215
216 /*
217  * The max_tr is used to snapshot the global_trace when a maximum
218  * latency is reached. Some tracers will use this to store a maximum
219  * trace while it continues examining live traces.
220  *
221  * The buffers for the max_tr are set up the same as the global_trace.
222  * When a snapshot is taken, the link list of the max_tr is swapped
223  * with the link list of the global_trace and the buffers are reset for
224  * the global_trace so the tracing can continue.
225  */
226 static struct trace_array       max_tr;
227
228 static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
229
230 int tracing_is_enabled(void)
231 {
232         return tracing_is_on();
233 }
234
235 /*
236  * trace_buf_size is the size in bytes that is allocated
237  * for a buffer. Note, the number of bytes is always rounded
238  * to page size.
239  *
240  * This number is purposely set to a low number of 16384.
241  * If the dump on oops happens, it will be much appreciated
242  * to not have to wait for all that output. Anyway this can be
243  * boot time and run time configurable.
244  */
245 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
246
247 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
248
249 /* trace_types holds a link list of available tracers. */
250 static struct tracer            *trace_types __read_mostly;
251
252 /* current_trace points to the tracer that is currently active */
253 static struct tracer            *current_trace __read_mostly = &nop_trace;
254
255 /*
256  * trace_types_lock is used to protect the trace_types list.
257  */
258 static DEFINE_MUTEX(trace_types_lock);
259
260 /*
261  * serialize the access of the ring buffer
262  *
263  * ring buffer serializes readers, but it is low level protection.
264  * The validity of the events (which returns by ring_buffer_peek() ..etc)
265  * are not protected by ring buffer.
266  *
267  * The content of events may become garbage if we allow other process consumes
268  * these events concurrently:
269  *   A) the page of the consumed events may become a normal page
270  *      (not reader page) in ring buffer, and this page will be rewrited
271  *      by events producer.
272  *   B) The page of the consumed events may become a page for splice_read,
273  *      and this page will be returned to system.
274  *
275  * These primitives allow multi process access to different cpu ring buffer
276  * concurrently.
277  *
278  * These primitives don't distinguish read-only and read-consume access.
279  * Multi read-only access are also serialized.
280  */
281
282 #ifdef CONFIG_SMP
283 static DECLARE_RWSEM(all_cpu_access_lock);
284 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
285
286 static inline void trace_access_lock(int cpu)
287 {
288         if (cpu == TRACE_PIPE_ALL_CPU) {
289                 /* gain it for accessing the whole ring buffer. */
290                 down_write(&all_cpu_access_lock);
291         } else {
292                 /* gain it for accessing a cpu ring buffer. */
293
294                 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
295                 down_read(&all_cpu_access_lock);
296
297                 /* Secondly block other access to this @cpu ring buffer. */
298                 mutex_lock(&per_cpu(cpu_access_lock, cpu));
299         }
300 }
301
302 static inline void trace_access_unlock(int cpu)
303 {
304         if (cpu == TRACE_PIPE_ALL_CPU) {
305                 up_write(&all_cpu_access_lock);
306         } else {
307                 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
308                 up_read(&all_cpu_access_lock);
309         }
310 }
311
312 static inline void trace_access_lock_init(void)
313 {
314         int cpu;
315
316         for_each_possible_cpu(cpu)
317                 mutex_init(&per_cpu(cpu_access_lock, cpu));
318 }
319
320 #else
321
322 static DEFINE_MUTEX(access_lock);
323
324 static inline void trace_access_lock(int cpu)
325 {
326         (void)cpu;
327         mutex_lock(&access_lock);
328 }
329
330 static inline void trace_access_unlock(int cpu)
331 {
332         (void)cpu;
333         mutex_unlock(&access_lock);
334 }
335
336 static inline void trace_access_lock_init(void)
337 {
338 }
339
340 #endif
341
342 /* trace_wait is a waitqueue for tasks blocked on trace_poll */
343 static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
344
345 /* trace_flags holds trace_options default values */
346 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
347         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
348         TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
349         TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS;
350
351 static int trace_stop_count;
352 static DEFINE_RAW_SPINLOCK(tracing_start_lock);
353
354 /**
355  * trace_wake_up - wake up tasks waiting for trace input
356  *
357  * Schedules a delayed work to wake up any task that is blocked on the
358  * trace_wait queue. These is used with trace_poll for tasks polling the
359  * trace.
360  */
361 static void trace_wake_up(struct irq_work *work)
362 {
363         wake_up_all(&trace_wait);
364
365 }
366
367 /**
368  * tracing_on - enable tracing buffers
369  *
370  * This function enables tracing buffers that may have been
371  * disabled with tracing_off.
372  */
373 void tracing_on(void)
374 {
375         if (global_trace.buffer)
376                 ring_buffer_record_on(global_trace.buffer);
377         /*
378          * This flag is only looked at when buffers haven't been
379          * allocated yet. We don't really care about the race
380          * between setting this flag and actually turning
381          * on the buffer.
382          */
383         global_trace.buffer_disabled = 0;
384 }
385 EXPORT_SYMBOL_GPL(tracing_on);
386
387 /**
388  * tracing_off - turn off tracing buffers
389  *
390  * This function stops the tracing buffers from recording data.
391  * It does not disable any overhead the tracers themselves may
392  * be causing. This function simply causes all recording to
393  * the ring buffers to fail.
394  */
395 void tracing_off(void)
396 {
397         if (global_trace.buffer)
398                 ring_buffer_record_off(global_trace.buffer);
399         /*
400          * This flag is only looked at when buffers haven't been
401          * allocated yet. We don't really care about the race
402          * between setting this flag and actually turning
403          * on the buffer.
404          */
405         global_trace.buffer_disabled = 1;
406 }
407 EXPORT_SYMBOL_GPL(tracing_off);
408
409 /**
410  * tracing_is_on - show state of ring buffers enabled
411  */
412 int tracing_is_on(void)
413 {
414         if (global_trace.buffer)
415                 return ring_buffer_record_is_on(global_trace.buffer);
416         return !global_trace.buffer_disabled;
417 }
418 EXPORT_SYMBOL_GPL(tracing_is_on);
419
420 static int __init set_buf_size(char *str)
421 {
422         unsigned long buf_size;
423
424         if (!str)
425                 return 0;
426         buf_size = memparse(str, &str);
427         /* nr_entries can not be zero */
428         if (buf_size == 0)
429                 return 0;
430         trace_buf_size = buf_size;
431         return 1;
432 }
433 __setup("trace_buf_size=", set_buf_size);
434
435 static int __init set_tracing_thresh(char *str)
436 {
437         unsigned long threshold;
438         int ret;
439
440         if (!str)
441                 return 0;
442         ret = kstrtoul(str, 0, &threshold);
443         if (ret < 0)
444                 return 0;
445         tracing_thresh = threshold * 1000;
446         return 1;
447 }
448 __setup("tracing_thresh=", set_tracing_thresh);
449
450 unsigned long nsecs_to_usecs(unsigned long nsecs)
451 {
452         return nsecs / 1000;
453 }
454
455 /* These must match the bit postions in trace_iterator_flags */
456 static const char *trace_options[] = {
457         "print-parent",
458         "sym-offset",
459         "sym-addr",
460         "verbose",
461         "raw",
462         "hex",
463         "bin",
464         "block",
465         "stacktrace",
466         "trace_printk",
467         "ftrace_preempt",
468         "branch",
469         "annotate",
470         "userstacktrace",
471         "sym-userobj",
472         "printk-msg-only",
473         "context-info",
474         "latency-format",
475         "sleep-time",
476         "graph-time",
477         "record-cmd",
478         "overwrite",
479         "disable_on_free",
480         "irq-info",
481         "markers",
482         NULL
483 };
484
485 static struct {
486         u64 (*func)(void);
487         const char *name;
488         int in_ns;              /* is this clock in nanoseconds? */
489 } trace_clocks[] = {
490         { trace_clock_local,    "local",        1 },
491         { trace_clock_global,   "global",       1 },
492         { trace_clock_counter,  "counter",      0 },
493         ARCH_TRACE_CLOCKS
494 };
495
496 int trace_clock_id;
497
498 /*
499  * trace_parser_get_init - gets the buffer for trace parser
500  */
501 int trace_parser_get_init(struct trace_parser *parser, int size)
502 {
503         memset(parser, 0, sizeof(*parser));
504
505         parser->buffer = kmalloc(size, GFP_KERNEL);
506         if (!parser->buffer)
507                 return 1;
508
509         parser->size = size;
510         return 0;
511 }
512
513 /*
514  * trace_parser_put - frees the buffer for trace parser
515  */
516 void trace_parser_put(struct trace_parser *parser)
517 {
518         kfree(parser->buffer);
519 }
520
521 /*
522  * trace_get_user - reads the user input string separated by  space
523  * (matched by isspace(ch))
524  *
525  * For each string found the 'struct trace_parser' is updated,
526  * and the function returns.
527  *
528  * Returns number of bytes read.
529  *
530  * See kernel/trace/trace.h for 'struct trace_parser' details.
531  */
532 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
533         size_t cnt, loff_t *ppos)
534 {
535         char ch;
536         size_t read = 0;
537         ssize_t ret;
538
539         if (!*ppos)
540                 trace_parser_clear(parser);
541
542         ret = get_user(ch, ubuf++);
543         if (ret)
544                 goto out;
545
546         read++;
547         cnt--;
548
549         /*
550          * The parser is not finished with the last write,
551          * continue reading the user input without skipping spaces.
552          */
553         if (!parser->cont) {
554                 /* skip white space */
555                 while (cnt && isspace(ch)) {
556                         ret = get_user(ch, ubuf++);
557                         if (ret)
558                                 goto out;
559                         read++;
560                         cnt--;
561                 }
562
563                 /* only spaces were written */
564                 if (isspace(ch)) {
565                         *ppos += read;
566                         ret = read;
567                         goto out;
568                 }
569
570                 parser->idx = 0;
571         }
572
573         /* read the non-space input */
574         while (cnt && !isspace(ch)) {
575                 if (parser->idx < parser->size - 1)
576                         parser->buffer[parser->idx++] = ch;
577                 else {
578                         ret = -EINVAL;
579                         goto out;
580                 }
581                 ret = get_user(ch, ubuf++);
582                 if (ret)
583                         goto out;
584                 read++;
585                 cnt--;
586         }
587
588         /* We either got finished input or we have to wait for another call. */
589         if (isspace(ch)) {
590                 parser->buffer[parser->idx] = 0;
591                 parser->cont = false;
592         } else {
593                 parser->cont = true;
594                 parser->buffer[parser->idx++] = ch;
595         }
596
597         *ppos += read;
598         ret = read;
599
600 out:
601         return ret;
602 }
603
604 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
605 {
606         int len;
607         int ret;
608
609         if (!cnt)
610                 return 0;
611
612         if (s->len <= s->readpos)
613                 return -EBUSY;
614
615         len = s->len - s->readpos;
616         if (cnt > len)
617                 cnt = len;
618         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
619         if (ret == cnt)
620                 return -EFAULT;
621
622         cnt -= ret;
623
624         s->readpos += cnt;
625         return cnt;
626 }
627
628 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
629 {
630         int len;
631
632         if (s->len <= s->readpos)
633                 return -EBUSY;
634
635         len = s->len - s->readpos;
636         if (cnt > len)
637                 cnt = len;
638         memcpy(buf, s->buffer + s->readpos, cnt);
639
640         s->readpos += cnt;
641         return cnt;
642 }
643
644 /*
645  * ftrace_max_lock is used to protect the swapping of buffers
646  * when taking a max snapshot. The buffers themselves are
647  * protected by per_cpu spinlocks. But the action of the swap
648  * needs its own lock.
649  *
650  * This is defined as a arch_spinlock_t in order to help
651  * with performance when lockdep debugging is enabled.
652  *
653  * It is also used in other places outside the update_max_tr
654  * so it needs to be defined outside of the
655  * CONFIG_TRACER_MAX_TRACE.
656  */
657 static arch_spinlock_t ftrace_max_lock =
658         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
659
660 unsigned long __read_mostly     tracing_thresh;
661
662 #ifdef CONFIG_TRACER_MAX_TRACE
663 unsigned long __read_mostly     tracing_max_latency;
664
665 /*
666  * Copy the new maximum trace into the separate maximum-trace
667  * structure. (this way the maximum trace is permanently saved,
668  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
669  */
670 static void
671 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
672 {
673         struct trace_array_cpu *data = tr->data[cpu];
674         struct trace_array_cpu *max_data;
675
676         max_tr.cpu = cpu;
677         max_tr.time_start = data->preempt_timestamp;
678
679         max_data = max_tr.data[cpu];
680         max_data->saved_latency = tracing_max_latency;
681         max_data->critical_start = data->critical_start;
682         max_data->critical_end = data->critical_end;
683
684         memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
685         max_data->pid = tsk->pid;
686         max_data->uid = task_uid(tsk);
687         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
688         max_data->policy = tsk->policy;
689         max_data->rt_priority = tsk->rt_priority;
690
691         /* record this tasks comm */
692         tracing_record_cmdline(tsk);
693 }
694
695 /**
696  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
697  * @tr: tracer
698  * @tsk: the task with the latency
699  * @cpu: The cpu that initiated the trace.
700  *
701  * Flip the buffers between the @tr and the max_tr and record information
702  * about which task was the cause of this latency.
703  */
704 void
705 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
706 {
707         struct ring_buffer *buf;
708
709         if (trace_stop_count)
710                 return;
711
712         WARN_ON_ONCE(!irqs_disabled());
713
714         if (!current_trace->allocated_snapshot) {
715                 /* Only the nop tracer should hit this when disabling */
716                 WARN_ON_ONCE(current_trace != &nop_trace);
717                 return;
718         }
719
720         arch_spin_lock(&ftrace_max_lock);
721
722         buf = tr->buffer;
723         tr->buffer = max_tr.buffer;
724         max_tr.buffer = buf;
725
726         __update_max_tr(tr, tsk, cpu);
727         arch_spin_unlock(&ftrace_max_lock);
728 }
729
730 /**
731  * update_max_tr_single - only copy one trace over, and reset the rest
732  * @tr - tracer
733  * @tsk - task with the latency
734  * @cpu - the cpu of the buffer to copy.
735  *
736  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
737  */
738 void
739 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
740 {
741         int ret;
742
743         if (trace_stop_count)
744                 return;
745
746         WARN_ON_ONCE(!irqs_disabled());
747         if (WARN_ON_ONCE(!current_trace->allocated_snapshot))
748                 return;
749
750         arch_spin_lock(&ftrace_max_lock);
751
752         ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
753
754         if (ret == -EBUSY) {
755                 /*
756                  * We failed to swap the buffer due to a commit taking
757                  * place on this CPU. We fail to record, but we reset
758                  * the max trace buffer (no one writes directly to it)
759                  * and flag that it failed.
760                  */
761                 trace_array_printk(&max_tr, _THIS_IP_,
762                         "Failed to swap buffers due to commit in progress\n");
763         }
764
765         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
766
767         __update_max_tr(tr, tsk, cpu);
768         arch_spin_unlock(&ftrace_max_lock);
769 }
770 #endif /* CONFIG_TRACER_MAX_TRACE */
771
772 static void default_wait_pipe(struct trace_iterator *iter)
773 {
774         DEFINE_WAIT(wait);
775
776         prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
777
778         /*
779          * The events can happen in critical sections where
780          * checking a work queue can cause deadlocks.
781          * After adding a task to the queue, this flag is set
782          * only to notify events to try to wake up the queue
783          * using irq_work.
784          *
785          * We don't clear it even if the buffer is no longer
786          * empty. The flag only causes the next event to run
787          * irq_work to do the work queue wake up. The worse
788          * that can happen if we race with !trace_empty() is that
789          * an event will cause an irq_work to try to wake up
790          * an empty queue.
791          *
792          * There's no reason to protect this flag either, as
793          * the work queue and irq_work logic will do the necessary
794          * synchronization for the wake ups. The only thing
795          * that is necessary is that the wake up happens after
796          * a task has been queued. It's OK for spurious wake ups.
797          */
798         trace_wakeup_needed = true;
799
800         if (trace_empty(iter))
801                 schedule();
802
803         finish_wait(&trace_wait, &wait);
804 }
805
806 /**
807  * register_tracer - register a tracer with the ftrace system.
808  * @type - the plugin for the tracer
809  *
810  * Register a new plugin tracer.
811  */
812 int register_tracer(struct tracer *type)
813 {
814         struct tracer *t;
815         int ret = 0;
816
817         if (!type->name) {
818                 pr_info("Tracer must have a name\n");
819                 return -1;
820         }
821
822         if (strlen(type->name) >= MAX_TRACER_SIZE) {
823                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
824                 return -1;
825         }
826
827         mutex_lock(&trace_types_lock);
828
829         tracing_selftest_running = true;
830
831         for (t = trace_types; t; t = t->next) {
832                 if (strcmp(type->name, t->name) == 0) {
833                         /* already found */
834                         pr_info("Tracer %s already registered\n",
835                                 type->name);
836                         ret = -1;
837                         goto out;
838                 }
839         }
840
841         if (!type->set_flag)
842                 type->set_flag = &dummy_set_flag;
843         if (!type->flags)
844                 type->flags = &dummy_tracer_flags;
845         else
846                 if (!type->flags->opts)
847                         type->flags->opts = dummy_tracer_opt;
848         if (!type->wait_pipe)
849                 type->wait_pipe = default_wait_pipe;
850
851
852 #ifdef CONFIG_FTRACE_STARTUP_TEST
853         if (type->selftest && !tracing_selftest_disabled) {
854                 struct tracer *saved_tracer = current_trace;
855                 struct trace_array *tr = &global_trace;
856
857                 /*
858                  * Run a selftest on this tracer.
859                  * Here we reset the trace buffer, and set the current
860                  * tracer to be this tracer. The tracer can then run some
861                  * internal tracing to verify that everything is in order.
862                  * If we fail, we do not register this tracer.
863                  */
864                 tracing_reset_online_cpus(tr);
865
866                 current_trace = type;
867
868                 if (type->use_max_tr) {
869                         /* If we expanded the buffers, make sure the max is expanded too */
870                         if (ring_buffer_expanded)
871                                 ring_buffer_resize(max_tr.buffer, trace_buf_size,
872                                                    RING_BUFFER_ALL_CPUS);
873                         type->allocated_snapshot = true;
874                 }
875
876                 /* the test is responsible for initializing and enabling */
877                 pr_info("Testing tracer %s: ", type->name);
878                 ret = type->selftest(type, tr);
879                 /* the test is responsible for resetting too */
880                 current_trace = saved_tracer;
881                 if (ret) {
882                         printk(KERN_CONT "FAILED!\n");
883                         /* Add the warning after printing 'FAILED' */
884                         WARN_ON(1);
885                         goto out;
886                 }
887                 /* Only reset on passing, to avoid touching corrupted buffers */
888                 tracing_reset_online_cpus(tr);
889
890                 if (type->use_max_tr) {
891                         type->allocated_snapshot = false;
892
893                         /* Shrink the max buffer again */
894                         if (ring_buffer_expanded)
895                                 ring_buffer_resize(max_tr.buffer, 1,
896                                                    RING_BUFFER_ALL_CPUS);
897                 }
898
899                 printk(KERN_CONT "PASSED\n");
900         }
901 #endif
902
903         type->next = trace_types;
904         trace_types = type;
905
906  out:
907         tracing_selftest_running = false;
908         mutex_unlock(&trace_types_lock);
909
910         if (ret || !default_bootup_tracer)
911                 goto out_unlock;
912
913         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
914                 goto out_unlock;
915
916         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
917         /* Do we want this tracer to start on bootup? */
918         tracing_set_tracer(type->name);
919         default_bootup_tracer = NULL;
920         /* disable other selftests, since this will break it. */
921         tracing_selftest_disabled = 1;
922 #ifdef CONFIG_FTRACE_STARTUP_TEST
923         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
924                type->name);
925 #endif
926
927  out_unlock:
928         return ret;
929 }
930
931 void tracing_reset(struct trace_array *tr, int cpu)
932 {
933         struct ring_buffer *buffer = tr->buffer;
934
935         if (!buffer)
936                 return;
937
938         ring_buffer_record_disable(buffer);
939
940         /* Make sure all commits have finished */
941         synchronize_sched();
942         ring_buffer_reset_cpu(buffer, cpu);
943
944         ring_buffer_record_enable(buffer);
945 }
946
947 void tracing_reset_online_cpus(struct trace_array *tr)
948 {
949         struct ring_buffer *buffer = tr->buffer;
950         int cpu;
951
952         if (!buffer)
953                 return;
954
955         ring_buffer_record_disable(buffer);
956
957         /* Make sure all commits have finished */
958         synchronize_sched();
959
960         tr->time_start = ftrace_now(tr->cpu);
961
962         for_each_online_cpu(cpu)
963                 ring_buffer_reset_cpu(buffer, cpu);
964
965         ring_buffer_record_enable(buffer);
966 }
967
968 void tracing_reset_current(int cpu)
969 {
970         tracing_reset(&global_trace, cpu);
971 }
972
973 void tracing_reset_current_online_cpus(void)
974 {
975         tracing_reset_online_cpus(&global_trace);
976 }
977
978 #define SAVED_CMDLINES 128
979 #define NO_CMDLINE_MAP UINT_MAX
980 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
981 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
982 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
983 static int cmdline_idx;
984 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
985
986 /* temporary disable recording */
987 static atomic_t trace_record_cmdline_disabled __read_mostly;
988
989 static void trace_init_cmdlines(void)
990 {
991         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
992         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
993         cmdline_idx = 0;
994 }
995
996 int is_tracing_stopped(void)
997 {
998         return trace_stop_count;
999 }
1000
1001 /**
1002  * ftrace_off_permanent - disable all ftrace code permanently
1003  *
1004  * This should only be called when a serious anomally has
1005  * been detected.  This will turn off the function tracing,
1006  * ring buffers, and other tracing utilites. It takes no
1007  * locks and can be called from any context.
1008  */
1009 void ftrace_off_permanent(void)
1010 {
1011         tracing_disabled = 1;
1012         ftrace_stop();
1013         tracing_off_permanent();
1014 }
1015
1016 /**
1017  * tracing_start - quick start of the tracer
1018  *
1019  * If tracing is enabled but was stopped by tracing_stop,
1020  * this will start the tracer back up.
1021  */
1022 void tracing_start(void)
1023 {
1024         struct ring_buffer *buffer;
1025         unsigned long flags;
1026
1027         if (tracing_disabled)
1028                 return;
1029
1030         raw_spin_lock_irqsave(&tracing_start_lock, flags);
1031         if (--trace_stop_count) {
1032                 if (trace_stop_count < 0) {
1033                         /* Someone screwed up their debugging */
1034                         WARN_ON_ONCE(1);
1035                         trace_stop_count = 0;
1036                 }
1037                 goto out;
1038         }
1039
1040         /* Prevent the buffers from switching */
1041         arch_spin_lock(&ftrace_max_lock);
1042
1043         buffer = global_trace.buffer;
1044         if (buffer)
1045                 ring_buffer_record_enable(buffer);
1046
1047         buffer = max_tr.buffer;
1048         if (buffer)
1049                 ring_buffer_record_enable(buffer);
1050
1051         arch_spin_unlock(&ftrace_max_lock);
1052
1053         ftrace_start();
1054  out:
1055         raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1056 }
1057
1058 /**
1059  * tracing_stop - quick stop of the tracer
1060  *
1061  * Light weight way to stop tracing. Use in conjunction with
1062  * tracing_start.
1063  */
1064 void tracing_stop(void)
1065 {
1066         struct ring_buffer *buffer;
1067         unsigned long flags;
1068
1069         ftrace_stop();
1070         raw_spin_lock_irqsave(&tracing_start_lock, flags);
1071         if (trace_stop_count++)
1072                 goto out;
1073
1074         /* Prevent the buffers from switching */
1075         arch_spin_lock(&ftrace_max_lock);
1076
1077         buffer = global_trace.buffer;
1078         if (buffer)
1079                 ring_buffer_record_disable(buffer);
1080
1081         buffer = max_tr.buffer;
1082         if (buffer)
1083                 ring_buffer_record_disable(buffer);
1084
1085         arch_spin_unlock(&ftrace_max_lock);
1086
1087  out:
1088         raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
1089 }
1090
1091 void trace_stop_cmdline_recording(void);
1092
1093 static void trace_save_cmdline(struct task_struct *tsk)
1094 {
1095         unsigned pid, idx;
1096
1097         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1098                 return;
1099
1100         /*
1101          * It's not the end of the world if we don't get
1102          * the lock, but we also don't want to spin
1103          * nor do we want to disable interrupts,
1104          * so if we miss here, then better luck next time.
1105          */
1106         if (!arch_spin_trylock(&trace_cmdline_lock))
1107                 return;
1108
1109         idx = map_pid_to_cmdline[tsk->pid];
1110         if (idx == NO_CMDLINE_MAP) {
1111                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1112
1113                 /*
1114                  * Check whether the cmdline buffer at idx has a pid
1115                  * mapped. We are going to overwrite that entry so we
1116                  * need to clear the map_pid_to_cmdline. Otherwise we
1117                  * would read the new comm for the old pid.
1118                  */
1119                 pid = map_cmdline_to_pid[idx];
1120                 if (pid != NO_CMDLINE_MAP)
1121                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1122
1123                 map_cmdline_to_pid[idx] = tsk->pid;
1124                 map_pid_to_cmdline[tsk->pid] = idx;
1125
1126                 cmdline_idx = idx;
1127         }
1128
1129         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1130
1131         arch_spin_unlock(&trace_cmdline_lock);
1132 }
1133
1134 void trace_find_cmdline(int pid, char comm[])
1135 {
1136         unsigned map;
1137
1138         if (!pid) {
1139                 strcpy(comm, "<idle>");
1140                 return;
1141         }
1142
1143         if (WARN_ON_ONCE(pid < 0)) {
1144                 strcpy(comm, "<XXX>");
1145                 return;
1146         }
1147
1148         if (pid > PID_MAX_DEFAULT) {
1149                 strcpy(comm, "<...>");
1150                 return;
1151         }
1152
1153         preempt_disable();
1154         arch_spin_lock(&trace_cmdline_lock);
1155         map = map_pid_to_cmdline[pid];
1156         if (map != NO_CMDLINE_MAP)
1157                 strcpy(comm, saved_cmdlines[map]);
1158         else
1159                 strcpy(comm, "<...>");
1160
1161         arch_spin_unlock(&trace_cmdline_lock);
1162         preempt_enable();
1163 }
1164
1165 void tracing_record_cmdline(struct task_struct *tsk)
1166 {
1167         if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1168                 return;
1169
1170         if (!__this_cpu_read(trace_cmdline_save))
1171                 return;
1172
1173         __this_cpu_write(trace_cmdline_save, false);
1174
1175         trace_save_cmdline(tsk);
1176 }
1177
1178 void
1179 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1180                              int pc)
1181 {
1182         struct task_struct *tsk = current;
1183
1184         entry->preempt_count            = pc & 0xff;
1185         entry->pid                      = (tsk) ? tsk->pid : 0;
1186         entry->flags =
1187 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1188                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1189 #else
1190                 TRACE_FLAG_IRQS_NOSUPPORT |
1191 #endif
1192                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1193                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1194                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1195 }
1196 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1197
1198 struct ring_buffer_event *
1199 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1200                           int type,
1201                           unsigned long len,
1202                           unsigned long flags, int pc)
1203 {
1204         struct ring_buffer_event *event;
1205
1206         event = ring_buffer_lock_reserve(buffer, len);
1207         if (event != NULL) {
1208                 struct trace_entry *ent = ring_buffer_event_data(event);
1209
1210                 tracing_generic_entry_update(ent, flags, pc);
1211                 ent->type = type;
1212         }
1213
1214         return event;
1215 }
1216
1217 void
1218 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1219 {
1220         __this_cpu_write(trace_cmdline_save, true);
1221         if (trace_wakeup_needed) {
1222                 trace_wakeup_needed = false;
1223                 /* irq_work_queue() supplies it's own memory barriers */
1224                 irq_work_queue(&trace_work_wakeup);
1225         }
1226         ring_buffer_unlock_commit(buffer, event);
1227 }
1228
1229 static inline void
1230 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1231                              struct ring_buffer_event *event,
1232                              unsigned long flags, int pc)
1233 {
1234         __buffer_unlock_commit(buffer, event);
1235
1236         ftrace_trace_stack(buffer, flags, 6, pc);
1237         ftrace_trace_userstack(buffer, flags, pc);
1238 }
1239
1240 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1241                                 struct ring_buffer_event *event,
1242                                 unsigned long flags, int pc)
1243 {
1244         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1245 }
1246 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1247
1248 struct ring_buffer_event *
1249 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1250                                   int type, unsigned long len,
1251                                   unsigned long flags, int pc)
1252 {
1253         *current_rb = global_trace.buffer;
1254         return trace_buffer_lock_reserve(*current_rb,
1255                                          type, len, flags, pc);
1256 }
1257 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1258
1259 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1260                                         struct ring_buffer_event *event,
1261                                         unsigned long flags, int pc)
1262 {
1263         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1264 }
1265 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1266
1267 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1268                                      struct ring_buffer_event *event,
1269                                      unsigned long flags, int pc,
1270                                      struct pt_regs *regs)
1271 {
1272         __buffer_unlock_commit(buffer, event);
1273
1274         ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1275         ftrace_trace_userstack(buffer, flags, pc);
1276 }
1277 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1278
1279 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1280                                          struct ring_buffer_event *event)
1281 {
1282         ring_buffer_discard_commit(buffer, event);
1283 }
1284 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1285
1286 void
1287 trace_function(struct trace_array *tr,
1288                unsigned long ip, unsigned long parent_ip, unsigned long flags,
1289                int pc)
1290 {
1291         struct ftrace_event_call *call = &event_function;
1292         struct ring_buffer *buffer = tr->buffer;
1293         struct ring_buffer_event *event;
1294         struct ftrace_entry *entry;
1295
1296         /* If we are reading the ring buffer, don't trace */
1297         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1298                 return;
1299
1300         event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1301                                           flags, pc);
1302         if (!event)
1303                 return;
1304         entry   = ring_buffer_event_data(event);
1305         entry->ip                       = ip;
1306         entry->parent_ip                = parent_ip;
1307
1308         if (!filter_check_discard(call, entry, buffer, event))
1309                 __buffer_unlock_commit(buffer, event);
1310 }
1311
1312 void
1313 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1314        unsigned long ip, unsigned long parent_ip, unsigned long flags,
1315        int pc)
1316 {
1317         if (likely(!atomic_read(&data->disabled)))
1318                 trace_function(tr, ip, parent_ip, flags, pc);
1319 }
1320
1321 #ifdef CONFIG_STACKTRACE
1322
1323 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1324 struct ftrace_stack {
1325         unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
1326 };
1327
1328 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1329 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1330
1331 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1332                                  unsigned long flags,
1333                                  int skip, int pc, struct pt_regs *regs)
1334 {
1335         struct ftrace_event_call *call = &event_kernel_stack;
1336         struct ring_buffer_event *event;
1337         struct stack_entry *entry;
1338         struct stack_trace trace;
1339         int use_stack;
1340         int size = FTRACE_STACK_ENTRIES;
1341
1342         trace.nr_entries        = 0;
1343         trace.skip              = skip;
1344
1345         /*
1346          * Since events can happen in NMIs there's no safe way to
1347          * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1348          * or NMI comes in, it will just have to use the default
1349          * FTRACE_STACK_SIZE.
1350          */
1351         preempt_disable_notrace();
1352
1353         use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1354         /*
1355          * We don't need any atomic variables, just a barrier.
1356          * If an interrupt comes in, we don't care, because it would
1357          * have exited and put the counter back to what we want.
1358          * We just need a barrier to keep gcc from moving things
1359          * around.
1360          */
1361         barrier();
1362         if (use_stack == 1) {
1363                 trace.entries           = &__get_cpu_var(ftrace_stack).calls[0];
1364                 trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
1365
1366                 if (regs)
1367                         save_stack_trace_regs(regs, &trace);
1368                 else
1369                         save_stack_trace(&trace);
1370
1371                 if (trace.nr_entries > size)
1372                         size = trace.nr_entries;
1373         } else
1374                 /* From now on, use_stack is a boolean */
1375                 use_stack = 0;
1376
1377         size *= sizeof(unsigned long);
1378
1379         event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1380                                           sizeof(*entry) + size, flags, pc);
1381         if (!event)
1382                 goto out;
1383         entry = ring_buffer_event_data(event);
1384
1385         memset(&entry->caller, 0, size);
1386
1387         if (use_stack)
1388                 memcpy(&entry->caller, trace.entries,
1389                        trace.nr_entries * sizeof(unsigned long));
1390         else {
1391                 trace.max_entries       = FTRACE_STACK_ENTRIES;
1392                 trace.entries           = entry->caller;
1393                 if (regs)
1394                         save_stack_trace_regs(regs, &trace);
1395                 else
1396                         save_stack_trace(&trace);
1397         }
1398
1399         entry->size = trace.nr_entries;
1400
1401         if (!filter_check_discard(call, entry, buffer, event))
1402                 __buffer_unlock_commit(buffer, event);
1403
1404  out:
1405         /* Again, don't let gcc optimize things here */
1406         barrier();
1407         __this_cpu_dec(ftrace_stack_reserve);
1408         preempt_enable_notrace();
1409
1410 }
1411
1412 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1413                              int skip, int pc, struct pt_regs *regs)
1414 {
1415         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1416                 return;
1417
1418         __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1419 }
1420
1421 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1422                         int skip, int pc)
1423 {
1424         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1425                 return;
1426
1427         __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1428 }
1429
1430 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1431                    int pc)
1432 {
1433         __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
1434 }
1435
1436 /**
1437  * trace_dump_stack - record a stack back trace in the trace buffer
1438  */
1439 void trace_dump_stack(void)
1440 {
1441         unsigned long flags;
1442
1443         if (tracing_disabled || tracing_selftest_running)
1444                 return;
1445
1446         local_save_flags(flags);
1447
1448         /* skipping 3 traces, seems to get us at the caller of this function */
1449         __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
1450 }
1451
1452 static DEFINE_PER_CPU(int, user_stack_count);
1453
1454 void
1455 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1456 {
1457         struct ftrace_event_call *call = &event_user_stack;
1458         struct ring_buffer_event *event;
1459         struct userstack_entry *entry;
1460         struct stack_trace trace;
1461
1462         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1463                 return;
1464
1465         /*
1466          * NMIs can not handle page faults, even with fix ups.
1467          * The save user stack can (and often does) fault.
1468          */
1469         if (unlikely(in_nmi()))
1470                 return;
1471
1472         /*
1473          * prevent recursion, since the user stack tracing may
1474          * trigger other kernel events.
1475          */
1476         preempt_disable();
1477         if (__this_cpu_read(user_stack_count))
1478                 goto out;
1479
1480         __this_cpu_inc(user_stack_count);
1481
1482         event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1483                                           sizeof(*entry), flags, pc);
1484         if (!event)
1485                 goto out_drop_count;
1486         entry   = ring_buffer_event_data(event);
1487
1488         entry->tgid             = current->tgid;
1489         memset(&entry->caller, 0, sizeof(entry->caller));
1490
1491         trace.nr_entries        = 0;
1492         trace.max_entries       = FTRACE_STACK_ENTRIES;
1493         trace.skip              = 0;
1494         trace.entries           = entry->caller;
1495
1496         save_stack_trace_user(&trace);
1497         if (!filter_check_discard(call, entry, buffer, event))
1498                 __buffer_unlock_commit(buffer, event);
1499
1500  out_drop_count:
1501         __this_cpu_dec(user_stack_count);
1502  out:
1503         preempt_enable();
1504 }
1505
1506 #ifdef UNUSED
1507 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1508 {
1509         ftrace_trace_userstack(tr, flags, preempt_count());
1510 }
1511 #endif /* UNUSED */
1512
1513 #endif /* CONFIG_STACKTRACE */
1514
1515 /* created for use with alloc_percpu */
1516 struct trace_buffer_struct {
1517         char buffer[TRACE_BUF_SIZE];
1518 };
1519
1520 static struct trace_buffer_struct *trace_percpu_buffer;
1521 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1522 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1523 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1524
1525 /*
1526  * The buffer used is dependent on the context. There is a per cpu
1527  * buffer for normal context, softirq contex, hard irq context and
1528  * for NMI context. Thise allows for lockless recording.
1529  *
1530  * Note, if the buffers failed to be allocated, then this returns NULL
1531  */
1532 static char *get_trace_buf(void)
1533 {
1534         struct trace_buffer_struct *percpu_buffer;
1535
1536         /*
1537          * If we have allocated per cpu buffers, then we do not
1538          * need to do any locking.
1539          */
1540         if (in_nmi())
1541                 percpu_buffer = trace_percpu_nmi_buffer;
1542         else if (in_irq())
1543                 percpu_buffer = trace_percpu_irq_buffer;
1544         else if (in_softirq())
1545                 percpu_buffer = trace_percpu_sirq_buffer;
1546         else
1547                 percpu_buffer = trace_percpu_buffer;
1548
1549         if (!percpu_buffer)
1550                 return NULL;
1551
1552         return this_cpu_ptr(&percpu_buffer->buffer[0]);
1553 }
1554
1555 static int alloc_percpu_trace_buffer(void)
1556 {
1557         struct trace_buffer_struct *buffers;
1558         struct trace_buffer_struct *sirq_buffers;
1559         struct trace_buffer_struct *irq_buffers;
1560         struct trace_buffer_struct *nmi_buffers;
1561
1562         buffers = alloc_percpu(struct trace_buffer_struct);
1563         if (!buffers)
1564                 goto err_warn;
1565
1566         sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1567         if (!sirq_buffers)
1568                 goto err_sirq;
1569
1570         irq_buffers = alloc_percpu(struct trace_buffer_struct);
1571         if (!irq_buffers)
1572                 goto err_irq;
1573
1574         nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1575         if (!nmi_buffers)
1576                 goto err_nmi;
1577
1578         trace_percpu_buffer = buffers;
1579         trace_percpu_sirq_buffer = sirq_buffers;
1580         trace_percpu_irq_buffer = irq_buffers;
1581         trace_percpu_nmi_buffer = nmi_buffers;
1582
1583         return 0;
1584
1585  err_nmi:
1586         free_percpu(irq_buffers);
1587  err_irq:
1588         free_percpu(sirq_buffers);
1589  err_sirq:
1590         free_percpu(buffers);
1591  err_warn:
1592         WARN(1, "Could not allocate percpu trace_printk buffer");
1593         return -ENOMEM;
1594 }
1595
1596 static int buffers_allocated;
1597
1598 void trace_printk_init_buffers(void)
1599 {
1600         if (buffers_allocated)
1601                 return;
1602
1603         if (alloc_percpu_trace_buffer())
1604                 return;
1605
1606         pr_info("ftrace: Allocated trace_printk buffers\n");
1607
1608         /* Expand the buffers to set size */
1609         tracing_update_buffers();
1610
1611         buffers_allocated = 1;
1612
1613         /*
1614          * trace_printk_init_buffers() can be called by modules.
1615          * If that happens, then we need to start cmdline recording
1616          * directly here. If the global_trace.buffer is already
1617          * allocated here, then this was called by module code.
1618          */
1619         if (global_trace.buffer)
1620                 tracing_start_cmdline_record();
1621 }
1622
1623 void trace_printk_start_comm(void)
1624 {
1625         /* Start tracing comms if trace printk is set */
1626         if (!buffers_allocated)
1627                 return;
1628         tracing_start_cmdline_record();
1629 }
1630
1631 static void trace_printk_start_stop_comm(int enabled)
1632 {
1633         if (!buffers_allocated)
1634                 return;
1635
1636         if (enabled)
1637                 tracing_start_cmdline_record();
1638         else
1639                 tracing_stop_cmdline_record();
1640 }
1641
1642 /**
1643  * trace_vbprintk - write binary msg to tracing buffer
1644  *
1645  */
1646 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1647 {
1648         struct ftrace_event_call *call = &event_bprint;
1649         struct ring_buffer_event *event;
1650         struct ring_buffer *buffer;
1651         struct trace_array *tr = &global_trace;
1652         struct bprint_entry *entry;
1653         unsigned long flags;
1654         char *tbuffer;
1655         int len = 0, size, pc;
1656
1657         if (unlikely(tracing_selftest_running || tracing_disabled))
1658                 return 0;
1659
1660         /* Don't pollute graph traces with trace_vprintk internals */
1661         pause_graph_tracing();
1662
1663         pc = preempt_count();
1664         preempt_disable_notrace();
1665
1666         tbuffer = get_trace_buf();
1667         if (!tbuffer) {
1668                 len = 0;
1669                 goto out;
1670         }
1671
1672         len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1673
1674         if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1675                 goto out;
1676
1677         local_save_flags(flags);
1678         size = sizeof(*entry) + sizeof(u32) * len;
1679         buffer = tr->buffer;
1680         event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1681                                           flags, pc);
1682         if (!event)
1683                 goto out;
1684         entry = ring_buffer_event_data(event);
1685         entry->ip                       = ip;
1686         entry->fmt                      = fmt;
1687
1688         memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1689         if (!filter_check_discard(call, entry, buffer, event)) {
1690                 __buffer_unlock_commit(buffer, event);
1691                 ftrace_trace_stack(buffer, flags, 6, pc);
1692         }
1693
1694 out:
1695         preempt_enable_notrace();
1696         unpause_graph_tracing();
1697
1698         return len;
1699 }
1700 EXPORT_SYMBOL_GPL(trace_vbprintk);
1701
1702 int trace_array_printk(struct trace_array *tr,
1703                        unsigned long ip, const char *fmt, ...)
1704 {
1705         int ret;
1706         va_list ap;
1707
1708         if (!(trace_flags & TRACE_ITER_PRINTK))
1709                 return 0;
1710
1711         va_start(ap, fmt);
1712         ret = trace_array_vprintk(tr, ip, fmt, ap);
1713         va_end(ap);
1714         return ret;
1715 }
1716
1717 int trace_array_vprintk(struct trace_array *tr,
1718                         unsigned long ip, const char *fmt, va_list args)
1719 {
1720         struct ftrace_event_call *call = &event_print;
1721         struct ring_buffer_event *event;
1722         struct ring_buffer *buffer;
1723         int len = 0, size, pc;
1724         struct print_entry *entry;
1725         unsigned long flags;
1726         char *tbuffer;
1727
1728         if (tracing_disabled || tracing_selftest_running)
1729                 return 0;
1730
1731         /* Don't pollute graph traces with trace_vprintk internals */
1732         pause_graph_tracing();
1733
1734         pc = preempt_count();
1735         preempt_disable_notrace();
1736
1737
1738         tbuffer = get_trace_buf();
1739         if (!tbuffer) {
1740                 len = 0;
1741                 goto out;
1742         }
1743
1744         len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1745         if (len > TRACE_BUF_SIZE)
1746                 goto out;
1747
1748         local_save_flags(flags);
1749         size = sizeof(*entry) + len + 1;
1750         buffer = tr->buffer;
1751         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1752                                           flags, pc);
1753         if (!event)
1754                 goto out;
1755         entry = ring_buffer_event_data(event);
1756         entry->ip = ip;
1757
1758         memcpy(&entry->buf, tbuffer, len);
1759         entry->buf[len] = '\0';
1760         if (!filter_check_discard(call, entry, buffer, event)) {
1761                 __buffer_unlock_commit(buffer, event);
1762                 ftrace_trace_stack(buffer, flags, 6, pc);
1763         }
1764  out:
1765         preempt_enable_notrace();
1766         unpause_graph_tracing();
1767
1768         return len;
1769 }
1770
1771 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1772 {
1773         return trace_array_vprintk(&global_trace, ip, fmt, args);
1774 }
1775 EXPORT_SYMBOL_GPL(trace_vprintk);
1776
1777 static void trace_iterator_increment(struct trace_iterator *iter)
1778 {
1779         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
1780
1781         iter->idx++;
1782         if (buf_iter)
1783                 ring_buffer_read(buf_iter, NULL);
1784 }
1785
1786 static struct trace_entry *
1787 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1788                 unsigned long *lost_events)
1789 {
1790         struct ring_buffer_event *event;
1791         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
1792
1793         if (buf_iter)
1794                 event = ring_buffer_iter_peek(buf_iter, ts);
1795         else
1796                 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1797                                          lost_events);
1798
1799         if (event) {
1800                 iter->ent_size = ring_buffer_event_length(event);
1801                 return ring_buffer_event_data(event);
1802         }
1803         iter->ent_size = 0;
1804         return NULL;
1805 }
1806
1807 static struct trace_entry *
1808 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1809                   unsigned long *missing_events, u64 *ent_ts)
1810 {
1811         struct ring_buffer *buffer = iter->tr->buffer;
1812         struct trace_entry *ent, *next = NULL;
1813         unsigned long lost_events = 0, next_lost = 0;
1814         int cpu_file = iter->cpu_file;
1815         u64 next_ts = 0, ts;
1816         int next_cpu = -1;
1817         int next_size = 0;
1818         int cpu;
1819
1820         /*
1821          * If we are in a per_cpu trace file, don't bother by iterating over
1822          * all cpu and peek directly.
1823          */
1824         if (cpu_file > TRACE_PIPE_ALL_CPU) {
1825                 if (ring_buffer_empty_cpu(buffer, cpu_file))
1826                         return NULL;
1827                 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
1828                 if (ent_cpu)
1829                         *ent_cpu = cpu_file;
1830
1831                 return ent;
1832         }
1833
1834         for_each_tracing_cpu(cpu) {
1835
1836                 if (ring_buffer_empty_cpu(buffer, cpu))
1837                         continue;
1838
1839                 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
1840
1841                 /*
1842                  * Pick the entry with the smallest timestamp:
1843                  */
1844                 if (ent && (!next || ts < next_ts)) {
1845                         next = ent;
1846                         next_cpu = cpu;
1847                         next_ts = ts;
1848                         next_lost = lost_events;
1849                         next_size = iter->ent_size;
1850                 }
1851         }
1852
1853         iter->ent_size = next_size;
1854
1855         if (ent_cpu)
1856                 *ent_cpu = next_cpu;
1857
1858         if (ent_ts)
1859                 *ent_ts = next_ts;
1860
1861         if (missing_events)
1862                 *missing_events = next_lost;
1863
1864         return next;
1865 }
1866
1867 /* Find the next real entry, without updating the iterator itself */
1868 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1869                                           int *ent_cpu, u64 *ent_ts)
1870 {
1871         return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
1872 }
1873
1874 /* Find the next real entry, and increment the iterator to the next entry */
1875 void *trace_find_next_entry_inc(struct trace_iterator *iter)
1876 {
1877         iter->ent = __find_next_entry(iter, &iter->cpu,
1878                                       &iter->lost_events, &iter->ts);
1879
1880         if (iter->ent)
1881                 trace_iterator_increment(iter);
1882
1883         return iter->ent ? iter : NULL;
1884 }
1885
1886 static void trace_consume(struct trace_iterator *iter)
1887 {
1888         ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1889                             &iter->lost_events);
1890 }
1891
1892 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
1893 {
1894         struct trace_iterator *iter = m->private;
1895         int i = (int)*pos;
1896         void *ent;
1897
1898         WARN_ON_ONCE(iter->leftover);
1899
1900         (*pos)++;
1901
1902         /* can't go backwards */
1903         if (iter->idx > i)
1904                 return NULL;
1905
1906         if (iter->idx < 0)
1907                 ent = trace_find_next_entry_inc(iter);
1908         else
1909                 ent = iter;
1910
1911         while (ent && iter->idx < i)
1912                 ent = trace_find_next_entry_inc(iter);
1913
1914         iter->pos = *pos;
1915
1916         return ent;
1917 }
1918
1919 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
1920 {
1921         struct trace_array *tr = iter->tr;
1922         struct ring_buffer_event *event;
1923         struct ring_buffer_iter *buf_iter;
1924         unsigned long entries = 0;
1925         u64 ts;
1926
1927         tr->data[cpu]->skipped_entries = 0;
1928
1929         buf_iter = trace_buffer_iter(iter, cpu);
1930         if (!buf_iter)
1931                 return;
1932
1933         ring_buffer_iter_reset(buf_iter);
1934
1935         /*
1936          * We could have the case with the max latency tracers
1937          * that a reset never took place on a cpu. This is evident
1938          * by the timestamp being before the start of the buffer.
1939          */
1940         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1941                 if (ts >= iter->tr->time_start)
1942                         break;
1943                 entries++;
1944                 ring_buffer_read(buf_iter, NULL);
1945         }
1946
1947         tr->data[cpu]->skipped_entries = entries;
1948 }
1949
1950 /*
1951  * The current tracer is copied to avoid a global locking
1952  * all around.
1953  */
1954 static void *s_start(struct seq_file *m, loff_t *pos)
1955 {
1956         struct trace_iterator *iter = m->private;
1957         int cpu_file = iter->cpu_file;
1958         void *p = NULL;
1959         loff_t l = 0;
1960         int cpu;
1961
1962         /*
1963          * copy the tracer to avoid using a global lock all around.
1964          * iter->trace is a copy of current_trace, the pointer to the
1965          * name may be used instead of a strcmp(), as iter->trace->name
1966          * will point to the same string as current_trace->name.
1967          */
1968         mutex_lock(&trace_types_lock);
1969         if (unlikely(current_trace && iter->trace->name != current_trace->name))
1970                 *iter->trace = *current_trace;
1971         mutex_unlock(&trace_types_lock);
1972
1973         if (iter->snapshot && iter->trace->use_max_tr)
1974                 return ERR_PTR(-EBUSY);
1975
1976         if (!iter->snapshot)
1977                 atomic_inc(&trace_record_cmdline_disabled);
1978
1979         if (*pos != iter->pos) {
1980                 iter->ent = NULL;
1981                 iter->cpu = 0;
1982                 iter->idx = -1;
1983
1984                 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1985                         for_each_tracing_cpu(cpu)
1986                                 tracing_iter_reset(iter, cpu);
1987                 } else
1988                         tracing_iter_reset(iter, cpu_file);
1989
1990                 iter->leftover = 0;
1991                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1992                         ;
1993
1994         } else {
1995                 /*
1996                  * If we overflowed the seq_file before, then we want
1997                  * to just reuse the trace_seq buffer again.
1998                  */
1999                 if (iter->leftover)
2000                         p = iter;
2001                 else {
2002                         l = *pos - 1;
2003                         p = s_next(m, p, &l);
2004                 }
2005         }
2006
2007         trace_event_read_lock();
2008         trace_access_lock(cpu_file);
2009         return p;
2010 }
2011
2012 static void s_stop(struct seq_file *m, void *p)
2013 {
2014         struct trace_iterator *iter = m->private;
2015
2016         if (iter->snapshot && iter->trace->use_max_tr)
2017                 return;
2018
2019         if (!iter->snapshot)
2020                 atomic_dec(&trace_record_cmdline_disabled);
2021         trace_access_unlock(iter->cpu_file);
2022         trace_event_read_unlock();
2023 }
2024
2025 static void
2026 get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries)
2027 {
2028         unsigned long count;
2029         int cpu;
2030
2031         *total = 0;
2032         *entries = 0;
2033
2034         for_each_tracing_cpu(cpu) {
2035                 count = ring_buffer_entries_cpu(tr->buffer, cpu);
2036                 /*
2037                  * If this buffer has skipped entries, then we hold all
2038                  * entries for the trace and we need to ignore the
2039                  * ones before the time stamp.
2040                  */
2041                 if (tr->data[cpu]->skipped_entries) {
2042                         count -= tr->data[cpu]->skipped_entries;
2043                         /* total is the same as the entries */
2044                         *total += count;
2045                 } else
2046                         *total += count +
2047                                 ring_buffer_overrun_cpu(tr->buffer, cpu);
2048                 *entries += count;
2049         }
2050 }
2051
2052 static void print_lat_help_header(struct seq_file *m)
2053 {
2054         seq_puts(m, "#                  _------=> CPU#            \n");
2055         seq_puts(m, "#                 / _-----=> irqs-off        \n");
2056         seq_puts(m, "#                | / _----=> need-resched    \n");
2057         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
2058         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
2059         seq_puts(m, "#                |||| /     delay             \n");
2060         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
2061         seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
2062 }
2063
2064 static void print_event_info(struct trace_array *tr, struct seq_file *m)
2065 {
2066         unsigned long total;
2067         unsigned long entries;
2068
2069         get_total_entries(tr, &total, &entries);
2070         seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
2071                    entries, total, num_online_cpus());
2072         seq_puts(m, "#\n");
2073 }
2074
2075 static void print_func_help_header(struct trace_array *tr, struct seq_file *m)
2076 {
2077         print_event_info(tr, m);
2078         seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2079         seq_puts(m, "#              | |       |          |         |\n");
2080 }
2081
2082 static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
2083 {
2084         print_event_info(tr, m);
2085         seq_puts(m, "#                              _-----=> irqs-off\n");
2086         seq_puts(m, "#                             / _----=> need-resched\n");
2087         seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
2088         seq_puts(m, "#                            || / _--=> preempt-depth\n");
2089         seq_puts(m, "#                            ||| /     delay\n");
2090         seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
2091         seq_puts(m, "#              | |       |   ||||       |         |\n");
2092 }
2093
2094 void
2095 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2096 {
2097         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2098         struct trace_array *tr = iter->tr;
2099         struct trace_array_cpu *data = tr->data[tr->cpu];
2100         struct tracer *type = current_trace;
2101         unsigned long entries;
2102         unsigned long total;
2103         const char *name = "preemption";
2104
2105         name = type->name;
2106
2107         get_total_entries(tr, &total, &entries);
2108
2109         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2110                    name, UTS_RELEASE);
2111         seq_puts(m, "# -----------------------------------"
2112                  "---------------------------------\n");
2113         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2114                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2115                    nsecs_to_usecs(data->saved_latency),
2116                    entries,
2117                    total,
2118                    tr->cpu,
2119 #if defined(CONFIG_PREEMPT_NONE)
2120                    "server",
2121 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2122                    "desktop",
2123 #elif defined(CONFIG_PREEMPT)
2124                    "preempt",
2125 #else
2126                    "unknown",
2127 #endif
2128                    /* These are reserved for later use */
2129                    0, 0, 0, 0);
2130 #ifdef CONFIG_SMP
2131         seq_printf(m, " #P:%d)\n", num_online_cpus());
2132 #else
2133         seq_puts(m, ")\n");
2134 #endif
2135         seq_puts(m, "#    -----------------\n");
2136         seq_printf(m, "#    | task: %.16s-%d "
2137                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2138                    data->comm, data->pid,
2139                    from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2140                    data->policy, data->rt_priority);
2141         seq_puts(m, "#    -----------------\n");
2142
2143         if (data->critical_start) {
2144                 seq_puts(m, "#  => started at: ");
2145                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2146                 trace_print_seq(m, &iter->seq);
2147                 seq_puts(m, "\n#  => ended at:   ");
2148                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2149                 trace_print_seq(m, &iter->seq);
2150                 seq_puts(m, "\n#\n");
2151         }
2152
2153         seq_puts(m, "#\n");
2154 }
2155
2156 static void test_cpu_buff_start(struct trace_iterator *iter)
2157 {
2158         struct trace_seq *s = &iter->seq;
2159
2160         if (!(trace_flags & TRACE_ITER_ANNOTATE))
2161                 return;
2162
2163         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2164                 return;
2165
2166         if (cpumask_test_cpu(iter->cpu, iter->started))
2167                 return;
2168
2169         if (iter->tr->data[iter->cpu]->skipped_entries)
2170                 return;
2171
2172         cpumask_set_cpu(iter->cpu, iter->started);
2173
2174         /* Don't print started cpu buffer for the first entry of the trace */
2175         if (iter->idx > 1)
2176                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2177                                 iter->cpu);
2178 }
2179
2180 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2181 {
2182         struct trace_seq *s = &iter->seq;
2183         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2184         struct trace_entry *entry;
2185         struct trace_event *event;
2186
2187         entry = iter->ent;
2188
2189         test_cpu_buff_start(iter);
2190
2191         event = ftrace_find_event(entry->type);
2192
2193         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2194                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2195                         if (!trace_print_lat_context(iter))
2196                                 goto partial;
2197                 } else {
2198                         if (!trace_print_context(iter))
2199                                 goto partial;
2200                 }
2201         }
2202
2203         if (event)
2204                 return event->funcs->trace(iter, sym_flags, event);
2205
2206         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2207                 goto partial;
2208
2209         return TRACE_TYPE_HANDLED;
2210 partial:
2211         return TRACE_TYPE_PARTIAL_LINE;
2212 }
2213
2214 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2215 {
2216         struct trace_seq *s = &iter->seq;
2217         struct trace_entry *entry;
2218         struct trace_event *event;
2219
2220         entry = iter->ent;
2221
2222         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2223                 if (!trace_seq_printf(s, "%d %d %llu ",
2224                                       entry->pid, iter->cpu, iter->ts))
2225                         goto partial;
2226         }
2227
2228         event = ftrace_find_event(entry->type);
2229         if (event)
2230                 return event->funcs->raw(iter, 0, event);
2231
2232         if (!trace_seq_printf(s, "%d ?\n", entry->type))
2233                 goto partial;
2234
2235         return TRACE_TYPE_HANDLED;
2236 partial:
2237         return TRACE_TYPE_PARTIAL_LINE;
2238 }
2239
2240 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2241 {
2242         struct trace_seq *s = &iter->seq;
2243         unsigned char newline = '\n';
2244         struct trace_entry *entry;
2245         struct trace_event *event;
2246
2247         entry = iter->ent;
2248
2249         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2250                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2251                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2252                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2253         }
2254
2255         event = ftrace_find_event(entry->type);
2256         if (event) {
2257                 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2258                 if (ret != TRACE_TYPE_HANDLED)
2259                         return ret;
2260         }
2261
2262         SEQ_PUT_FIELD_RET(s, newline);
2263
2264         return TRACE_TYPE_HANDLED;
2265 }
2266
2267 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2268 {
2269         struct trace_seq *s = &iter->seq;
2270         struct trace_entry *entry;
2271         struct trace_event *event;
2272
2273         entry = iter->ent;
2274
2275         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2276                 SEQ_PUT_FIELD_RET(s, entry->pid);
2277                 SEQ_PUT_FIELD_RET(s, iter->cpu);
2278                 SEQ_PUT_FIELD_RET(s, iter->ts);
2279         }
2280
2281         event = ftrace_find_event(entry->type);
2282         return event ? event->funcs->binary(iter, 0, event) :
2283                 TRACE_TYPE_HANDLED;
2284 }
2285
2286 int trace_empty(struct trace_iterator *iter)
2287 {
2288         struct ring_buffer_iter *buf_iter;
2289         int cpu;
2290
2291         /* If we are looking at one CPU buffer, only check that one */
2292         if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2293                 cpu = iter->cpu_file;
2294                 buf_iter = trace_buffer_iter(iter, cpu);
2295                 if (buf_iter) {
2296                         if (!ring_buffer_iter_empty(buf_iter))
2297                                 return 0;
2298                 } else {
2299                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2300                                 return 0;
2301                 }
2302                 return 1;
2303         }
2304
2305         for_each_tracing_cpu(cpu) {
2306                 buf_iter = trace_buffer_iter(iter, cpu);
2307                 if (buf_iter) {
2308                         if (!ring_buffer_iter_empty(buf_iter))
2309                                 return 0;
2310                 } else {
2311                         if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2312                                 return 0;
2313                 }
2314         }
2315
2316         return 1;
2317 }
2318
2319 /*  Called with trace_event_read_lock() held. */
2320 enum print_line_t print_trace_line(struct trace_iterator *iter)
2321 {
2322         enum print_line_t ret;
2323
2324         if (iter->lost_events &&
2325             !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2326                                  iter->cpu, iter->lost_events))
2327                 return TRACE_TYPE_PARTIAL_LINE;
2328
2329         if (iter->trace && iter->trace->print_line) {
2330                 ret = iter->trace->print_line(iter);
2331                 if (ret != TRACE_TYPE_UNHANDLED)
2332                         return ret;
2333         }
2334
2335         if (iter->ent->type == TRACE_BPRINT &&
2336                         trace_flags & TRACE_ITER_PRINTK &&
2337                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2338                 return trace_print_bprintk_msg_only(iter);
2339
2340         if (iter->ent->type == TRACE_PRINT &&
2341                         trace_flags & TRACE_ITER_PRINTK &&
2342                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2343                 return trace_print_printk_msg_only(iter);
2344
2345         if (trace_flags & TRACE_ITER_BIN)
2346                 return print_bin_fmt(iter);
2347
2348         if (trace_flags & TRACE_ITER_HEX)
2349                 return print_hex_fmt(iter);
2350
2351         if (trace_flags & TRACE_ITER_RAW)
2352                 return print_raw_fmt(iter);
2353
2354         return print_trace_fmt(iter);
2355 }
2356
2357 void trace_latency_header(struct seq_file *m)
2358 {
2359         struct trace_iterator *iter = m->private;
2360
2361         /* print nothing if the buffers are empty */
2362         if (trace_empty(iter))
2363                 return;
2364
2365         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2366                 print_trace_header(m, iter);
2367
2368         if (!(trace_flags & TRACE_ITER_VERBOSE))
2369                 print_lat_help_header(m);
2370 }
2371
2372 void trace_default_header(struct seq_file *m)
2373 {
2374         struct trace_iterator *iter = m->private;
2375
2376         if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2377                 return;
2378
2379         if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2380                 /* print nothing if the buffers are empty */
2381                 if (trace_empty(iter))
2382                         return;
2383                 print_trace_header(m, iter);
2384                 if (!(trace_flags & TRACE_ITER_VERBOSE))
2385                         print_lat_help_header(m);
2386         } else {
2387                 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2388                         if (trace_flags & TRACE_ITER_IRQ_INFO)
2389                                 print_func_help_header_irq(iter->tr, m);
2390                         else
2391                                 print_func_help_header(iter->tr, m);
2392                 }
2393         }
2394 }
2395
2396 static void test_ftrace_alive(struct seq_file *m)
2397 {
2398         if (!ftrace_is_dead())
2399                 return;
2400         seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2401         seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
2402 }
2403
2404 #ifdef CONFIG_TRACER_MAX_TRACE
2405 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2406 {
2407         if (iter->trace->allocated_snapshot)
2408                 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2409         else
2410                 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2411
2412         seq_printf(m, "# Snapshot commands:\n");
2413         seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2414         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2415         seq_printf(m, "#                      Takes a snapshot of the main buffer.\n");
2416         seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2417         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2418         seq_printf(m, "#                       is not a '0' or '1')\n");
2419 }
2420 #else
2421 /* Should never be called */
2422 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2423 #endif
2424
2425 static int s_show(struct seq_file *m, void *v)
2426 {
2427         struct trace_iterator *iter = v;
2428         int ret;
2429
2430         if (iter->ent == NULL) {
2431                 if (iter->tr) {
2432                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
2433                         seq_puts(m, "#\n");
2434                         test_ftrace_alive(m);
2435                 }
2436                 if (iter->snapshot && trace_empty(iter))
2437                         print_snapshot_help(m, iter);
2438                 else if (iter->trace && iter->trace->print_header)
2439                         iter->trace->print_header(m);
2440                 else
2441                         trace_default_header(m);
2442
2443         } else if (iter->leftover) {
2444                 /*
2445                  * If we filled the seq_file buffer earlier, we
2446                  * want to just show it now.
2447                  */
2448                 ret = trace_print_seq(m, &iter->seq);
2449
2450                 /* ret should this time be zero, but you never know */
2451                 iter->leftover = ret;
2452
2453         } else {
2454                 print_trace_line(iter);
2455                 ret = trace_print_seq(m, &iter->seq);
2456                 /*
2457                  * If we overflow the seq_file buffer, then it will
2458                  * ask us for this data again at start up.
2459                  * Use that instead.
2460                  *  ret is 0 if seq_file write succeeded.
2461                  *        -1 otherwise.
2462                  */
2463                 iter->leftover = ret;
2464         }
2465
2466         return 0;
2467 }
2468
2469 static const struct seq_operations tracer_seq_ops = {
2470         .start          = s_start,
2471         .next           = s_next,
2472         .stop           = s_stop,
2473         .show           = s_show,
2474 };
2475
2476 static struct trace_iterator *
2477 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2478 {
2479         long cpu_file = (long) inode->i_private;
2480         struct trace_iterator *iter;
2481         int cpu;
2482
2483         if (tracing_disabled)
2484                 return ERR_PTR(-ENODEV);
2485
2486         iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2487         if (!iter)
2488                 return ERR_PTR(-ENOMEM);
2489
2490         iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2491                                     GFP_KERNEL);
2492         if (!iter->buffer_iter)
2493                 goto release;
2494
2495         /*
2496          * We make a copy of the current tracer to avoid concurrent
2497          * changes on it while we are reading.
2498          */
2499         mutex_lock(&trace_types_lock);
2500         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2501         if (!iter->trace)
2502                 goto fail;
2503
2504         *iter->trace = *current_trace;
2505
2506         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2507                 goto fail;
2508
2509         if (current_trace->print_max || snapshot)
2510                 iter->tr = &max_tr;
2511         else
2512                 iter->tr = &global_trace;
2513         iter->snapshot = snapshot;
2514         iter->pos = -1;
2515         mutex_init(&iter->mutex);
2516         iter->cpu_file = cpu_file;
2517
2518         /* Notify the tracer early; before we stop tracing. */
2519         if (iter->trace && iter->trace->open)
2520                 iter->trace->open(iter);
2521
2522         /* Annotate start of buffers if we had overruns */
2523         if (ring_buffer_overruns(iter->tr->buffer))
2524                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2525
2526         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2527         if (trace_clocks[trace_clock_id].in_ns)
2528                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2529
2530         /* stop the trace while dumping if we are not opening "snapshot" */
2531         if (!iter->snapshot)
2532                 tracing_stop();
2533
2534         if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2535                 for_each_tracing_cpu(cpu) {
2536                         iter->buffer_iter[cpu] =
2537                                 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2538                 }
2539                 ring_buffer_read_prepare_sync();
2540                 for_each_tracing_cpu(cpu) {
2541                         ring_buffer_read_start(iter->buffer_iter[cpu]);
2542                         tracing_iter_reset(iter, cpu);
2543                 }
2544         } else {
2545                 cpu = iter->cpu_file;
2546                 iter->buffer_iter[cpu] =
2547                         ring_buffer_read_prepare(iter->tr->buffer, cpu);
2548                 ring_buffer_read_prepare_sync();
2549                 ring_buffer_read_start(iter->buffer_iter[cpu]);
2550                 tracing_iter_reset(iter, cpu);
2551         }
2552
2553         mutex_unlock(&trace_types_lock);
2554
2555         return iter;
2556
2557  fail:
2558         mutex_unlock(&trace_types_lock);
2559         kfree(iter->trace);
2560         kfree(iter->buffer_iter);
2561 release:
2562         seq_release_private(inode, file);
2563         return ERR_PTR(-ENOMEM);
2564 }
2565
2566 int tracing_open_generic(struct inode *inode, struct file *filp)
2567 {
2568         if (tracing_disabled)
2569                 return -ENODEV;
2570
2571         filp->private_data = inode->i_private;
2572         return 0;
2573 }
2574
2575 static int tracing_release(struct inode *inode, struct file *file)
2576 {
2577         struct seq_file *m = file->private_data;
2578         struct trace_iterator *iter;
2579         int cpu;
2580
2581         if (!(file->f_mode & FMODE_READ))
2582                 return 0;
2583
2584         iter = m->private;
2585
2586         mutex_lock(&trace_types_lock);
2587         for_each_tracing_cpu(cpu) {
2588                 if (iter->buffer_iter[cpu])
2589                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2590         }
2591
2592         if (iter->trace && iter->trace->close)
2593                 iter->trace->close(iter);
2594
2595         if (!iter->snapshot)
2596                 /* reenable tracing if it was previously enabled */
2597                 tracing_start();
2598         mutex_unlock(&trace_types_lock);
2599
2600         mutex_destroy(&iter->mutex);
2601         free_cpumask_var(iter->started);
2602         kfree(iter->trace);
2603         kfree(iter->buffer_iter);
2604         seq_release_private(inode, file);
2605         return 0;
2606 }
2607
2608 static int tracing_open(struct inode *inode, struct file *file)
2609 {
2610         struct trace_iterator *iter;
2611         int ret = 0;
2612
2613         /* If this file was open for write, then erase contents */
2614         if ((file->f_mode & FMODE_WRITE) &&
2615             (file->f_flags & O_TRUNC)) {
2616                 long cpu = (long) inode->i_private;
2617
2618                 if (cpu == TRACE_PIPE_ALL_CPU)
2619                         tracing_reset_online_cpus(&global_trace);
2620                 else
2621                         tracing_reset(&global_trace, cpu);
2622         }
2623
2624         if (file->f_mode & FMODE_READ) {
2625                 iter = __tracing_open(inode, file, false);
2626                 if (IS_ERR(iter))
2627                         ret = PTR_ERR(iter);
2628                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2629                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
2630         }
2631         return ret;
2632 }
2633
2634 static void *
2635 t_next(struct seq_file *m, void *v, loff_t *pos)
2636 {
2637         struct tracer *t = v;
2638
2639         (*pos)++;
2640
2641         if (t)
2642                 t = t->next;
2643
2644         return t;
2645 }
2646
2647 static void *t_start(struct seq_file *m, loff_t *pos)
2648 {
2649         struct tracer *t;
2650         loff_t l = 0;
2651
2652         mutex_lock(&trace_types_lock);
2653         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2654                 ;
2655
2656         return t;
2657 }
2658
2659 static void t_stop(struct seq_file *m, void *p)
2660 {
2661         mutex_unlock(&trace_types_lock);
2662 }
2663
2664 static int t_show(struct seq_file *m, void *v)
2665 {
2666         struct tracer *t = v;
2667
2668         if (!t)
2669                 return 0;
2670
2671         seq_printf(m, "%s", t->name);
2672         if (t->next)
2673                 seq_putc(m, ' ');
2674         else
2675                 seq_putc(m, '\n');
2676
2677         return 0;
2678 }
2679
2680 static const struct seq_operations show_traces_seq_ops = {
2681         .start          = t_start,
2682         .next           = t_next,
2683         .stop           = t_stop,
2684         .show           = t_show,
2685 };
2686
2687 static int show_traces_open(struct inode *inode, struct file *file)
2688 {
2689         if (tracing_disabled)
2690                 return -ENODEV;
2691
2692         return seq_open(file, &show_traces_seq_ops);
2693 }
2694
2695 static ssize_t
2696 tracing_write_stub(struct file *filp, const char __user *ubuf,
2697                    size_t count, loff_t *ppos)
2698 {
2699         return count;
2700 }
2701
2702 static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2703 {
2704         if (file->f_mode & FMODE_READ)
2705                 return seq_lseek(file, offset, origin);
2706         else
2707                 return 0;
2708 }
2709
2710 static const struct file_operations tracing_fops = {
2711         .open           = tracing_open,
2712         .read           = seq_read,
2713         .write          = tracing_write_stub,
2714         .llseek         = tracing_seek,
2715         .release        = tracing_release,
2716 };
2717
2718 static const struct file_operations show_traces_fops = {
2719         .open           = show_traces_open,
2720         .read           = seq_read,
2721         .release        = seq_release,
2722         .llseek         = seq_lseek,
2723 };
2724
2725 /*
2726  * Only trace on a CPU if the bitmask is set:
2727  */
2728 static cpumask_var_t tracing_cpumask;
2729
2730 /*
2731  * The tracer itself will not take this lock, but still we want
2732  * to provide a consistent cpumask to user-space:
2733  */
2734 static DEFINE_MUTEX(tracing_cpumask_update_lock);
2735
2736 /*
2737  * Temporary storage for the character representation of the
2738  * CPU bitmask (and one more byte for the newline):
2739  */
2740 static char mask_str[NR_CPUS + 1];
2741
2742 static ssize_t
2743 tracing_cpumask_read(struct file *filp, char __user *ubuf,
2744                      size_t count, loff_t *ppos)
2745 {
2746         int len;
2747
2748         mutex_lock(&tracing_cpumask_update_lock);
2749
2750         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
2751         if (count - len < 2) {
2752                 count = -EINVAL;
2753                 goto out_err;
2754         }
2755         len += sprintf(mask_str + len, "\n");
2756         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2757
2758 out_err:
2759         mutex_unlock(&tracing_cpumask_update_lock);
2760
2761         return count;
2762 }
2763
2764 static ssize_t
2765 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2766                       size_t count, loff_t *ppos)
2767 {
2768         int err, cpu;
2769         cpumask_var_t tracing_cpumask_new;
2770
2771         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2772                 return -ENOMEM;
2773
2774         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
2775         if (err)
2776                 goto err_unlock;
2777
2778         mutex_lock(&tracing_cpumask_update_lock);
2779
2780         local_irq_disable();
2781         arch_spin_lock(&ftrace_max_lock);
2782         for_each_tracing_cpu(cpu) {
2783                 /*
2784                  * Increase/decrease the disabled counter if we are
2785                  * about to flip a bit in the cpumask:
2786                  */
2787                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2788                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2789                         atomic_inc(&global_trace.data[cpu]->disabled);
2790                         ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
2791                 }
2792                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2793                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
2794                         atomic_dec(&global_trace.data[cpu]->disabled);
2795                         ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
2796                 }
2797         }
2798         arch_spin_unlock(&ftrace_max_lock);
2799         local_irq_enable();
2800
2801         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
2802
2803         mutex_unlock(&tracing_cpumask_update_lock);
2804         free_cpumask_var(tracing_cpumask_new);
2805
2806         return count;
2807
2808 err_unlock:
2809         free_cpumask_var(tracing_cpumask_new);
2810
2811         return err;
2812 }
2813
2814 static const struct file_operations tracing_cpumask_fops = {
2815         .open           = tracing_open_generic,
2816         .read           = tracing_cpumask_read,
2817         .write          = tracing_cpumask_write,
2818         .llseek         = generic_file_llseek,
2819 };
2820
2821 static int tracing_trace_options_show(struct seq_file *m, void *v)
2822 {
2823         struct tracer_opt *trace_opts;
2824         u32 tracer_flags;
2825         int i;
2826
2827         mutex_lock(&trace_types_lock);
2828         tracer_flags = current_trace->flags->val;
2829         trace_opts = current_trace->flags->opts;
2830
2831         for (i = 0; trace_options[i]; i++) {
2832                 if (trace_flags & (1 << i))
2833                         seq_printf(m, "%s\n", trace_options[i]);
2834                 else
2835                         seq_printf(m, "no%s\n", trace_options[i]);
2836         }
2837
2838         for (i = 0; trace_opts[i].name; i++) {
2839                 if (tracer_flags & trace_opts[i].bit)
2840                         seq_printf(m, "%s\n", trace_opts[i].name);
2841                 else
2842                         seq_printf(m, "no%s\n", trace_opts[i].name);
2843         }
2844         mutex_unlock(&trace_types_lock);
2845
2846         return 0;
2847 }
2848
2849 static int __set_tracer_option(struct tracer *trace,
2850                                struct tracer_flags *tracer_flags,
2851                                struct tracer_opt *opts, int neg)
2852 {
2853         int ret;
2854
2855         ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
2856         if (ret)
2857                 return ret;
2858
2859         if (neg)
2860                 tracer_flags->val &= ~opts->bit;
2861         else
2862                 tracer_flags->val |= opts->bit;
2863         return 0;
2864 }
2865
2866 /* Try to assign a tracer specific option */
2867 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2868 {
2869         struct tracer_flags *tracer_flags = trace->flags;
2870         struct tracer_opt *opts = NULL;
2871         int i;
2872
2873         for (i = 0; tracer_flags->opts[i].name; i++) {
2874                 opts = &tracer_flags->opts[i];
2875
2876                 if (strcmp(cmp, opts->name) == 0)
2877                         return __set_tracer_option(trace, trace->flags,
2878                                                    opts, neg);
2879         }
2880
2881         return -EINVAL;
2882 }
2883
2884 /* Some tracers require overwrite to stay enabled */
2885 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
2886 {
2887         if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
2888                 return -1;
2889
2890         return 0;
2891 }
2892
2893 int set_tracer_flag(unsigned int mask, int enabled)
2894 {
2895         /* do nothing if flag is already set */
2896         if (!!(trace_flags & mask) == !!enabled)
2897                 return 0;
2898
2899         /* Give the tracer a chance to approve the change */
2900         if (current_trace->flag_changed)
2901                 if (current_trace->flag_changed(current_trace, mask, !!enabled))
2902                         return -EINVAL;
2903
2904         if (enabled)
2905                 trace_flags |= mask;
2906         else
2907                 trace_flags &= ~mask;
2908
2909         if (mask == TRACE_ITER_RECORD_CMD)
2910                 trace_event_enable_cmd_record(enabled);
2911
2912         if (mask == TRACE_ITER_OVERWRITE) {
2913                 ring_buffer_change_overwrite(global_trace.buffer, enabled);
2914 #ifdef CONFIG_TRACER_MAX_TRACE
2915                 ring_buffer_change_overwrite(max_tr.buffer, enabled);
2916 #endif
2917         }
2918
2919         if (mask == TRACE_ITER_PRINTK)
2920                 trace_printk_start_stop_comm(enabled);
2921
2922         return 0;
2923 }
2924
2925 static int trace_set_options(char *option)
2926 {
2927         char *cmp;
2928         int neg = 0;
2929         int ret = -ENODEV;
2930         int i;
2931
2932         cmp = strstrip(option);
2933
2934         if (strncmp(cmp, "no", 2) == 0) {
2935                 neg = 1;
2936                 cmp += 2;
2937         }
2938
2939         mutex_lock(&trace_types_lock);
2940
2941         for (i = 0; trace_options[i]; i++) {
2942                 if (strcmp(cmp, trace_options[i]) == 0) {
2943                         ret = set_tracer_flag(1 << i, !neg);
2944                         break;
2945                 }
2946         }
2947
2948         /* If no option could be set, test the specific tracer options */
2949         if (!trace_options[i])
2950                 ret = set_tracer_option(current_trace, cmp, neg);
2951
2952         mutex_unlock(&trace_types_lock);
2953
2954         return ret;
2955 }
2956
2957 static ssize_t
2958 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2959                         size_t cnt, loff_t *ppos)
2960 {
2961         char buf[64];
2962         int ret;
2963
2964         if (cnt >= sizeof(buf))
2965                 return -EINVAL;
2966
2967         if (copy_from_user(&buf, ubuf, cnt))
2968                 return -EFAULT;
2969
2970         buf[cnt] = 0;
2971
2972         ret = trace_set_options(buf);
2973         if (ret < 0)
2974                 return ret;
2975
2976         *ppos += cnt;
2977
2978         return cnt;
2979 }
2980
2981 static int tracing_trace_options_open(struct inode *inode, struct file *file)
2982 {
2983         if (tracing_disabled)
2984                 return -ENODEV;
2985         return single_open(file, tracing_trace_options_show, NULL);
2986 }
2987
2988 static const struct file_operations tracing_iter_fops = {
2989         .open           = tracing_trace_options_open,
2990         .read           = seq_read,
2991         .llseek         = seq_lseek,
2992         .release        = single_release,
2993         .write          = tracing_trace_options_write,
2994 };
2995
2996 static const char readme_msg[] =
2997         "tracing mini-HOWTO:\n\n"
2998         "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2999         "# cat /sys/kernel/debug/tracing/available_tracers\n"
3000         "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
3001         "# cat /sys/kernel/debug/tracing/current_tracer\n"
3002         "nop\n"
3003         "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
3004         "# cat /sys/kernel/debug/tracing/current_tracer\n"
3005         "wakeup\n"
3006         "# cat /sys/kernel/debug/tracing/trace_options\n"
3007         "noprint-parent nosym-offset nosym-addr noverbose\n"
3008         "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
3009         "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
3010         "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
3011         "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
3012 ;
3013
3014 static ssize_t
3015 tracing_readme_read(struct file *filp, char __user *ubuf,
3016                        size_t cnt, loff_t *ppos)
3017 {
3018         return simple_read_from_buffer(ubuf, cnt, ppos,
3019                                         readme_msg, strlen(readme_msg));
3020 }
3021
3022 static const struct file_operations tracing_readme_fops = {
3023         .open           = tracing_open_generic,
3024         .read           = tracing_readme_read,
3025         .llseek         = generic_file_llseek,
3026 };
3027
3028 static ssize_t
3029 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3030                                 size_t cnt, loff_t *ppos)
3031 {
3032         char *buf_comm;
3033         char *file_buf;
3034         char *buf;
3035         int len = 0;
3036         int pid;
3037         int i;
3038
3039         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3040         if (!file_buf)
3041                 return -ENOMEM;
3042
3043         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3044         if (!buf_comm) {
3045                 kfree(file_buf);
3046                 return -ENOMEM;
3047         }
3048
3049         buf = file_buf;
3050
3051         for (i = 0; i < SAVED_CMDLINES; i++) {
3052                 int r;
3053
3054                 pid = map_cmdline_to_pid[i];
3055                 if (pid == -1 || pid == NO_CMDLINE_MAP)
3056                         continue;
3057
3058                 trace_find_cmdline(pid, buf_comm);
3059                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3060                 buf += r;
3061                 len += r;
3062         }
3063
3064         len = simple_read_from_buffer(ubuf, cnt, ppos,
3065                                       file_buf, len);
3066
3067         kfree(file_buf);
3068         kfree(buf_comm);
3069
3070         return len;
3071 }
3072
3073 static const struct file_operations tracing_saved_cmdlines_fops = {
3074     .open       = tracing_open_generic,
3075     .read       = tracing_saved_cmdlines_read,
3076     .llseek     = generic_file_llseek,
3077 };
3078
3079 static ssize_t
3080 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3081                        size_t cnt, loff_t *ppos)
3082 {
3083         char buf[MAX_TRACER_SIZE+2];
3084         int r;
3085
3086         mutex_lock(&trace_types_lock);
3087         r = sprintf(buf, "%s\n", current_trace->name);
3088         mutex_unlock(&trace_types_lock);
3089
3090         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3091 }
3092
3093 int tracer_init(struct tracer *t, struct trace_array *tr)
3094 {
3095         tracing_reset_online_cpus(tr);
3096         return t->init(tr);
3097 }
3098
3099 static void set_buffer_entries(struct trace_array *tr, unsigned long val)
3100 {
3101         int cpu;
3102         for_each_tracing_cpu(cpu)
3103                 tr->data[cpu]->entries = val;
3104 }
3105
3106 /* resize @tr's buffer to the size of @size_tr's entries */
3107 static int resize_buffer_duplicate_size(struct trace_array *tr,
3108                                         struct trace_array *size_tr, int cpu_id)
3109 {
3110         int cpu, ret = 0;
3111
3112         if (cpu_id == RING_BUFFER_ALL_CPUS) {
3113                 for_each_tracing_cpu(cpu) {
3114                         ret = ring_buffer_resize(tr->buffer,
3115                                         size_tr->data[cpu]->entries, cpu);
3116                         if (ret < 0)
3117                                 break;
3118                         tr->data[cpu]->entries = size_tr->data[cpu]->entries;
3119                 }
3120         } else {
3121                 ret = ring_buffer_resize(tr->buffer,
3122                                         size_tr->data[cpu_id]->entries, cpu_id);
3123                 if (ret == 0)
3124                         tr->data[cpu_id]->entries =
3125                                 size_tr->data[cpu_id]->entries;
3126         }
3127
3128         return ret;
3129 }
3130
3131 static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
3132 {
3133         int ret;
3134
3135         /*
3136          * If kernel or user changes the size of the ring buffer
3137          * we use the size that was given, and we can forget about
3138          * expanding it later.
3139          */
3140         ring_buffer_expanded = 1;
3141
3142         /* May be called before buffers are initialized */
3143         if (!global_trace.buffer)
3144                 return 0;
3145
3146         ret = ring_buffer_resize(global_trace.buffer, size, cpu);
3147         if (ret < 0)
3148                 return ret;
3149
3150         if (!current_trace->use_max_tr)
3151                 goto out;
3152
3153         ret = ring_buffer_resize(max_tr.buffer, size, cpu);
3154         if (ret < 0) {
3155                 int r = resize_buffer_duplicate_size(&global_trace,
3156                                                      &global_trace, cpu);
3157                 if (r < 0) {
3158                         /*
3159                          * AARGH! We are left with different
3160                          * size max buffer!!!!
3161                          * The max buffer is our "snapshot" buffer.
3162                          * When a tracer needs a snapshot (one of the
3163                          * latency tracers), it swaps the max buffer
3164                          * with the saved snap shot. We succeeded to
3165                          * update the size of the main buffer, but failed to
3166                          * update the size of the max buffer. But when we tried
3167                          * to reset the main buffer to the original size, we
3168                          * failed there too. This is very unlikely to
3169                          * happen, but if it does, warn and kill all
3170                          * tracing.
3171                          */
3172                         WARN_ON(1);
3173                         tracing_disabled = 1;
3174                 }
3175                 return ret;
3176         }
3177
3178         if (cpu == RING_BUFFER_ALL_CPUS)
3179                 set_buffer_entries(&max_tr, size);
3180         else
3181                 max_tr.data[cpu]->entries = size;
3182
3183  out:
3184         if (cpu == RING_BUFFER_ALL_CPUS)
3185                 set_buffer_entries(&global_trace, size);
3186         else
3187                 global_trace.data[cpu]->entries = size;
3188
3189         return ret;
3190 }
3191
3192 static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
3193 {
3194         int ret = size;
3195
3196         mutex_lock(&trace_types_lock);
3197
3198         if (cpu_id != RING_BUFFER_ALL_CPUS) {
3199                 /* make sure, this cpu is enabled in the mask */
3200                 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3201                         ret = -EINVAL;
3202                         goto out;
3203                 }
3204         }
3205
3206         ret = __tracing_resize_ring_buffer(size, cpu_id);
3207         if (ret < 0)
3208                 ret = -ENOMEM;
3209
3210 out:
3211         mutex_unlock(&trace_types_lock);
3212
3213         return ret;
3214 }
3215
3216
3217 /**
3218  * tracing_update_buffers - used by tracing facility to expand ring buffers
3219  *
3220  * To save on memory when the tracing is never used on a system with it
3221  * configured in. The ring buffers are set to a minimum size. But once
3222  * a user starts to use the tracing facility, then they need to grow
3223  * to their default size.
3224  *
3225  * This function is to be called when a tracer is about to be used.
3226  */
3227 int tracing_update_buffers(void)
3228 {
3229         int ret = 0;
3230
3231         mutex_lock(&trace_types_lock);
3232         if (!ring_buffer_expanded)
3233                 ret = __tracing_resize_ring_buffer(trace_buf_size,
3234                                                 RING_BUFFER_ALL_CPUS);
3235         mutex_unlock(&trace_types_lock);
3236
3237         return ret;
3238 }
3239
3240 struct trace_option_dentry;
3241
3242 static struct trace_option_dentry *
3243 create_trace_option_files(struct tracer *tracer);
3244
3245 static void
3246 destroy_trace_option_files(struct trace_option_dentry *topts);
3247
3248 static int tracing_set_tracer(const char *buf)
3249 {
3250         static struct trace_option_dentry *topts;
3251         struct trace_array *tr = &global_trace;
3252         struct tracer *t;
3253         bool had_max_tr;
3254         int ret = 0;
3255
3256         mutex_lock(&trace_types_lock);
3257
3258         if (!ring_buffer_expanded) {
3259                 ret = __tracing_resize_ring_buffer(trace_buf_size,
3260                                                 RING_BUFFER_ALL_CPUS);
3261                 if (ret < 0)
3262                         goto out;
3263                 ret = 0;
3264         }
3265
3266         for (t = trace_types; t; t = t->next) {
3267                 if (strcmp(t->name, buf) == 0)
3268                         break;
3269         }
3270         if (!t) {
3271                 ret = -EINVAL;
3272                 goto out;
3273         }
3274         if (t == current_trace)
3275                 goto out;
3276
3277         trace_branch_disable();
3278
3279         current_trace->enabled = false;
3280
3281         if (current_trace->reset)
3282                 current_trace->reset(tr);
3283
3284         had_max_tr = current_trace->allocated_snapshot;
3285         current_trace = &nop_trace;
3286
3287         if (had_max_tr && !t->use_max_tr) {
3288                 /*
3289                  * We need to make sure that the update_max_tr sees that
3290                  * current_trace changed to nop_trace to keep it from
3291                  * swapping the buffers after we resize it.
3292                  * The update_max_tr is called from interrupts disabled
3293                  * so a synchronized_sched() is sufficient.
3294                  */
3295                 synchronize_sched();
3296                 /*
3297                  * We don't free the ring buffer. instead, resize it because
3298                  * The max_tr ring buffer has some state (e.g. ring->clock) and
3299                  * we want preserve it.
3300                  */
3301                 ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3302                 set_buffer_entries(&max_tr, 1);
3303                 tracing_reset_online_cpus(&max_tr);
3304                 current_trace->allocated_snapshot = false;
3305         }
3306         destroy_trace_option_files(topts);
3307
3308         topts = create_trace_option_files(t);
3309         if (t->use_max_tr && !had_max_tr) {
3310                 /* we need to make per cpu buffer sizes equivalent */
3311                 ret = resize_buffer_duplicate_size(&max_tr, &global_trace,
3312                                                    RING_BUFFER_ALL_CPUS);
3313                 if (ret < 0)
3314                         goto out;
3315                 t->allocated_snapshot = true;
3316         }
3317
3318         if (t->init) {
3319                 ret = tracer_init(t, tr);
3320                 if (ret)
3321                         goto out;
3322         }
3323
3324         current_trace = t;
3325         current_trace->enabled = true;
3326         trace_branch_enable(tr);
3327  out:
3328         mutex_unlock(&trace_types_lock);
3329
3330         return ret;
3331 }
3332
3333 static ssize_t
3334 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3335                         size_t cnt, loff_t *ppos)
3336 {
3337         char buf[MAX_TRACER_SIZE+1];
3338         int i;
3339         size_t ret;
3340         int err;
3341
3342         ret = cnt;
3343
3344         if (cnt > MAX_TRACER_SIZE)
3345                 cnt = MAX_TRACER_SIZE;
3346
3347         if (copy_from_user(&buf, ubuf, cnt))
3348                 return -EFAULT;
3349
3350         buf[cnt] = 0;
3351
3352         /* strip ending whitespace. */
3353         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3354                 buf[i] = 0;
3355
3356         err = tracing_set_tracer(buf);
3357         if (err)
3358                 return err;
3359
3360         *ppos += ret;
3361
3362         return ret;
3363 }
3364
3365 static ssize_t
3366 tracing_max_lat_read(struct file *filp, char __user *ubuf,
3367                      size_t cnt, loff_t *ppos)
3368 {
3369         unsigned long *ptr = filp->private_data;
3370         char buf[64];
3371         int r;
3372
3373         r = snprintf(buf, sizeof(buf), "%ld\n",
3374                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3375         if (r > sizeof(buf))
3376                 r = sizeof(buf);
3377         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3378 }
3379
3380 static ssize_t
3381 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3382                       size_t cnt, loff_t *ppos)
3383 {
3384         unsigned long *ptr = filp->private_data;
3385         unsigned long val;
3386         int ret;
3387
3388         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3389         if (ret)
3390                 return ret;
3391
3392         *ptr = val * 1000;
3393
3394         return cnt;
3395 }
3396
3397 static int tracing_open_pipe(struct inode *inode, struct file *filp)
3398 {
3399         long cpu_file = (long) inode->i_private;
3400         struct trace_iterator *iter;
3401         int ret = 0;
3402
3403         if (tracing_disabled)
3404                 return -ENODEV;
3405
3406         mutex_lock(&trace_types_lock);
3407
3408         /* create a buffer to store the information to pass to userspace */
3409         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3410         if (!iter) {
3411                 ret = -ENOMEM;
3412                 goto out;
3413         }
3414
3415         /*
3416          * We make a copy of the current tracer to avoid concurrent
3417          * changes on it while we are reading.
3418          */
3419         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3420         if (!iter->trace) {
3421                 ret = -ENOMEM;
3422                 goto fail;
3423         }
3424         *iter->trace = *current_trace;
3425
3426         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3427                 ret = -ENOMEM;
3428                 goto fail;
3429         }
3430
3431         /* trace pipe does not show start of buffer */
3432         cpumask_setall(iter->started);
3433
3434         if (trace_flags & TRACE_ITER_LATENCY_FMT)
3435                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3436
3437         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3438         if (trace_clocks[trace_clock_id].in_ns)
3439                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3440
3441         iter->cpu_file = cpu_file;
3442         iter->tr = &global_trace;
3443         mutex_init(&iter->mutex);
3444         filp->private_data = iter;
3445
3446         if (iter->trace->pipe_open)
3447                 iter->trace->pipe_open(iter);
3448
3449         nonseekable_open(inode, filp);
3450 out:
3451         mutex_unlock(&trace_types_lock);
3452         return ret;
3453
3454 fail:
3455         kfree(iter->trace);
3456         kfree(iter);
3457         mutex_unlock(&trace_types_lock);
3458         return ret;
3459 }
3460
3461 static int tracing_release_pipe(struct inode *inode, struct file *file)
3462 {
3463         struct trace_iterator *iter = file->private_data;
3464
3465         mutex_lock(&trace_types_lock);
3466
3467         if (iter->trace->pipe_close)
3468                 iter->trace->pipe_close(iter);
3469
3470         mutex_unlock(&trace_types_lock);
3471
3472         free_cpumask_var(iter->started);
3473         mutex_destroy(&iter->mutex);
3474         kfree(iter->trace);
3475         kfree(iter);
3476
3477         return 0;
3478 }
3479
3480 static unsigned int
3481 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3482 {
3483         struct trace_iterator *iter = filp->private_data;
3484
3485         if (trace_flags & TRACE_ITER_BLOCK) {
3486                 /*
3487                  * Always select as readable when in blocking mode
3488                  */
3489                 return POLLIN | POLLRDNORM;
3490         } else {
3491                 if (!trace_empty(iter))
3492                         return POLLIN | POLLRDNORM;
3493                 poll_wait(filp, &trace_wait, poll_table);
3494                 if (!trace_empty(iter))
3495                         return POLLIN | POLLRDNORM;
3496
3497                 return 0;
3498         }
3499 }
3500
3501 /*
3502  * This is a make-shift waitqueue.
3503  * A tracer might use this callback on some rare cases:
3504  *
3505  *  1) the current tracer might hold the runqueue lock when it wakes up
3506  *     a reader, hence a deadlock (sched, function, and function graph tracers)
3507  *  2) the function tracers, trace all functions, we don't want
3508  *     the overhead of calling wake_up and friends
3509  *     (and tracing them too)
3510  *
3511  *     Anyway, this is really very primitive wakeup.
3512  */
3513 void poll_wait_pipe(struct trace_iterator *iter)
3514 {
3515         set_current_state(TASK_INTERRUPTIBLE);
3516         /* sleep for 100 msecs, and try again. */
3517         schedule_timeout(HZ / 10);
3518 }
3519
3520 /* Must be called with trace_types_lock mutex held. */
3521 static int tracing_wait_pipe(struct file *filp)
3522 {
3523         struct trace_iterator *iter = filp->private_data;
3524
3525         while (trace_empty(iter)) {
3526
3527                 if ((filp->f_flags & O_NONBLOCK)) {
3528                         return -EAGAIN;
3529                 }
3530
3531                 mutex_unlock(&iter->mutex);
3532
3533                 iter->trace->wait_pipe(iter);
3534
3535                 mutex_lock(&iter->mutex);
3536
3537                 if (signal_pending(current))
3538                         return -EINTR;
3539
3540                 /*
3541                  * We block until we read something and tracing is disabled.
3542                  * We still block if tracing is disabled, but we have never
3543                  * read anything. This allows a user to cat this file, and
3544                  * then enable tracing. But after we have read something,
3545                  * we give an EOF when tracing is again disabled.
3546                  *
3547                  * iter->pos will be 0 if we haven't read anything.
3548                  */
3549                 if (!tracing_is_enabled() && iter->pos)
3550                         break;
3551         }
3552
3553         return 1;
3554 }
3555
3556 /*
3557  * Consumer reader.
3558  */
3559 static ssize_t
3560 tracing_read_pipe(struct file *filp, char __user *ubuf,
3561                   size_t cnt, loff_t *ppos)
3562 {
3563         struct trace_iterator *iter = filp->private_data;
3564         ssize_t sret;
3565
3566         /* return any leftover data */
3567         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3568         if (sret != -EBUSY)
3569                 return sret;
3570
3571         trace_seq_init(&iter->seq);
3572
3573         /* copy the tracer to avoid using a global lock all around */
3574         mutex_lock(&trace_types_lock);
3575         if (unlikely(iter->trace->name != current_trace->name))
3576                 *iter->trace = *current_trace;
3577         mutex_unlock(&trace_types_lock);
3578
3579         /*
3580          * Avoid more than one consumer on a single file descriptor
3581          * This is just a matter of traces coherency, the ring buffer itself
3582          * is protected.
3583          */
3584         mutex_lock(&iter->mutex);
3585         if (iter->trace->read) {
3586                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3587                 if (sret)
3588                         goto out;
3589         }
3590
3591 waitagain:
3592         sret = tracing_wait_pipe(filp);
3593         if (sret <= 0)
3594                 goto out;
3595
3596         /* stop when tracing is finished */
3597         if (trace_empty(iter)) {
3598                 sret = 0;
3599                 goto out;
3600         }
3601
3602         if (cnt >= PAGE_SIZE)
3603                 cnt = PAGE_SIZE - 1;
3604
3605         /* reset all but tr, trace, and overruns */
3606         memset(&iter->seq, 0,
3607                sizeof(struct trace_iterator) -
3608                offsetof(struct trace_iterator, seq));
3609         iter->pos = -1;
3610
3611         trace_event_read_lock();
3612         trace_access_lock(iter->cpu_file);
3613         while (trace_find_next_entry_inc(iter) != NULL) {
3614                 enum print_line_t ret;
3615                 int len = iter->seq.len;
3616
3617                 ret = print_trace_line(iter);
3618                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3619                         /* don't print partial lines */
3620                         iter->seq.len = len;
3621                         break;
3622                 }
3623                 if (ret != TRACE_TYPE_NO_CONSUME)
3624                         trace_consume(iter);
3625
3626                 if (iter->seq.len >= cnt)
3627                         break;
3628
3629                 /*
3630                  * Setting the full flag means we reached the trace_seq buffer
3631                  * size and we should leave by partial output condition above.
3632                  * One of the trace_seq_* functions is not used properly.
3633                  */
3634                 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3635                           iter->ent->type);
3636         }
3637         trace_access_unlock(iter->cpu_file);
3638         trace_event_read_unlock();
3639
3640         /* Now copy what we have to the user */
3641         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3642         if (iter->seq.readpos >= iter->seq.len)
3643                 trace_seq_init(&iter->seq);
3644
3645         /*
3646          * If there was nothing to send to user, in spite of consuming trace
3647          * entries, go back to wait for more entries.
3648          */
3649         if (sret == -EBUSY)
3650                 goto waitagain;
3651
3652 out:
3653         mutex_unlock(&iter->mutex);
3654
3655         return sret;
3656 }
3657
3658 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3659                                      struct pipe_buffer *buf)
3660 {
3661         __free_page(buf->page);
3662 }
3663
3664 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3665                                      unsigned int idx)
3666 {
3667         __free_page(spd->pages[idx]);
3668 }
3669
3670 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3671         .can_merge              = 0,
3672         .map                    = generic_pipe_buf_map,
3673         .unmap                  = generic_pipe_buf_unmap,
3674         .confirm                = generic_pipe_buf_confirm,
3675         .release                = tracing_pipe_buf_release,
3676         .steal                  = generic_pipe_buf_steal,
3677         .get                    = generic_pipe_buf_get,
3678 };
3679
3680 static size_t
3681 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3682 {
3683         size_t count;
3684         int ret;
3685
3686         /* Seq buffer is page-sized, exactly what we need. */
3687         for (;;) {
3688                 count = iter->seq.len;
3689                 ret = print_trace_line(iter);
3690                 count = iter->seq.len - count;
3691                 if (rem < count) {
3692                         rem = 0;
3693                         iter->seq.len -= count;
3694                         break;
3695                 }
3696                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3697                         iter->seq.len -= count;
3698                         break;
3699                 }
3700
3701                 if (ret != TRACE_TYPE_NO_CONSUME)
3702                         trace_consume(iter);
3703                 rem -= count;
3704                 if (!trace_find_next_entry_inc(iter))   {
3705                         rem = 0;
3706                         iter->ent = NULL;
3707                         break;
3708                 }
3709         }
3710
3711         return rem;
3712 }
3713
3714 static ssize_t tracing_splice_read_pipe(struct file *filp,
3715                                         loff_t *ppos,
3716                                         struct pipe_inode_info *pipe,
3717                                         size_t len,
3718                                         unsigned int flags)
3719 {
3720         struct page *pages_def[PIPE_DEF_BUFFERS];
3721         struct partial_page partial_def[PIPE_DEF_BUFFERS];
3722         struct trace_iterator *iter = filp->private_data;
3723         struct splice_pipe_desc spd = {
3724                 .pages          = pages_def,
3725                 .partial        = partial_def,
3726                 .nr_pages       = 0, /* This gets updated below. */
3727                 .nr_pages_max   = PIPE_DEF_BUFFERS,
3728                 .flags          = flags,
3729                 .ops            = &tracing_pipe_buf_ops,
3730                 .spd_release    = tracing_spd_release_pipe,
3731         };
3732         ssize_t ret;
3733         size_t rem;
3734         unsigned int i;
3735
3736         if (splice_grow_spd(pipe, &spd))
3737                 return -ENOMEM;
3738
3739         /* copy the tracer to avoid using a global lock all around */
3740         mutex_lock(&trace_types_lock);
3741         if (unlikely(iter->trace->name != current_trace->name))
3742                 *iter->trace = *current_trace;
3743         mutex_unlock(&trace_types_lock);
3744
3745         mutex_lock(&iter->mutex);
3746
3747         if (iter->trace->splice_read) {
3748                 ret = iter->trace->splice_read(iter, filp,
3749                                                ppos, pipe, len, flags);
3750                 if (ret)
3751                         goto out_err;
3752         }
3753
3754         ret = tracing_wait_pipe(filp);
3755         if (ret <= 0)
3756                 goto out_err;
3757
3758         if (!iter->ent && !trace_find_next_entry_inc(iter)) {
3759                 ret = -EFAULT;
3760                 goto out_err;
3761         }
3762
3763         trace_event_read_lock();
3764         trace_access_lock(iter->cpu_file);
3765
3766         /* Fill as many pages as possible. */
3767         for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3768                 spd.pages[i] = alloc_page(GFP_KERNEL);
3769                 if (!spd.pages[i])
3770                         break;
3771
3772                 rem = tracing_fill_pipe_page(rem, iter);
3773
3774                 /* Copy the data into the page, so we can start over. */
3775                 ret = trace_seq_to_buffer(&iter->seq,
3776                                           page_address(spd.pages[i]),
3777                                           iter->seq.len);
3778                 if (ret < 0) {
3779                         __free_page(spd.pages[i]);
3780                         break;
3781                 }
3782                 spd.partial[i].offset = 0;
3783                 spd.partial[i].len = iter->seq.len;
3784
3785                 trace_seq_init(&iter->seq);
3786         }
3787
3788         trace_access_unlock(iter->cpu_file);
3789         trace_event_read_unlock();
3790         mutex_unlock(&iter->mutex);
3791
3792         spd.nr_pages = i;
3793
3794         ret = splice_to_pipe(pipe, &spd);
3795 out:
3796         splice_shrink_spd(&spd);
3797         return ret;
3798
3799 out_err:
3800         mutex_unlock(&iter->mutex);
3801         goto out;
3802 }
3803
3804 struct ftrace_entries_info {
3805         struct trace_array      *tr;
3806         int                     cpu;
3807 };
3808
3809 static int tracing_entries_open(struct inode *inode, struct file *filp)
3810 {
3811         struct ftrace_entries_info *info;
3812
3813         if (tracing_disabled)
3814                 return -ENODEV;
3815
3816         info = kzalloc(sizeof(*info), GFP_KERNEL);
3817         if (!info)
3818                 return -ENOMEM;
3819
3820         info->tr = &global_trace;
3821         info->cpu = (unsigned long)inode->i_private;
3822
3823         filp->private_data = info;
3824
3825         return 0;
3826 }
3827
3828 static ssize_t
3829 tracing_entries_read(struct file *filp, char __user *ubuf,
3830                      size_t cnt, loff_t *ppos)
3831 {
3832         struct ftrace_entries_info *info = filp->private_data;
3833         struct trace_array *tr = info->tr;
3834         char buf[64];
3835         int r = 0;
3836         ssize_t ret;
3837
3838         mutex_lock(&trace_types_lock);
3839
3840         if (info->cpu == RING_BUFFER_ALL_CPUS) {
3841                 int cpu, buf_size_same;
3842                 unsigned long size;
3843
3844                 size = 0;
3845                 buf_size_same = 1;
3846                 /* check if all cpu sizes are same */
3847                 for_each_tracing_cpu(cpu) {
3848                         /* fill in the size from first enabled cpu */
3849                         if (size == 0)
3850                                 size = tr->data[cpu]->entries;
3851                         if (size != tr->data[cpu]->entries) {
3852                                 buf_size_same = 0;
3853                                 break;
3854                         }
3855                 }
3856
3857                 if (buf_size_same) {
3858                         if (!ring_buffer_expanded)
3859                                 r = sprintf(buf, "%lu (expanded: %lu)\n",
3860                                             size >> 10,
3861                                             trace_buf_size >> 10);
3862                         else
3863                                 r = sprintf(buf, "%lu\n", size >> 10);
3864                 } else
3865                         r = sprintf(buf, "X\n");
3866         } else
3867                 r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3868
3869         mutex_unlock(&trace_types_lock);
3870
3871         ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3872         return ret;
3873 }
3874
3875 static ssize_t
3876 tracing_entries_write(struct file *filp, const char __user *ubuf,
3877                       size_t cnt, loff_t *ppos)
3878 {
3879         struct ftrace_entries_info *info = filp->private_data;
3880         unsigned long val;
3881         int ret;
3882
3883         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3884         if (ret)
3885                 return ret;
3886
3887         /* must have at least 1 entry */
3888         if (!val)
3889                 return -EINVAL;
3890
3891         /* value is in KB */
3892         val <<= 10;
3893
3894         ret = tracing_resize_ring_buffer(val, info->cpu);
3895         if (ret < 0)
3896                 return ret;
3897
3898         *ppos += cnt;
3899
3900         return cnt;
3901 }
3902
3903 static int
3904 tracing_entries_release(struct inode *inode, struct file *filp)
3905 {
3906         struct ftrace_entries_info *info = filp->private_data;
3907
3908         kfree(info);
3909
3910         return 0;
3911 }
3912
3913 static ssize_t
3914 tracing_total_entries_read(struct file *filp, char __user *ubuf,
3915                                 size_t cnt, loff_t *ppos)
3916 {
3917         struct trace_array *tr = filp->private_data;
3918         char buf[64];
3919         int r, cpu;
3920         unsigned long size = 0, expanded_size = 0;
3921
3922         mutex_lock(&trace_types_lock);
3923         for_each_tracing_cpu(cpu) {
3924                 size += tr->data[cpu]->entries >> 10;
3925                 if (!ring_buffer_expanded)
3926                         expanded_size += trace_buf_size >> 10;
3927         }
3928         if (ring_buffer_expanded)
3929                 r = sprintf(buf, "%lu\n", size);
3930         else
3931                 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3932         mutex_unlock(&trace_types_lock);
3933
3934         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3935 }
3936
3937 static ssize_t
3938 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
3939                           size_t cnt, loff_t *ppos)
3940 {
3941         /*
3942          * There is no need to read what the user has written, this function
3943          * is just to make sure that there is no error when "echo" is used
3944          */
3945
3946         *ppos += cnt;
3947
3948         return cnt;
3949 }
3950
3951 static int
3952 tracing_free_buffer_release(struct inode *inode, struct file *filp)
3953 {
3954         /* disable tracing ? */
3955         if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3956                 tracing_off();
3957         /* resize the ring buffer to 0 */
3958         tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
3959
3960         return 0;
3961 }
3962
3963 static ssize_t
3964 tracing_mark_write(struct file *filp, const char __user *ubuf,
3965                                         size_t cnt, loff_t *fpos)
3966 {
3967         unsigned long addr = (unsigned long)ubuf;
3968         struct ring_buffer_event *event;
3969         struct ring_buffer *buffer;
3970         struct print_entry *entry;
3971         unsigned long irq_flags;
3972         struct page *pages[2];
3973         void *map_page[2];
3974         int nr_pages = 1;
3975         ssize_t written;
3976         int offset;
3977         int size;
3978         int len;
3979         int ret;
3980         int i;
3981
3982         if (tracing_disabled)
3983                 return -EINVAL;
3984
3985         if (!(trace_flags & TRACE_ITER_MARKERS))
3986                 return -EINVAL;
3987
3988         if (cnt > TRACE_BUF_SIZE)
3989                 cnt = TRACE_BUF_SIZE;
3990
3991         /*
3992          * Userspace is injecting traces into the kernel trace buffer.
3993          * We want to be as non intrusive as possible.
3994          * To do so, we do not want to allocate any special buffers
3995          * or take any locks, but instead write the userspace data
3996          * straight into the ring buffer.
3997          *
3998          * First we need to pin the userspace buffer into memory,
3999          * which, most likely it is, because it just referenced it.
4000          * But there's no guarantee that it is. By using get_user_pages_fast()
4001          * and kmap_atomic/kunmap_atomic() we can get access to the
4002          * pages directly. We then write the data directly into the
4003          * ring buffer.
4004          */
4005         BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4006
4007         /* check if we cross pages */
4008         if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4009                 nr_pages = 2;
4010
4011         offset = addr & (PAGE_SIZE - 1);
4012         addr &= PAGE_MASK;
4013
4014         ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4015         if (ret < nr_pages) {
4016                 while (--ret >= 0)
4017                         put_page(pages[ret]);
4018                 written = -EFAULT;
4019                 goto out;
4020         }
4021
4022         for (i = 0; i < nr_pages; i++)
4023                 map_page[i] = kmap_atomic(pages[i]);
4024
4025         local_save_flags(irq_flags);
4026         size = sizeof(*entry) + cnt + 2; /* possible \n added */
4027         buffer = global_trace.buffer;
4028         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4029                                           irq_flags, preempt_count());
4030         if (!event) {
4031                 /* Ring buffer disabled, return as if not open for write */
4032                 written = -EBADF;
4033                 goto out_unlock;
4034         }
4035
4036         entry = ring_buffer_event_data(event);
4037         entry->ip = _THIS_IP_;
4038
4039         if (nr_pages == 2) {
4040                 len = PAGE_SIZE - offset;
4041                 memcpy(&entry->buf, map_page[0] + offset, len);
4042                 memcpy(&entry->buf[len], map_page[1], cnt - len);
4043         } else
4044                 memcpy(&entry->buf, map_page[0] + offset, cnt);
4045
4046         if (entry->buf[cnt - 1] != '\n') {
4047                 entry->buf[cnt] = '\n';
4048                 entry->buf[cnt + 1] = '\0';
4049         } else
4050                 entry->buf[cnt] = '\0';
4051
4052         __buffer_unlock_commit(buffer, event);
4053
4054         written = cnt;
4055
4056         *fpos += written;
4057
4058  out_unlock:
4059         for (i = 0; i < nr_pages; i++){
4060                 kunmap_atomic(map_page[i]);
4061                 put_page(pages[i]);
4062         }
4063  out:
4064         return written;
4065 }
4066
4067 static int tracing_clock_show(struct seq_file *m, void *v)
4068 {
4069         int i;
4070
4071         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4072                 seq_printf(m,
4073                         "%s%s%s%s", i ? " " : "",
4074                         i == trace_clock_id ? "[" : "", trace_clocks[i].name,
4075                         i == trace_clock_id ? "]" : "");
4076         seq_putc(m, '\n');
4077
4078         return 0;
4079 }
4080
4081 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4082                                    size_t cnt, loff_t *fpos)
4083 {
4084         char buf[64];
4085         const char *clockstr;
4086         int i;
4087
4088         if (cnt >= sizeof(buf))
4089                 return -EINVAL;
4090
4091         if (copy_from_user(&buf, ubuf, cnt))
4092                 return -EFAULT;
4093
4094         buf[cnt] = 0;
4095
4096         clockstr = strstrip(buf);
4097
4098         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4099                 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4100                         break;
4101         }
4102         if (i == ARRAY_SIZE(trace_clocks))
4103                 return -EINVAL;
4104
4105         trace_clock_id = i;
4106
4107         mutex_lock(&trace_types_lock);
4108
4109         ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
4110         if (max_tr.buffer)
4111                 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
4112
4113         /*
4114          * New clock may not be consistent with the previous clock.
4115          * Reset the buffer so that it doesn't have incomparable timestamps.
4116          */
4117         tracing_reset_online_cpus(&global_trace);
4118         tracing_reset_online_cpus(&max_tr);
4119
4120         mutex_unlock(&trace_types_lock);
4121
4122         *fpos += cnt;
4123
4124         return cnt;
4125 }
4126
4127 static int tracing_clock_open(struct inode *inode, struct file *file)
4128 {
4129         if (tracing_disabled)
4130                 return -ENODEV;
4131         return single_open(file, tracing_clock_show, NULL);
4132 }
4133
4134 #ifdef CONFIG_TRACER_SNAPSHOT
4135 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4136 {
4137         struct trace_iterator *iter;
4138         int ret = 0;
4139
4140         if (file->f_mode & FMODE_READ) {
4141                 iter = __tracing_open(inode, file, true);
4142                 if (IS_ERR(iter))
4143                         ret = PTR_ERR(iter);
4144         }
4145         return ret;
4146 }
4147
4148 static ssize_t
4149 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4150                        loff_t *ppos)
4151 {
4152         unsigned long val;
4153         int ret;
4154
4155         ret = tracing_update_buffers();
4156         if (ret < 0)
4157                 return ret;
4158
4159         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4160         if (ret)
4161                 return ret;
4162
4163         mutex_lock(&trace_types_lock);
4164
4165         if (current_trace->use_max_tr) {
4166                 ret = -EBUSY;
4167                 goto out;
4168         }
4169
4170         switch (val) {
4171         case 0:
4172                 if (current_trace->allocated_snapshot) {
4173                         /* free spare buffer */
4174                         ring_buffer_resize(max_tr.buffer, 1,
4175                                            RING_BUFFER_ALL_CPUS);
4176                         set_buffer_entries(&max_tr, 1);
4177                         tracing_reset_online_cpus(&max_tr);
4178                         current_trace->allocated_snapshot = false;
4179                 }
4180                 break;
4181         case 1:
4182                 if (!current_trace->allocated_snapshot) {
4183                         /* allocate spare buffer */
4184                         ret = resize_buffer_duplicate_size(&max_tr,
4185                                         &global_trace, RING_BUFFER_ALL_CPUS);
4186                         if (ret < 0)
4187                                 break;
4188                         current_trace->allocated_snapshot = true;
4189                 }
4190
4191                 local_irq_disable();
4192                 /* Now, we're going to swap */
4193                 update_max_tr(&global_trace, current, smp_processor_id());
4194                 local_irq_enable();
4195                 break;
4196         default:
4197                 if (current_trace->allocated_snapshot)
4198                         tracing_reset_online_cpus(&max_tr);
4199                 break;
4200         }
4201
4202         if (ret >= 0) {
4203                 *ppos += cnt;
4204                 ret = cnt;
4205         }
4206 out:
4207         mutex_unlock(&trace_types_lock);
4208         return ret;
4209 }
4210 #endif /* CONFIG_TRACER_SNAPSHOT */
4211
4212
4213 static const struct file_operations tracing_max_lat_fops = {
4214         .open           = tracing_open_generic,
4215         .read           = tracing_max_lat_read,
4216         .write          = tracing_max_lat_write,
4217         .llseek         = generic_file_llseek,
4218 };
4219
4220 static const struct file_operations set_tracer_fops = {
4221         .open           = tracing_open_generic,
4222         .read           = tracing_set_trace_read,
4223         .write          = tracing_set_trace_write,
4224         .llseek         = generic_file_llseek,
4225 };
4226
4227 static const struct file_operations tracing_pipe_fops = {
4228         .open           = tracing_open_pipe,
4229         .poll           = tracing_poll_pipe,
4230         .read           = tracing_read_pipe,
4231         .splice_read    = tracing_splice_read_pipe,
4232         .release        = tracing_release_pipe,
4233         .llseek         = no_llseek,
4234 };
4235
4236 static const struct file_operations tracing_entries_fops = {
4237         .open           = tracing_entries_open,
4238         .read           = tracing_entries_read,
4239         .write          = tracing_entries_write,
4240         .release        = tracing_entries_release,
4241         .llseek         = generic_file_llseek,
4242 };
4243
4244 static const struct file_operations tracing_total_entries_fops = {
4245         .open           = tracing_open_generic,
4246         .read           = tracing_total_entries_read,
4247         .llseek         = generic_file_llseek,
4248 };
4249
4250 static const struct file_operations tracing_free_buffer_fops = {
4251         .write          = tracing_free_buffer_write,
4252         .release        = tracing_free_buffer_release,
4253 };
4254
4255 static const struct file_operations tracing_mark_fops = {
4256         .open           = tracing_open_generic,
4257         .write          = tracing_mark_write,
4258         .llseek         = generic_file_llseek,
4259 };
4260
4261 static const struct file_operations trace_clock_fops = {
4262         .open           = tracing_clock_open,
4263         .read           = seq_read,
4264         .llseek         = seq_lseek,
4265         .release        = single_release,
4266         .write          = tracing_clock_write,
4267 };
4268
4269 #ifdef CONFIG_TRACER_SNAPSHOT
4270 static const struct file_operations snapshot_fops = {
4271         .open           = tracing_snapshot_open,
4272         .read           = seq_read,
4273         .write          = tracing_snapshot_write,
4274         .llseek         = tracing_seek,
4275         .release        = tracing_release,
4276 };
4277 #endif /* CONFIG_TRACER_SNAPSHOT */
4278
4279 struct ftrace_buffer_info {
4280         struct trace_array      *tr;
4281         void                    *spare;
4282         int                     cpu;
4283         unsigned int            read;
4284 };
4285
4286 static int tracing_buffers_open(struct inode *inode, struct file *filp)
4287 {
4288         int cpu = (int)(long)inode->i_private;
4289         struct ftrace_buffer_info *info;
4290
4291         if (tracing_disabled)
4292                 return -ENODEV;
4293
4294         info = kzalloc(sizeof(*info), GFP_KERNEL);
4295         if (!info)
4296                 return -ENOMEM;
4297
4298         info->tr        = &global_trace;
4299         info->cpu       = cpu;
4300         info->spare     = NULL;
4301         /* Force reading ring buffer for first read */
4302         info->read      = (unsigned int)-1;
4303
4304         filp->private_data = info;
4305
4306         return nonseekable_open(inode, filp);
4307 }
4308
4309 static ssize_t
4310 tracing_buffers_read(struct file *filp, char __user *ubuf,
4311                      size_t count, loff_t *ppos)
4312 {
4313         struct ftrace_buffer_info *info = filp->private_data;
4314         ssize_t ret;
4315         size_t size;
4316
4317         if (!count)
4318                 return 0;
4319
4320         if (!info->spare)
4321                 info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
4322         if (!info->spare)
4323                 return -ENOMEM;
4324
4325         /* Do we have previous read data to read? */
4326         if (info->read < PAGE_SIZE)
4327                 goto read;
4328
4329         trace_access_lock(info->cpu);
4330         ret = ring_buffer_read_page(info->tr->buffer,
4331                                     &info->spare,
4332                                     count,
4333                                     info->cpu, 0);
4334         trace_access_unlock(info->cpu);
4335         if (ret < 0)
4336                 return 0;
4337
4338         info->read = 0;
4339
4340 read:
4341         size = PAGE_SIZE - info->read;
4342         if (size > count)
4343                 size = count;
4344
4345         ret = copy_to_user(ubuf, info->spare + info->read, size);
4346         if (ret == size)
4347                 return -EFAULT;
4348         size -= ret;
4349
4350         *ppos += size;
4351         info->read += size;
4352
4353         return size;
4354 }
4355
4356 static int tracing_buffers_release(struct inode *inode, struct file *file)
4357 {
4358         struct ftrace_buffer_info *info = file->private_data;
4359
4360         if (info->spare)
4361                 ring_buffer_free_read_page(info->tr->buffer, info->spare);
4362         kfree(info);
4363
4364         return 0;
4365 }
4366
4367 struct buffer_ref {
4368         struct ring_buffer      *buffer;
4369         void                    *page;
4370         int                     ref;
4371 };
4372
4373 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4374                                     struct pipe_buffer *buf)
4375 {
4376         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4377
4378         if (--ref->ref)
4379                 return;
4380
4381         ring_buffer_free_read_page(ref->buffer, ref->page);
4382         kfree(ref);
4383         buf->private = 0;
4384 }
4385
4386 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4387                                 struct pipe_buffer *buf)
4388 {
4389         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4390
4391         ref->ref++;
4392 }
4393
4394 /* Pipe buffer operations for a buffer. */
4395 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4396         .can_merge              = 0,
4397         .map                    = generic_pipe_buf_map,
4398         .unmap                  = generic_pipe_buf_unmap,
4399         .confirm                = generic_pipe_buf_confirm,
4400         .release                = buffer_pipe_buf_release,
4401         .steal                  = generic_pipe_buf_steal,
4402         .get                    = buffer_pipe_buf_get,
4403 };
4404
4405 /*
4406  * Callback from splice_to_pipe(), if we need to release some pages
4407  * at the end of the spd in case we error'ed out in filling the pipe.
4408  */
4409 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4410 {
4411         struct buffer_ref *ref =
4412                 (struct buffer_ref *)spd->partial[i].private;
4413
4414         if (--ref->ref)
4415                 return;
4416
4417         ring_buffer_free_read_page(ref->buffer, ref->page);
4418         kfree(ref);
4419         spd->partial[i].private = 0;
4420 }
4421
4422 static ssize_t
4423 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4424                             struct pipe_inode_info *pipe, size_t len,
4425                             unsigned int flags)
4426 {
4427         struct ftrace_buffer_info *info = file->private_data;
4428         struct partial_page partial_def[PIPE_DEF_BUFFERS];
4429         struct page *pages_def[PIPE_DEF_BUFFERS];
4430         struct splice_pipe_desc spd = {
4431                 .pages          = pages_def,
4432                 .partial        = partial_def,
4433                 .nr_pages_max   = PIPE_DEF_BUFFERS,
4434                 .flags          = flags,
4435                 .ops            = &buffer_pipe_buf_ops,
4436                 .spd_release    = buffer_spd_release,
4437         };
4438         struct buffer_ref *ref;
4439         int entries, size, i;
4440         size_t ret;
4441
4442         if (splice_grow_spd(pipe, &spd))
4443                 return -ENOMEM;
4444
4445         if (*ppos & (PAGE_SIZE - 1)) {
4446                 ret = -EINVAL;
4447                 goto out;
4448         }
4449
4450         if (len & (PAGE_SIZE - 1)) {
4451                 if (len < PAGE_SIZE) {
4452                         ret = -EINVAL;
4453                         goto out;
4454                 }
4455                 len &= PAGE_MASK;
4456         }
4457
4458         trace_access_lock(info->cpu);
4459         entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4460
4461         for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4462                 struct page *page;
4463                 int r;
4464
4465                 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4466                 if (!ref)
4467                         break;
4468
4469                 ref->ref = 1;
4470                 ref->buffer = info->tr->buffer;
4471                 ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
4472                 if (!ref->page) {
4473                         kfree(ref);
4474                         break;
4475                 }
4476
4477                 r = ring_buffer_read_page(ref->buffer, &ref->page,
4478                                           len, info->cpu, 1);
4479                 if (r < 0) {
4480                         ring_buffer_free_read_page(ref->buffer, ref->page);
4481                         kfree(ref);
4482                         break;
4483                 }
4484
4485                 /*
4486                  * zero out any left over data, this is going to
4487                  * user land.
4488                  */
4489                 size = ring_buffer_page_len(ref->page);
4490                 if (size < PAGE_SIZE)
4491                         memset(ref->page + size, 0, PAGE_SIZE - size);
4492
4493                 page = virt_to_page(ref->page);
4494
4495                 spd.pages[i] = page;
4496                 spd.partial[i].len = PAGE_SIZE;
4497                 spd.partial[i].offset = 0;
4498                 spd.partial[i].private = (unsigned long)ref;
4499                 spd.nr_pages++;
4500                 *ppos += PAGE_SIZE;
4501
4502                 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4503         }
4504
4505         trace_access_unlock(info->cpu);
4506         spd.nr_pages = i;
4507
4508         /* did we read anything? */
4509         if (!spd.nr_pages) {
4510                 if (flags & SPLICE_F_NONBLOCK)
4511                         ret = -EAGAIN;
4512                 else
4513                         ret = 0;
4514                 /* TODO: block */
4515                 goto out;
4516         }
4517
4518         ret = splice_to_pipe(pipe, &spd);
4519         splice_shrink_spd(&spd);
4520 out:
4521         return ret;
4522 }
4523
4524 static const struct file_operations tracing_buffers_fops = {
4525         .open           = tracing_buffers_open,
4526         .read           = tracing_buffers_read,
4527         .release        = tracing_buffers_release,
4528         .splice_read    = tracing_buffers_splice_read,
4529         .llseek         = no_llseek,
4530 };
4531
4532 static ssize_t
4533 tracing_stats_read(struct file *filp, char __user *ubuf,
4534                    size_t count, loff_t *ppos)
4535 {
4536         unsigned long cpu = (unsigned long)filp->private_data;
4537         struct trace_array *tr = &global_trace;
4538         struct trace_seq *s;
4539         unsigned long cnt;
4540         unsigned long long t;
4541         unsigned long usec_rem;
4542
4543         s = kmalloc(sizeof(*s), GFP_KERNEL);
4544         if (!s)
4545                 return -ENOMEM;
4546
4547         trace_seq_init(s);
4548
4549         cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4550         trace_seq_printf(s, "entries: %ld\n", cnt);
4551
4552         cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4553         trace_seq_printf(s, "overrun: %ld\n", cnt);
4554
4555         cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4556         trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4557
4558         cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4559         trace_seq_printf(s, "bytes: %ld\n", cnt);
4560
4561         if (trace_clocks[trace_clock_id].in_ns) {
4562                 /* local or global for trace_clock */
4563                 t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4564                 usec_rem = do_div(t, USEC_PER_SEC);
4565                 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
4566                                                                 t, usec_rem);
4567
4568                 t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4569                 usec_rem = do_div(t, USEC_PER_SEC);
4570                 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4571         } else {
4572                 /* counter or tsc mode for trace_clock */
4573                 trace_seq_printf(s, "oldest event ts: %llu\n",
4574                                 ring_buffer_oldest_event_ts(tr->buffer, cpu));
4575
4576                 trace_seq_printf(s, "now ts: %llu\n",
4577                                 ring_buffer_time_stamp(tr->buffer, cpu));
4578         }
4579
4580         cnt = ring_buffer_dropped_events_cpu(tr->buffer, cpu);
4581         trace_seq_printf(s, "dropped events: %ld\n", cnt);
4582
4583         cnt = ring_buffer_read_events_cpu(tr->buffer, cpu);
4584         trace_seq_printf(s, "read events: %ld\n", cnt);
4585
4586         count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4587
4588         kfree(s);
4589
4590         return count;
4591 }
4592
4593 static const struct file_operations tracing_stats_fops = {
4594         .open           = tracing_open_generic,
4595         .read           = tracing_stats_read,
4596         .llseek         = generic_file_llseek,
4597 };
4598
4599 #ifdef CONFIG_DYNAMIC_FTRACE
4600
4601 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
4602 {
4603         return 0;
4604 }
4605
4606 static ssize_t
4607 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
4608                   size_t cnt, loff_t *ppos)
4609 {
4610         static char ftrace_dyn_info_buffer[1024];
4611         static DEFINE_MUTEX(dyn_info_mutex);
4612         unsigned long *p = filp->private_data;
4613         char *buf = ftrace_dyn_info_buffer;
4614         int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
4615         int r;
4616
4617         mutex_lock(&dyn_info_mutex);
4618         r = sprintf(buf, "%ld ", *p);
4619
4620         r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
4621         buf[r++] = '\n';
4622
4623         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4624
4625         mutex_unlock(&dyn_info_mutex);
4626
4627         return r;
4628 }
4629
4630 static const struct file_operations tracing_dyn_info_fops = {
4631         .open           = tracing_open_generic,
4632         .read           = tracing_read_dyn_info,
4633         .llseek         = generic_file_llseek,
4634 };
4635 #endif
4636
4637 static struct dentry *d_tracer;
4638
4639 struct dentry *tracing_init_dentry(void)
4640 {
4641         static int once;
4642
4643         if (d_tracer)
4644                 return d_tracer;
4645
4646         if (!debugfs_initialized())
4647                 return NULL;
4648
4649         d_tracer = debugfs_create_dir("tracing", NULL);
4650
4651         if (!d_tracer && !once) {
4652                 once = 1;
4653                 pr_warning("Could not create debugfs directory 'tracing'\n");
4654                 return NULL;
4655         }
4656
4657         return d_tracer;
4658 }
4659
4660 static struct dentry *d_percpu;
4661
4662 static struct dentry *tracing_dentry_percpu(void)
4663 {
4664         static int once;
4665         struct dentry *d_tracer;
4666
4667         if (d_percpu)
4668                 return d_percpu;
4669
4670         d_tracer = tracing_init_dentry();
4671
4672         if (!d_tracer)
4673                 return NULL;
4674
4675         d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4676
4677         if (!d_percpu && !once) {
4678                 once = 1;
4679                 pr_warning("Could not create debugfs directory 'per_cpu'\n");
4680                 return NULL;
4681         }
4682
4683         return d_percpu;
4684 }
4685
4686 static void tracing_init_debugfs_percpu(long cpu)
4687 {
4688         struct dentry *d_percpu = tracing_dentry_percpu();
4689         struct dentry *d_cpu;
4690         char cpu_dir[30]; /* 30 characters should be more than enough */
4691
4692         if (!d_percpu)
4693                 return;
4694
4695         snprintf(cpu_dir, 30, "cpu%ld", cpu);
4696         d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4697         if (!d_cpu) {
4698                 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4699                 return;
4700         }
4701
4702         /* per cpu trace_pipe */
4703         trace_create_file("trace_pipe", 0444, d_cpu,
4704                         (void *) cpu, &tracing_pipe_fops);
4705
4706         /* per cpu trace */
4707         trace_create_file("trace", 0644, d_cpu,
4708                         (void *) cpu, &tracing_fops);
4709
4710         trace_create_file("trace_pipe_raw", 0444, d_cpu,
4711                         (void *) cpu, &tracing_buffers_fops);
4712
4713         trace_create_file("stats", 0444, d_cpu,
4714                         (void *) cpu, &tracing_stats_fops);
4715
4716         trace_create_file("buffer_size_kb", 0444, d_cpu,
4717                         (void *) cpu, &tracing_entries_fops);
4718 }
4719
4720 #ifdef CONFIG_FTRACE_SELFTEST
4721 /* Let selftest have access to static functions in this file */
4722 #include "trace_selftest.c"
4723 #endif
4724
4725 struct trace_option_dentry {
4726         struct tracer_opt               *opt;
4727         struct tracer_flags             *flags;
4728         struct dentry                   *entry;
4729 };
4730
4731 static ssize_t
4732 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4733                         loff_t *ppos)
4734 {
4735         struct trace_option_dentry *topt = filp->private_data;
4736         char *buf;
4737
4738         if (topt->flags->val & topt->opt->bit)
4739                 buf = "1\n";
4740         else
4741                 buf = "0\n";
4742
4743         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4744 }
4745
4746 static ssize_t
4747 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4748                          loff_t *ppos)
4749 {
4750         struct trace_option_dentry *topt = filp->private_data;
4751         unsigned long val;
4752         int ret;
4753
4754         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4755         if (ret)
4756                 return ret;
4757
4758         if (val != 0 && val != 1)
4759                 return -EINVAL;
4760
4761         if (!!(topt->flags->val & topt->opt->bit) != val) {
4762                 mutex_lock(&trace_types_lock);
4763                 ret = __set_tracer_option(current_trace, topt->flags,
4764                                           topt->opt, !val);
4765                 mutex_unlock(&trace_types_lock);
4766                 if (ret)
4767                         return ret;
4768         }
4769
4770         *ppos += cnt;
4771
4772         return cnt;
4773 }
4774
4775
4776 static const struct file_operations trace_options_fops = {
4777         .open = tracing_open_generic,
4778         .read = trace_options_read,
4779         .write = trace_options_write,
4780         .llseek = generic_file_llseek,
4781 };
4782
4783 static ssize_t
4784 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4785                         loff_t *ppos)
4786 {
4787         long index = (long)filp->private_data;
4788         char *buf;
4789
4790         if (trace_flags & (1 << index))
4791                 buf = "1\n";
4792         else
4793                 buf = "0\n";
4794
4795         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4796 }
4797
4798 static ssize_t
4799 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4800                          loff_t *ppos)
4801 {
4802         long index = (long)filp->private_data;
4803         unsigned long val;
4804         int ret;
4805
4806         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4807         if (ret)
4808                 return ret;
4809
4810         if (val != 0 && val != 1)
4811                 return -EINVAL;
4812
4813         mutex_lock(&trace_types_lock);
4814         ret = set_tracer_flag(1 << index, val);
4815         mutex_unlock(&trace_types_lock);
4816
4817         if (ret < 0)
4818                 return ret;
4819
4820         *ppos += cnt;
4821
4822         return cnt;
4823 }
4824
4825 static const struct file_operations trace_options_core_fops = {
4826         .open = tracing_open_generic,
4827         .read = trace_options_core_read,
4828         .write = trace_options_core_write,
4829         .llseek = generic_file_llseek,
4830 };
4831
4832 struct dentry *trace_create_file(const char *name,
4833                                  umode_t mode,
4834                                  struct dentry *parent,
4835                                  void *data,
4836                                  const struct file_operations *fops)
4837 {
4838         struct dentry *ret;
4839
4840         ret = debugfs_create_file(name, mode, parent, data, fops);
4841         if (!ret)
4842                 pr_warning("Could not create debugfs '%s' entry\n", name);
4843
4844         return ret;
4845 }
4846
4847
4848 static struct dentry *trace_options_init_dentry(void)
4849 {
4850         struct dentry *d_tracer;
4851         static struct dentry *t_options;
4852
4853         if (t_options)
4854                 return t_options;
4855
4856         d_tracer = tracing_init_dentry();
4857         if (!d_tracer)
4858                 return NULL;
4859
4860         t_options = debugfs_create_dir("options", d_tracer);
4861         if (!t_options) {
4862                 pr_warning("Could not create debugfs directory 'options'\n");
4863                 return NULL;
4864         }
4865
4866         return t_options;
4867 }
4868
4869 static void
4870 create_trace_option_file(struct trace_option_dentry *topt,
4871                          struct tracer_flags *flags,
4872                          struct tracer_opt *opt)
4873 {
4874         struct dentry *t_options;
4875
4876         t_options = trace_options_init_dentry();
4877         if (!t_options)
4878                 return;
4879
4880         topt->flags = flags;
4881         topt->opt = opt;
4882
4883         topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
4884                                     &trace_options_fops);
4885
4886 }
4887
4888 static struct trace_option_dentry *
4889 create_trace_option_files(struct tracer *tracer)
4890 {
4891         struct trace_option_dentry *topts;
4892         struct tracer_flags *flags;
4893         struct tracer_opt *opts;
4894         int cnt;
4895
4896         if (!tracer)
4897                 return NULL;
4898
4899         flags = tracer->flags;
4900
4901         if (!flags || !flags->opts)
4902                 return NULL;
4903
4904         opts = flags->opts;
4905
4906         for (cnt = 0; opts[cnt].name; cnt++)
4907                 ;
4908
4909         topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
4910         if (!topts)
4911                 return NULL;
4912
4913         for (cnt = 0; opts[cnt].name; cnt++)
4914                 create_trace_option_file(&topts[cnt], flags,
4915                                          &opts[cnt]);
4916
4917         return topts;
4918 }
4919
4920 static void
4921 destroy_trace_option_files(struct trace_option_dentry *topts)
4922 {
4923         int cnt;
4924
4925         if (!topts)
4926                 return;
4927
4928         for (cnt = 0; topts[cnt].opt; cnt++) {
4929                 if (topts[cnt].entry)
4930                         debugfs_remove(topts[cnt].entry);
4931         }
4932
4933         kfree(topts);
4934 }
4935
4936 static struct dentry *
4937 create_trace_option_core_file(const char *option, long index)
4938 {
4939         struct dentry *t_options;
4940
4941         t_options = trace_options_init_dentry();
4942         if (!t_options)
4943                 return NULL;
4944
4945         return trace_create_file(option, 0644, t_options, (void *)index,
4946                                     &trace_options_core_fops);
4947 }
4948
4949 static __init void create_trace_options_dir(void)
4950 {
4951         struct dentry *t_options;
4952         int i;
4953
4954         t_options = trace_options_init_dentry();
4955         if (!t_options)
4956                 return;
4957
4958         for (i = 0; trace_options[i]; i++)
4959                 create_trace_option_core_file(trace_options[i], i);
4960 }
4961
4962 static ssize_t
4963 rb_simple_read(struct file *filp, char __user *ubuf,
4964                size_t cnt, loff_t *ppos)
4965 {
4966         struct trace_array *tr = filp->private_data;
4967         struct ring_buffer *buffer = tr->buffer;
4968         char buf[64];
4969         int r;
4970
4971         if (buffer)
4972                 r = ring_buffer_record_is_on(buffer);
4973         else
4974                 r = 0;
4975
4976         r = sprintf(buf, "%d\n", r);
4977
4978         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4979 }
4980
4981 static ssize_t
4982 rb_simple_write(struct file *filp, const char __user *ubuf,
4983                 size_t cnt, loff_t *ppos)
4984 {
4985         struct trace_array *tr = filp->private_data;
4986         struct ring_buffer *buffer = tr->buffer;
4987         unsigned long val;
4988         int ret;
4989
4990         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4991         if (ret)
4992                 return ret;
4993
4994         if (buffer) {
4995                 mutex_lock(&trace_types_lock);
4996                 if (val) {
4997                         ring_buffer_record_on(buffer);
4998                         if (current_trace->start)
4999                                 current_trace->start(tr);
5000                 } else {
5001                         ring_buffer_record_off(buffer);
5002                         if (current_trace->stop)
5003                                 current_trace->stop(tr);
5004                 }
5005                 mutex_unlock(&trace_types_lock);
5006         }
5007
5008         (*ppos)++;
5009
5010         return cnt;
5011 }
5012
5013 static const struct file_operations rb_simple_fops = {
5014         .open           = tracing_open_generic,
5015         .read           = rb_simple_read,
5016         .write          = rb_simple_write,
5017         .llseek         = default_llseek,
5018 };
5019
5020 static __init int tracer_init_debugfs(void)
5021 {
5022         struct dentry *d_tracer;
5023         int cpu;
5024
5025         trace_access_lock_init();
5026
5027         d_tracer = tracing_init_dentry();
5028
5029         trace_create_file("trace_options", 0644, d_tracer,
5030                         NULL, &tracing_iter_fops);
5031
5032         trace_create_file("tracing_cpumask", 0644, d_tracer,
5033                         NULL, &tracing_cpumask_fops);
5034
5035         trace_create_file("trace", 0644, d_tracer,
5036                         (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
5037
5038         trace_create_file("available_tracers", 0444, d_tracer,
5039                         &global_trace, &show_traces_fops);
5040
5041         trace_create_file("current_tracer", 0644, d_tracer,
5042                         &global_trace, &set_tracer_fops);
5043
5044 #ifdef CONFIG_TRACER_MAX_TRACE
5045         trace_create_file("tracing_max_latency", 0644, d_tracer,
5046                         &tracing_max_latency, &tracing_max_lat_fops);
5047 #endif
5048
5049         trace_create_file("tracing_thresh", 0644, d_tracer,
5050                         &tracing_thresh, &tracing_max_lat_fops);
5051
5052         trace_create_file("README", 0444, d_tracer,
5053                         NULL, &tracing_readme_fops);
5054
5055         trace_create_file("trace_pipe", 0444, d_tracer,
5056                         (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
5057
5058         trace_create_file("buffer_size_kb", 0644, d_tracer,
5059                         (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
5060
5061         trace_create_file("buffer_total_size_kb", 0444, d_tracer,
5062                         &global_trace, &tracing_total_entries_fops);
5063
5064         trace_create_file("free_buffer", 0644, d_tracer,
5065                         &global_trace, &tracing_free_buffer_fops);
5066
5067         trace_create_file("trace_marker", 0220, d_tracer,
5068                         NULL, &tracing_mark_fops);
5069
5070         trace_create_file("saved_cmdlines", 0444, d_tracer,
5071                         NULL, &tracing_saved_cmdlines_fops);
5072
5073         trace_create_file("trace_clock", 0644, d_tracer, NULL,
5074                           &trace_clock_fops);
5075
5076         trace_create_file("tracing_on", 0644, d_tracer,
5077                             &global_trace, &rb_simple_fops);
5078
5079 #ifdef CONFIG_DYNAMIC_FTRACE
5080         trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
5081                         &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
5082 #endif
5083
5084 #ifdef CONFIG_TRACER_SNAPSHOT
5085         trace_create_file("snapshot", 0644, d_tracer,
5086                           (void *) TRACE_PIPE_ALL_CPU, &snapshot_fops);
5087 #endif
5088
5089         create_trace_options_dir();
5090
5091         for_each_tracing_cpu(cpu)
5092                 tracing_init_debugfs_percpu(cpu);
5093
5094         return 0;
5095 }
5096
5097 static int trace_panic_handler(struct notifier_block *this,
5098                                unsigned long event, void *unused)
5099 {
5100         if (ftrace_dump_on_oops)
5101                 ftrace_dump(ftrace_dump_on_oops);
5102         return NOTIFY_OK;
5103 }
5104
5105 static struct notifier_block trace_panic_notifier = {
5106         .notifier_call  = trace_panic_handler,
5107         .next           = NULL,
5108         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
5109 };
5110
5111 static int trace_die_handler(struct notifier_block *self,
5112                              unsigned long val,
5113                              void *data)
5114 {
5115         switch (val) {
5116         case DIE_OOPS:
5117                 if (ftrace_dump_on_oops)
5118                         ftrace_dump(ftrace_dump_on_oops);
5119                 break;
5120         default:
5121                 break;
5122         }
5123         return NOTIFY_OK;
5124 }
5125
5126 static struct notifier_block trace_die_notifier = {
5127         .notifier_call = trace_die_handler,
5128         .priority = 200
5129 };
5130
5131 /*
5132  * printk is set to max of 1024, we really don't need it that big.
5133  * Nothing should be printing 1000 characters anyway.
5134  */
5135 #define TRACE_MAX_PRINT         1000
5136
5137 /*
5138  * Define here KERN_TRACE so that we have one place to modify
5139  * it if we decide to change what log level the ftrace dump
5140  * should be at.
5141  */
5142 #define KERN_TRACE              KERN_EMERG
5143
5144 void
5145 trace_printk_seq(struct trace_seq *s)
5146 {
5147         /* Probably should print a warning here. */
5148         if (s->len >= 1000)
5149                 s->len = 1000;
5150
5151         /* should be zero ended, but we are paranoid. */
5152         s->buffer[s->len] = 0;
5153
5154         printk(KERN_TRACE "%s", s->buffer);
5155
5156         trace_seq_init(s);
5157 }
5158
5159 void trace_init_global_iter(struct trace_iterator *iter)
5160 {
5161         iter->tr = &global_trace;
5162         iter->trace = current_trace;
5163         iter->cpu_file = TRACE_PIPE_ALL_CPU;
5164 }
5165
5166 static void
5167 __ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
5168 {
5169         static arch_spinlock_t ftrace_dump_lock =
5170                 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
5171         /* use static because iter can be a bit big for the stack */
5172         static struct trace_iterator iter;
5173         unsigned int old_userobj;
5174         static int dump_ran;
5175         unsigned long flags;
5176         int cnt = 0, cpu;
5177
5178         /* only one dump */
5179         local_irq_save(flags);
5180         arch_spin_lock(&ftrace_dump_lock);
5181         if (dump_ran)
5182                 goto out;
5183
5184         dump_ran = 1;
5185
5186         tracing_off();
5187
5188         /* Did function tracer already get disabled? */
5189         if (ftrace_is_dead()) {
5190                 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
5191                 printk("#          MAY BE MISSING FUNCTION EVENTS\n");
5192         }
5193
5194         if (disable_tracing)
5195                 ftrace_kill();
5196
5197         /* Simulate the iterator */
5198         trace_init_global_iter(&iter);
5199
5200         for_each_tracing_cpu(cpu) {
5201                 atomic_inc(&iter.tr->data[cpu]->disabled);
5202         }
5203
5204         old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
5205
5206         /* don't look at user memory in panic mode */
5207         trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
5208
5209         switch (oops_dump_mode) {
5210         case DUMP_ALL:
5211                 iter.cpu_file = TRACE_PIPE_ALL_CPU;
5212                 break;
5213         case DUMP_ORIG:
5214                 iter.cpu_file = raw_smp_processor_id();
5215                 break;
5216         case DUMP_NONE:
5217                 goto out_enable;
5218         default:
5219                 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
5220                 iter.cpu_file = TRACE_PIPE_ALL_CPU;
5221         }
5222
5223         printk(KERN_TRACE "Dumping ftrace buffer:\n");
5224
5225         /*
5226          * We need to stop all tracing on all CPUS to read the
5227          * the next buffer. This is a bit expensive, but is
5228          * not done often. We fill all what we can read,
5229          * and then release the locks again.
5230          */
5231
5232         while (!trace_empty(&iter)) {
5233
5234                 if (!cnt)
5235                         printk(KERN_TRACE "---------------------------------\n");
5236
5237                 cnt++;
5238
5239                 /* reset all but tr, trace, and overruns */
5240                 memset(&iter.seq, 0,
5241                        sizeof(struct trace_iterator) -
5242                        offsetof(struct trace_iterator, seq));
5243                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
5244                 iter.pos = -1;
5245
5246                 if (trace_find_next_entry_inc(&iter) != NULL) {
5247                         int ret;
5248
5249                         ret = print_trace_line(&iter);
5250                         if (ret != TRACE_TYPE_NO_CONSUME)
5251                                 trace_consume(&iter);
5252                 }
5253                 touch_nmi_watchdog();
5254
5255                 trace_printk_seq(&iter.seq);
5256         }
5257
5258         if (!cnt)
5259                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
5260         else
5261                 printk(KERN_TRACE "---------------------------------\n");
5262
5263  out_enable:
5264         /* Re-enable tracing if requested */
5265         if (!disable_tracing) {
5266                 trace_flags |= old_userobj;
5267
5268                 for_each_tracing_cpu(cpu) {
5269                         atomic_dec(&iter.tr->data[cpu]->disabled);
5270                 }
5271                 tracing_on();
5272         }
5273
5274  out:
5275         arch_spin_unlock(&ftrace_dump_lock);
5276         local_irq_restore(flags);
5277 }
5278
5279 /* By default: disable tracing after the dump */
5280 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
5281 {
5282         __ftrace_dump(true, oops_dump_mode);
5283 }
5284 EXPORT_SYMBOL_GPL(ftrace_dump);
5285
5286 __init static int tracer_alloc_buffers(void)
5287 {
5288         int ring_buf_size;
5289         enum ring_buffer_flags rb_flags;
5290         int i;
5291         int ret = -ENOMEM;
5292
5293
5294         if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
5295                 goto out;
5296
5297         if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
5298                 goto out_free_buffer_mask;
5299
5300         /* Only allocate trace_printk buffers if a trace_printk exists */
5301         if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
5302                 /* Must be called before global_trace.buffer is allocated */
5303                 trace_printk_init_buffers();
5304
5305         /* To save memory, keep the ring buffer size to its minimum */
5306         if (ring_buffer_expanded)
5307                 ring_buf_size = trace_buf_size;
5308         else
5309                 ring_buf_size = 1;
5310
5311         rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5312
5313         cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
5314         cpumask_copy(tracing_cpumask, cpu_all_mask);
5315
5316         /* TODO: make the number of buffers hot pluggable with CPUS */
5317         global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
5318         if (!global_trace.buffer) {
5319                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
5320                 WARN_ON(1);
5321                 goto out_free_cpumask;
5322         }
5323         if (global_trace.buffer_disabled)
5324                 tracing_off();
5325
5326
5327 #ifdef CONFIG_TRACER_MAX_TRACE
5328         max_tr.buffer = ring_buffer_alloc(1, rb_flags);
5329         if (!max_tr.buffer) {
5330                 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
5331                 WARN_ON(1);
5332                 ring_buffer_free(global_trace.buffer);
5333                 goto out_free_cpumask;
5334         }
5335 #endif
5336
5337         /* Allocate the first page for all buffers */
5338         for_each_tracing_cpu(i) {
5339                 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
5340                 max_tr.data[i] = &per_cpu(max_tr_data, i);
5341         }
5342
5343         set_buffer_entries(&global_trace,
5344                            ring_buffer_size(global_trace.buffer, 0));
5345 #ifdef CONFIG_TRACER_MAX_TRACE
5346         set_buffer_entries(&max_tr, 1);
5347 #endif
5348
5349         trace_init_cmdlines();
5350         init_irq_work(&trace_work_wakeup, trace_wake_up);
5351
5352         register_tracer(&nop_trace);
5353
5354         /* All seems OK, enable tracing */
5355         tracing_disabled = 0;
5356
5357         atomic_notifier_chain_register(&panic_notifier_list,
5358                                        &trace_panic_notifier);
5359
5360         register_die_notifier(&trace_die_notifier);
5361
5362         while (trace_boot_options) {
5363                 char *option;
5364
5365                 option = strsep(&trace_boot_options, ",");
5366                 trace_set_options(option);
5367         }
5368
5369         return 0;
5370
5371 out_free_cpumask:
5372         free_cpumask_var(tracing_cpumask);
5373 out_free_buffer_mask:
5374         free_cpumask_var(tracing_buffer_mask);
5375 out:
5376         return ret;
5377 }
5378
5379 __init static int clear_boot_tracer(void)
5380 {
5381         /*
5382          * The default tracer at boot buffer is an init section.
5383          * This function is called in lateinit. If we did not
5384          * find the boot tracer, then clear it out, to prevent
5385          * later registration from accessing the buffer that is
5386          * about to be freed.
5387          */
5388         if (!default_bootup_tracer)
5389                 return 0;
5390
5391         printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5392                default_bootup_tracer);
5393         default_bootup_tracer = NULL;
5394
5395         return 0;
5396 }
5397
5398 early_initcall(tracer_alloc_buffers);
5399 fs_initcall(tracer_init_debugfs);
5400 late_initcall(clear_boot_tracer);