0582a01a81e36c98ecfc2574b804075ea26c1273
[linux-3.10.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 Nadia Yvette Chambers
13  */
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
40 #include <linux/fs.h>
41 #include <linux/sched/rt.h>
42
43 #include "trace.h"
44 #include "trace_output.h"
45
46 /*
47  * On boot up, the ring buffer is set to the minimum size, so that
48  * we do not waste memory on systems that are not using tracing.
49  */
50 bool ring_buffer_expanded;
51
52 /*
53  * We need to change this state when a selftest is running.
54  * A selftest will lurk into the ring-buffer to count the
55  * entries inserted during the selftest although some concurrent
56  * insertions into the ring-buffer such as trace_printk could occurred
57  * at the same time, giving false positive or negative results.
58  */
59 static bool __read_mostly tracing_selftest_running;
60
61 /*
62  * If a tracer is running, we do not want to run SELFTEST.
63  */
64 bool __read_mostly tracing_selftest_disabled;
65
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
68         { }
69 };
70
71 static struct tracer_flags dummy_tracer_flags = {
72         .val = 0,
73         .opts = dummy_tracer_opt
74 };
75
76 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77 {
78         return 0;
79 }
80
81 /*
82  * To prevent the comm cache from being overwritten when no
83  * tracing is active, only save the comm when a trace event
84  * occurred.
85  */
86 static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88 /*
89  * Kill all tracing for good (never come back).
90  * It is initialized to 1 but will turn to zero if the initialization
91  * of the tracer is successful. But that is the only place that sets
92  * this back to zero.
93  */
94 static int tracing_disabled = 1;
95
96 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
97
98 cpumask_var_t __read_mostly     tracing_buffer_mask;
99
100 /*
101  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102  *
103  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104  * is set, then ftrace_dump is called. This will output the contents
105  * of the ftrace buffers to the console.  This is very useful for
106  * capturing traces that lead to crashes and outputing it to a
107  * serial console.
108  *
109  * It is default off, but you can enable it with either specifying
110  * "ftrace_dump_on_oops" in the kernel command line, or setting
111  * /proc/sys/kernel/ftrace_dump_on_oops
112  * Set 1 if you want to dump buffers of all CPUs
113  * Set 2 if you want to dump the buffer of the CPU that triggered oops
114  */
115
116 enum ftrace_dump_mode ftrace_dump_on_oops;
117
118 static int tracing_set_tracer(const char *buf);
119
120 #define MAX_TRACER_SIZE         100
121 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
122 static char *default_bootup_tracer;
123
124 static bool allocate_snapshot;
125
126 static int __init set_cmdline_ftrace(char *str)
127 {
128         strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
129         default_bootup_tracer = bootup_tracer_buf;
130         /* We are using ftrace early, expand it */
131         ring_buffer_expanded = true;
132         return 1;
133 }
134 __setup("ftrace=", set_cmdline_ftrace);
135
136 static int __init set_ftrace_dump_on_oops(char *str)
137 {
138         if (*str++ != '=' || !*str) {
139                 ftrace_dump_on_oops = DUMP_ALL;
140                 return 1;
141         }
142
143         if (!strcmp("orig_cpu", str)) {
144                 ftrace_dump_on_oops = DUMP_ORIG;
145                 return 1;
146         }
147
148         return 0;
149 }
150 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
151
152 static int __init boot_alloc_snapshot(char *str)
153 {
154         allocate_snapshot = true;
155         /* We also need the main ring buffer expanded */
156         ring_buffer_expanded = true;
157         return 1;
158 }
159 __setup("alloc_snapshot", boot_alloc_snapshot);
160
161
162 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
163 static char *trace_boot_options __initdata;
164
165 static int __init set_trace_boot_options(char *str)
166 {
167         strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
168         trace_boot_options = trace_boot_options_buf;
169         return 0;
170 }
171 __setup("trace_options=", set_trace_boot_options);
172
173 unsigned long long ns2usecs(cycle_t nsec)
174 {
175         nsec += 500;
176         do_div(nsec, 1000);
177         return nsec;
178 }
179
180 /*
181  * The global_trace is the descriptor that holds the tracing
182  * buffers for the live tracing. For each CPU, it contains
183  * a link list of pages that will store trace entries. The
184  * page descriptor of the pages in the memory is used to hold
185  * the link list by linking the lru item in the page descriptor
186  * to each of the pages in the buffer per CPU.
187  *
188  * For each active CPU there is a data field that holds the
189  * pages for the buffer for that CPU. Each CPU has the same number
190  * of pages allocated for its buffer.
191  */
192 static struct trace_array       global_trace;
193
194 LIST_HEAD(ftrace_trace_arrays);
195
196 int trace_array_get(struct trace_array *this_tr)
197 {
198         struct trace_array *tr;
199         int ret = -ENODEV;
200
201         mutex_lock(&trace_types_lock);
202         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
203                 if (tr == this_tr) {
204                         tr->ref++;
205                         ret = 0;
206                         break;
207                 }
208         }
209         mutex_unlock(&trace_types_lock);
210
211         return ret;
212 }
213
214 static void __trace_array_put(struct trace_array *this_tr)
215 {
216         WARN_ON(!this_tr->ref);
217         this_tr->ref--;
218 }
219
220 void trace_array_put(struct trace_array *this_tr)
221 {
222         mutex_lock(&trace_types_lock);
223         __trace_array_put(this_tr);
224         mutex_unlock(&trace_types_lock);
225 }
226
227 int filter_current_check_discard(struct ring_buffer *buffer,
228                                  struct ftrace_event_call *call, void *rec,
229                                  struct ring_buffer_event *event)
230 {
231         return filter_check_discard(call, rec, buffer, event);
232 }
233 EXPORT_SYMBOL_GPL(filter_current_check_discard);
234
235 cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
236 {
237         u64 ts;
238
239         /* Early boot up does not have a buffer yet */
240         if (!buf->buffer)
241                 return trace_clock_local();
242
243         ts = ring_buffer_time_stamp(buf->buffer, cpu);
244         ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
245
246         return ts;
247 }
248
249 cycle_t ftrace_now(int cpu)
250 {
251         return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
252 }
253
254 /**
255  * tracing_is_enabled - Show if global_trace has been disabled
256  *
257  * Shows if the global trace has been enabled or not. It uses the
258  * mirror flag "buffer_disabled" to be used in fast paths such as for
259  * the irqsoff tracer. But it may be inaccurate due to races. If you
260  * need to know the accurate state, use tracing_is_on() which is a little
261  * slower, but accurate.
262  */
263 int tracing_is_enabled(void)
264 {
265         /*
266          * For quick access (irqsoff uses this in fast path), just
267          * return the mirror variable of the state of the ring buffer.
268          * It's a little racy, but we don't really care.
269          */
270         smp_rmb();
271         return !global_trace.buffer_disabled;
272 }
273
274 /*
275  * trace_buf_size is the size in bytes that is allocated
276  * for a buffer. Note, the number of bytes is always rounded
277  * to page size.
278  *
279  * This number is purposely set to a low number of 16384.
280  * If the dump on oops happens, it will be much appreciated
281  * to not have to wait for all that output. Anyway this can be
282  * boot time and run time configurable.
283  */
284 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
285
286 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
287
288 /* trace_types holds a link list of available tracers. */
289 static struct tracer            *trace_types __read_mostly;
290
291 /*
292  * trace_types_lock is used to protect the trace_types list.
293  */
294 DEFINE_MUTEX(trace_types_lock);
295
296 /*
297  * serialize the access of the ring buffer
298  *
299  * ring buffer serializes readers, but it is low level protection.
300  * The validity of the events (which returns by ring_buffer_peek() ..etc)
301  * are not protected by ring buffer.
302  *
303  * The content of events may become garbage if we allow other process consumes
304  * these events concurrently:
305  *   A) the page of the consumed events may become a normal page
306  *      (not reader page) in ring buffer, and this page will be rewrited
307  *      by events producer.
308  *   B) The page of the consumed events may become a page for splice_read,
309  *      and this page will be returned to system.
310  *
311  * These primitives allow multi process access to different cpu ring buffer
312  * concurrently.
313  *
314  * These primitives don't distinguish read-only and read-consume access.
315  * Multi read-only access are also serialized.
316  */
317
318 #ifdef CONFIG_SMP
319 static DECLARE_RWSEM(all_cpu_access_lock);
320 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
321
322 static inline void trace_access_lock(int cpu)
323 {
324         if (cpu == RING_BUFFER_ALL_CPUS) {
325                 /* gain it for accessing the whole ring buffer. */
326                 down_write(&all_cpu_access_lock);
327         } else {
328                 /* gain it for accessing a cpu ring buffer. */
329
330                 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
331                 down_read(&all_cpu_access_lock);
332
333                 /* Secondly block other access to this @cpu ring buffer. */
334                 mutex_lock(&per_cpu(cpu_access_lock, cpu));
335         }
336 }
337
338 static inline void trace_access_unlock(int cpu)
339 {
340         if (cpu == RING_BUFFER_ALL_CPUS) {
341                 up_write(&all_cpu_access_lock);
342         } else {
343                 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
344                 up_read(&all_cpu_access_lock);
345         }
346 }
347
348 static inline void trace_access_lock_init(void)
349 {
350         int cpu;
351
352         for_each_possible_cpu(cpu)
353                 mutex_init(&per_cpu(cpu_access_lock, cpu));
354 }
355
356 #else
357
358 static DEFINE_MUTEX(access_lock);
359
360 static inline void trace_access_lock(int cpu)
361 {
362         (void)cpu;
363         mutex_lock(&access_lock);
364 }
365
366 static inline void trace_access_unlock(int cpu)
367 {
368         (void)cpu;
369         mutex_unlock(&access_lock);
370 }
371
372 static inline void trace_access_lock_init(void)
373 {
374 }
375
376 #endif
377
378 /* trace_flags holds trace_options default values */
379 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
380         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
381         TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
382         TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
383
384 void tracer_tracing_on(struct trace_array *tr)
385 {
386         if (tr->trace_buffer.buffer)
387                 ring_buffer_record_on(tr->trace_buffer.buffer);
388         /*
389          * This flag is looked at when buffers haven't been allocated
390          * yet, or by some tracers (like irqsoff), that just want to
391          * know if the ring buffer has been disabled, but it can handle
392          * races of where it gets disabled but we still do a record.
393          * As the check is in the fast path of the tracers, it is more
394          * important to be fast than accurate.
395          */
396         tr->buffer_disabled = 0;
397         /* Make the flag seen by readers */
398         smp_wmb();
399 }
400
401 /**
402  * tracing_on - enable tracing buffers
403  *
404  * This function enables tracing buffers that may have been
405  * disabled with tracing_off.
406  */
407 void tracing_on(void)
408 {
409         tracer_tracing_on(&global_trace);
410 }
411 EXPORT_SYMBOL_GPL(tracing_on);
412
413 /**
414  * __trace_puts - write a constant string into the trace buffer.
415  * @ip:    The address of the caller
416  * @str:   The constant string to write
417  * @size:  The size of the string.
418  */
419 int __trace_puts(unsigned long ip, const char *str, int size)
420 {
421         struct ring_buffer_event *event;
422         struct ring_buffer *buffer;
423         struct print_entry *entry;
424         unsigned long irq_flags;
425         int alloc;
426
427         alloc = sizeof(*entry) + size + 2; /* possible \n added */
428
429         local_save_flags(irq_flags);
430         buffer = global_trace.trace_buffer.buffer;
431         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
432                                           irq_flags, preempt_count());
433         if (!event)
434                 return 0;
435
436         entry = ring_buffer_event_data(event);
437         entry->ip = ip;
438
439         memcpy(&entry->buf, str, size);
440
441         /* Add a newline if necessary */
442         if (entry->buf[size - 1] != '\n') {
443                 entry->buf[size] = '\n';
444                 entry->buf[size + 1] = '\0';
445         } else
446                 entry->buf[size] = '\0';
447
448         __buffer_unlock_commit(buffer, event);
449
450         return size;
451 }
452 EXPORT_SYMBOL_GPL(__trace_puts);
453
454 /**
455  * __trace_bputs - write the pointer to a constant string into trace buffer
456  * @ip:    The address of the caller
457  * @str:   The constant string to write to the buffer to
458  */
459 int __trace_bputs(unsigned long ip, const char *str)
460 {
461         struct ring_buffer_event *event;
462         struct ring_buffer *buffer;
463         struct bputs_entry *entry;
464         unsigned long irq_flags;
465         int size = sizeof(struct bputs_entry);
466
467         local_save_flags(irq_flags);
468         buffer = global_trace.trace_buffer.buffer;
469         event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
470                                           irq_flags, preempt_count());
471         if (!event)
472                 return 0;
473
474         entry = ring_buffer_event_data(event);
475         entry->ip                       = ip;
476         entry->str                      = str;
477
478         __buffer_unlock_commit(buffer, event);
479
480         return 1;
481 }
482 EXPORT_SYMBOL_GPL(__trace_bputs);
483
484 #ifdef CONFIG_TRACER_SNAPSHOT
485 /**
486  * trace_snapshot - take a snapshot of the current buffer.
487  *
488  * This causes a swap between the snapshot buffer and the current live
489  * tracing buffer. You can use this to take snapshots of the live
490  * trace when some condition is triggered, but continue to trace.
491  *
492  * Note, make sure to allocate the snapshot with either
493  * a tracing_snapshot_alloc(), or by doing it manually
494  * with: echo 1 > /sys/kernel/debug/tracing/snapshot
495  *
496  * If the snapshot buffer is not allocated, it will stop tracing.
497  * Basically making a permanent snapshot.
498  */
499 void tracing_snapshot(void)
500 {
501         struct trace_array *tr = &global_trace;
502         struct tracer *tracer = tr->current_trace;
503         unsigned long flags;
504
505         if (in_nmi()) {
506                 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
507                 internal_trace_puts("*** snapshot is being ignored        ***\n");
508                 return;
509         }
510
511         if (!tr->allocated_snapshot) {
512                 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
513                 internal_trace_puts("*** stopping trace here!   ***\n");
514                 tracing_off();
515                 return;
516         }
517
518         /* Note, snapshot can not be used when the tracer uses it */
519         if (tracer->use_max_tr) {
520                 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
521                 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
522                 return;
523         }
524
525         local_irq_save(flags);
526         update_max_tr(tr, current, smp_processor_id());
527         local_irq_restore(flags);
528 }
529 EXPORT_SYMBOL_GPL(tracing_snapshot);
530
531 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
532                                         struct trace_buffer *size_buf, int cpu_id);
533 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
534
535 static int alloc_snapshot(struct trace_array *tr)
536 {
537         int ret;
538
539         if (!tr->allocated_snapshot) {
540
541                 /* allocate spare buffer */
542                 ret = resize_buffer_duplicate_size(&tr->max_buffer,
543                                    &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
544                 if (ret < 0)
545                         return ret;
546
547                 tr->allocated_snapshot = true;
548         }
549
550         return 0;
551 }
552
553 void free_snapshot(struct trace_array *tr)
554 {
555         /*
556          * We don't free the ring buffer. instead, resize it because
557          * The max_tr ring buffer has some state (e.g. ring->clock) and
558          * we want preserve it.
559          */
560         ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
561         set_buffer_entries(&tr->max_buffer, 1);
562         tracing_reset_online_cpus(&tr->max_buffer);
563         tr->allocated_snapshot = false;
564 }
565
566 /**
567  * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
568  *
569  * This is similar to trace_snapshot(), but it will allocate the
570  * snapshot buffer if it isn't already allocated. Use this only
571  * where it is safe to sleep, as the allocation may sleep.
572  *
573  * This causes a swap between the snapshot buffer and the current live
574  * tracing buffer. You can use this to take snapshots of the live
575  * trace when some condition is triggered, but continue to trace.
576  */
577 void tracing_snapshot_alloc(void)
578 {
579         struct trace_array *tr = &global_trace;
580         int ret;
581
582         ret = alloc_snapshot(tr);
583         if (WARN_ON(ret < 0))
584                 return;
585
586         tracing_snapshot();
587 }
588 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
589 #else
590 void tracing_snapshot(void)
591 {
592         WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
593 }
594 EXPORT_SYMBOL_GPL(tracing_snapshot);
595 void tracing_snapshot_alloc(void)
596 {
597         /* Give warning */
598         tracing_snapshot();
599 }
600 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
601 #endif /* CONFIG_TRACER_SNAPSHOT */
602
603 void tracer_tracing_off(struct trace_array *tr)
604 {
605         if (tr->trace_buffer.buffer)
606                 ring_buffer_record_off(tr->trace_buffer.buffer);
607         /*
608          * This flag is looked at when buffers haven't been allocated
609          * yet, or by some tracers (like irqsoff), that just want to
610          * know if the ring buffer has been disabled, but it can handle
611          * races of where it gets disabled but we still do a record.
612          * As the check is in the fast path of the tracers, it is more
613          * important to be fast than accurate.
614          */
615         tr->buffer_disabled = 1;
616         /* Make the flag seen by readers */
617         smp_wmb();
618 }
619
620 /**
621  * tracing_off - turn off tracing buffers
622  *
623  * This function stops the tracing buffers from recording data.
624  * It does not disable any overhead the tracers themselves may
625  * be causing. This function simply causes all recording to
626  * the ring buffers to fail.
627  */
628 void tracing_off(void)
629 {
630         tracer_tracing_off(&global_trace);
631 }
632 EXPORT_SYMBOL_GPL(tracing_off);
633
634 /**
635  * tracer_tracing_is_on - show real state of ring buffer enabled
636  * @tr : the trace array to know if ring buffer is enabled
637  *
638  * Shows real state of the ring buffer if it is enabled or not.
639  */
640 int tracer_tracing_is_on(struct trace_array *tr)
641 {
642         if (tr->trace_buffer.buffer)
643                 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
644         return !tr->buffer_disabled;
645 }
646
647 /**
648  * tracing_is_on - show state of ring buffers enabled
649  */
650 int tracing_is_on(void)
651 {
652         return tracer_tracing_is_on(&global_trace);
653 }
654 EXPORT_SYMBOL_GPL(tracing_is_on);
655
656 static int __init set_buf_size(char *str)
657 {
658         unsigned long buf_size;
659
660         if (!str)
661                 return 0;
662         buf_size = memparse(str, &str);
663         /* nr_entries can not be zero */
664         if (buf_size == 0)
665                 return 0;
666         trace_buf_size = buf_size;
667         return 1;
668 }
669 __setup("trace_buf_size=", set_buf_size);
670
671 static int __init set_tracing_thresh(char *str)
672 {
673         unsigned long threshold;
674         int ret;
675
676         if (!str)
677                 return 0;
678         ret = kstrtoul(str, 0, &threshold);
679         if (ret < 0)
680                 return 0;
681         tracing_thresh = threshold * 1000;
682         return 1;
683 }
684 __setup("tracing_thresh=", set_tracing_thresh);
685
686 unsigned long nsecs_to_usecs(unsigned long nsecs)
687 {
688         return nsecs / 1000;
689 }
690
691 /* These must match the bit postions in trace_iterator_flags */
692 static const char *trace_options[] = {
693         "print-parent",
694         "sym-offset",
695         "sym-addr",
696         "verbose",
697         "raw",
698         "hex",
699         "bin",
700         "block",
701         "stacktrace",
702         "trace_printk",
703         "ftrace_preempt",
704         "branch",
705         "annotate",
706         "userstacktrace",
707         "sym-userobj",
708         "printk-msg-only",
709         "context-info",
710         "latency-format",
711         "sleep-time",
712         "graph-time",
713         "record-cmd",
714         "overwrite",
715         "disable_on_free",
716         "irq-info",
717         "markers",
718         "function-trace",
719         NULL
720 };
721
722 static struct {
723         u64 (*func)(void);
724         const char *name;
725         int in_ns;              /* is this clock in nanoseconds? */
726 } trace_clocks[] = {
727         { trace_clock_local,    "local",        1 },
728         { trace_clock_global,   "global",       1 },
729         { trace_clock_counter,  "counter",      0 },
730         { trace_clock_jiffies,  "uptime",       1 },
731         { trace_clock,          "perf",         1 },
732         ARCH_TRACE_CLOCKS
733 };
734
735 /*
736  * trace_parser_get_init - gets the buffer for trace parser
737  */
738 int trace_parser_get_init(struct trace_parser *parser, int size)
739 {
740         memset(parser, 0, sizeof(*parser));
741
742         parser->buffer = kmalloc(size, GFP_KERNEL);
743         if (!parser->buffer)
744                 return 1;
745
746         parser->size = size;
747         return 0;
748 }
749
750 /*
751  * trace_parser_put - frees the buffer for trace parser
752  */
753 void trace_parser_put(struct trace_parser *parser)
754 {
755         kfree(parser->buffer);
756 }
757
758 /*
759  * trace_get_user - reads the user input string separated by  space
760  * (matched by isspace(ch))
761  *
762  * For each string found the 'struct trace_parser' is updated,
763  * and the function returns.
764  *
765  * Returns number of bytes read.
766  *
767  * See kernel/trace/trace.h for 'struct trace_parser' details.
768  */
769 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
770         size_t cnt, loff_t *ppos)
771 {
772         char ch;
773         size_t read = 0;
774         ssize_t ret;
775
776         if (!*ppos)
777                 trace_parser_clear(parser);
778
779         ret = get_user(ch, ubuf++);
780         if (ret)
781                 goto out;
782
783         read++;
784         cnt--;
785
786         /*
787          * The parser is not finished with the last write,
788          * continue reading the user input without skipping spaces.
789          */
790         if (!parser->cont) {
791                 /* skip white space */
792                 while (cnt && isspace(ch)) {
793                         ret = get_user(ch, ubuf++);
794                         if (ret)
795                                 goto out;
796                         read++;
797                         cnt--;
798                 }
799
800                 /* only spaces were written */
801                 if (isspace(ch)) {
802                         *ppos += read;
803                         ret = read;
804                         goto out;
805                 }
806
807                 parser->idx = 0;
808         }
809
810         /* read the non-space input */
811         while (cnt && !isspace(ch)) {
812                 if (parser->idx < parser->size - 1)
813                         parser->buffer[parser->idx++] = ch;
814                 else {
815                         ret = -EINVAL;
816                         goto out;
817                 }
818                 ret = get_user(ch, ubuf++);
819                 if (ret)
820                         goto out;
821                 read++;
822                 cnt--;
823         }
824
825         /* We either got finished input or we have to wait for another call. */
826         if (isspace(ch)) {
827                 parser->buffer[parser->idx] = 0;
828                 parser->cont = false;
829         } else {
830                 parser->cont = true;
831                 parser->buffer[parser->idx++] = ch;
832         }
833
834         *ppos += read;
835         ret = read;
836
837 out:
838         return ret;
839 }
840
841 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
842 {
843         int len;
844         int ret;
845
846         if (!cnt)
847                 return 0;
848
849         if (s->len <= s->readpos)
850                 return -EBUSY;
851
852         len = s->len - s->readpos;
853         if (cnt > len)
854                 cnt = len;
855         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
856         if (ret == cnt)
857                 return -EFAULT;
858
859         cnt -= ret;
860
861         s->readpos += cnt;
862         return cnt;
863 }
864
865 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
866 {
867         int len;
868
869         if (s->len <= s->readpos)
870                 return -EBUSY;
871
872         len = s->len - s->readpos;
873         if (cnt > len)
874                 cnt = len;
875         memcpy(buf, s->buffer + s->readpos, cnt);
876
877         s->readpos += cnt;
878         return cnt;
879 }
880
881 /*
882  * ftrace_max_lock is used to protect the swapping of buffers
883  * when taking a max snapshot. The buffers themselves are
884  * protected by per_cpu spinlocks. But the action of the swap
885  * needs its own lock.
886  *
887  * This is defined as a arch_spinlock_t in order to help
888  * with performance when lockdep debugging is enabled.
889  *
890  * It is also used in other places outside the update_max_tr
891  * so it needs to be defined outside of the
892  * CONFIG_TRACER_MAX_TRACE.
893  */
894 static arch_spinlock_t ftrace_max_lock =
895         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
896
897 unsigned long __read_mostly     tracing_thresh;
898
899 #ifdef CONFIG_TRACER_MAX_TRACE
900 unsigned long __read_mostly     tracing_max_latency;
901
902 /*
903  * Copy the new maximum trace into the separate maximum-trace
904  * structure. (this way the maximum trace is permanently saved,
905  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
906  */
907 static void
908 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
909 {
910         struct trace_buffer *trace_buf = &tr->trace_buffer;
911         struct trace_buffer *max_buf = &tr->max_buffer;
912         struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
913         struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
914
915         max_buf->cpu = cpu;
916         max_buf->time_start = data->preempt_timestamp;
917
918         max_data->saved_latency = tracing_max_latency;
919         max_data->critical_start = data->critical_start;
920         max_data->critical_end = data->critical_end;
921
922         memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
923         max_data->pid = tsk->pid;
924         /*
925          * If tsk == current, then use current_uid(), as that does not use
926          * RCU. The irq tracer can be called out of RCU scope.
927          */
928         if (tsk == current)
929                 max_data->uid = current_uid();
930         else
931                 max_data->uid = task_uid(tsk);
932
933         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
934         max_data->policy = tsk->policy;
935         max_data->rt_priority = tsk->rt_priority;
936
937         /* record this tasks comm */
938         tracing_record_cmdline(tsk);
939 }
940
941 /**
942  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
943  * @tr: tracer
944  * @tsk: the task with the latency
945  * @cpu: The cpu that initiated the trace.
946  *
947  * Flip the buffers between the @tr and the max_tr and record information
948  * about which task was the cause of this latency.
949  */
950 void
951 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
952 {
953         struct ring_buffer *buf;
954
955         if (tr->stop_count)
956                 return;
957
958         WARN_ON_ONCE(!irqs_disabled());
959
960         if (!tr->allocated_snapshot) {
961                 /* Only the nop tracer should hit this when disabling */
962                 WARN_ON_ONCE(tr->current_trace != &nop_trace);
963                 return;
964         }
965
966         arch_spin_lock(&ftrace_max_lock);
967
968         buf = tr->trace_buffer.buffer;
969         tr->trace_buffer.buffer = tr->max_buffer.buffer;
970         tr->max_buffer.buffer = buf;
971
972         __update_max_tr(tr, tsk, cpu);
973         arch_spin_unlock(&ftrace_max_lock);
974 }
975
976 /**
977  * update_max_tr_single - only copy one trace over, and reset the rest
978  * @tr - tracer
979  * @tsk - task with the latency
980  * @cpu - the cpu of the buffer to copy.
981  *
982  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
983  */
984 void
985 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
986 {
987         int ret;
988
989         if (tr->stop_count)
990                 return;
991
992         WARN_ON_ONCE(!irqs_disabled());
993         if (!tr->allocated_snapshot) {
994                 /* Only the nop tracer should hit this when disabling */
995                 WARN_ON_ONCE(tr->current_trace != &nop_trace);
996                 return;
997         }
998
999         arch_spin_lock(&ftrace_max_lock);
1000
1001         ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1002
1003         if (ret == -EBUSY) {
1004                 /*
1005                  * We failed to swap the buffer due to a commit taking
1006                  * place on this CPU. We fail to record, but we reset
1007                  * the max trace buffer (no one writes directly to it)
1008                  * and flag that it failed.
1009                  */
1010                 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1011                         "Failed to swap buffers due to commit in progress\n");
1012         }
1013
1014         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1015
1016         __update_max_tr(tr, tsk, cpu);
1017         arch_spin_unlock(&ftrace_max_lock);
1018 }
1019 #endif /* CONFIG_TRACER_MAX_TRACE */
1020
1021 static void default_wait_pipe(struct trace_iterator *iter)
1022 {
1023         /* Iterators are static, they should be filled or empty */
1024         if (trace_buffer_iter(iter, iter->cpu_file))
1025                 return;
1026
1027         ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1028 }
1029
1030 #ifdef CONFIG_FTRACE_STARTUP_TEST
1031 static int run_tracer_selftest(struct tracer *type)
1032 {
1033         struct trace_array *tr = &global_trace;
1034         struct tracer *saved_tracer = tr->current_trace;
1035         int ret;
1036
1037         if (!type->selftest || tracing_selftest_disabled)
1038                 return 0;
1039
1040         /*
1041          * Run a selftest on this tracer.
1042          * Here we reset the trace buffer, and set the current
1043          * tracer to be this tracer. The tracer can then run some
1044          * internal tracing to verify that everything is in order.
1045          * If we fail, we do not register this tracer.
1046          */
1047         tracing_reset_online_cpus(&tr->trace_buffer);
1048
1049         tr->current_trace = type;
1050
1051 #ifdef CONFIG_TRACER_MAX_TRACE
1052         if (type->use_max_tr) {
1053                 /* If we expanded the buffers, make sure the max is expanded too */
1054                 if (ring_buffer_expanded)
1055                         ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1056                                            RING_BUFFER_ALL_CPUS);
1057                 tr->allocated_snapshot = true;
1058         }
1059 #endif
1060
1061         /* the test is responsible for initializing and enabling */
1062         pr_info("Testing tracer %s: ", type->name);
1063         ret = type->selftest(type, tr);
1064         /* the test is responsible for resetting too */
1065         tr->current_trace = saved_tracer;
1066         if (ret) {
1067                 printk(KERN_CONT "FAILED!\n");
1068                 /* Add the warning after printing 'FAILED' */
1069                 WARN_ON(1);
1070                 return -1;
1071         }
1072         /* Only reset on passing, to avoid touching corrupted buffers */
1073         tracing_reset_online_cpus(&tr->trace_buffer);
1074
1075 #ifdef CONFIG_TRACER_MAX_TRACE
1076         if (type->use_max_tr) {
1077                 tr->allocated_snapshot = false;
1078
1079                 /* Shrink the max buffer again */
1080                 if (ring_buffer_expanded)
1081                         ring_buffer_resize(tr->max_buffer.buffer, 1,
1082                                            RING_BUFFER_ALL_CPUS);
1083         }
1084 #endif
1085
1086         printk(KERN_CONT "PASSED\n");
1087         return 0;
1088 }
1089 #else
1090 static inline int run_tracer_selftest(struct tracer *type)
1091 {
1092         return 0;
1093 }
1094 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1095
1096 /**
1097  * register_tracer - register a tracer with the ftrace system.
1098  * @type - the plugin for the tracer
1099  *
1100  * Register a new plugin tracer.
1101  */
1102 int register_tracer(struct tracer *type)
1103 {
1104         struct tracer *t;
1105         int ret = 0;
1106
1107         if (!type->name) {
1108                 pr_info("Tracer must have a name\n");
1109                 return -1;
1110         }
1111
1112         if (strlen(type->name) >= MAX_TRACER_SIZE) {
1113                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1114                 return -1;
1115         }
1116
1117         mutex_lock(&trace_types_lock);
1118
1119         tracing_selftest_running = true;
1120
1121         for (t = trace_types; t; t = t->next) {
1122                 if (strcmp(type->name, t->name) == 0) {
1123                         /* already found */
1124                         pr_info("Tracer %s already registered\n",
1125                                 type->name);
1126                         ret = -1;
1127                         goto out;
1128                 }
1129         }
1130
1131         if (!type->set_flag)
1132                 type->set_flag = &dummy_set_flag;
1133         if (!type->flags)
1134                 type->flags = &dummy_tracer_flags;
1135         else
1136                 if (!type->flags->opts)
1137                         type->flags->opts = dummy_tracer_opt;
1138         if (!type->wait_pipe)
1139                 type->wait_pipe = default_wait_pipe;
1140
1141         ret = run_tracer_selftest(type);
1142         if (ret < 0)
1143                 goto out;
1144
1145         type->next = trace_types;
1146         trace_types = type;
1147
1148  out:
1149         tracing_selftest_running = false;
1150         mutex_unlock(&trace_types_lock);
1151
1152         if (ret || !default_bootup_tracer)
1153                 goto out_unlock;
1154
1155         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1156                 goto out_unlock;
1157
1158         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1159         /* Do we want this tracer to start on bootup? */
1160         tracing_set_tracer(type->name);
1161         default_bootup_tracer = NULL;
1162         /* disable other selftests, since this will break it. */
1163         tracing_selftest_disabled = true;
1164 #ifdef CONFIG_FTRACE_STARTUP_TEST
1165         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1166                type->name);
1167 #endif
1168
1169  out_unlock:
1170         return ret;
1171 }
1172
1173 void tracing_reset(struct trace_buffer *buf, int cpu)
1174 {
1175         struct ring_buffer *buffer = buf->buffer;
1176
1177         if (!buffer)
1178                 return;
1179
1180         ring_buffer_record_disable(buffer);
1181
1182         /* Make sure all commits have finished */
1183         synchronize_sched();
1184         ring_buffer_reset_cpu(buffer, cpu);
1185
1186         ring_buffer_record_enable(buffer);
1187 }
1188
1189 void tracing_reset_online_cpus(struct trace_buffer *buf)
1190 {
1191         struct ring_buffer *buffer = buf->buffer;
1192         int cpu;
1193
1194         if (!buffer)
1195                 return;
1196
1197         ring_buffer_record_disable(buffer);
1198
1199         /* Make sure all commits have finished */
1200         synchronize_sched();
1201
1202         buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1203
1204         for_each_online_cpu(cpu)
1205                 ring_buffer_reset_cpu(buffer, cpu);
1206
1207         ring_buffer_record_enable(buffer);
1208 }
1209
1210 /* Must have trace_types_lock held */
1211 void tracing_reset_all_online_cpus(void)
1212 {
1213         struct trace_array *tr;
1214
1215         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1216                 tracing_reset_online_cpus(&tr->trace_buffer);
1217 #ifdef CONFIG_TRACER_MAX_TRACE
1218                 tracing_reset_online_cpus(&tr->max_buffer);
1219 #endif
1220         }
1221 }
1222
1223 #define SAVED_CMDLINES 128
1224 #define NO_CMDLINE_MAP UINT_MAX
1225 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1226 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1227 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1228 static int cmdline_idx;
1229 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1230
1231 /* temporary disable recording */
1232 static atomic_t trace_record_cmdline_disabled __read_mostly;
1233
1234 static void trace_init_cmdlines(void)
1235 {
1236         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1237         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
1238         cmdline_idx = 0;
1239 }
1240
1241 int is_tracing_stopped(void)
1242 {
1243         return global_trace.stop_count;
1244 }
1245
1246 /**
1247  * ftrace_off_permanent - disable all ftrace code permanently
1248  *
1249  * This should only be called when a serious anomally has
1250  * been detected.  This will turn off the function tracing,
1251  * ring buffers, and other tracing utilites. It takes no
1252  * locks and can be called from any context.
1253  */
1254 void ftrace_off_permanent(void)
1255 {
1256         tracing_disabled = 1;
1257         ftrace_stop();
1258         tracing_off_permanent();
1259 }
1260
1261 /**
1262  * tracing_start - quick start of the tracer
1263  *
1264  * If tracing is enabled but was stopped by tracing_stop,
1265  * this will start the tracer back up.
1266  */
1267 void tracing_start(void)
1268 {
1269         struct ring_buffer *buffer;
1270         unsigned long flags;
1271
1272         if (tracing_disabled)
1273                 return;
1274
1275         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1276         if (--global_trace.stop_count) {
1277                 if (global_trace.stop_count < 0) {
1278                         /* Someone screwed up their debugging */
1279                         WARN_ON_ONCE(1);
1280                         global_trace.stop_count = 0;
1281                 }
1282                 goto out;
1283         }
1284
1285         /* Prevent the buffers from switching */
1286         arch_spin_lock(&ftrace_max_lock);
1287
1288         buffer = global_trace.trace_buffer.buffer;
1289         if (buffer)
1290                 ring_buffer_record_enable(buffer);
1291
1292 #ifdef CONFIG_TRACER_MAX_TRACE
1293         buffer = global_trace.max_buffer.buffer;
1294         if (buffer)
1295                 ring_buffer_record_enable(buffer);
1296 #endif
1297
1298         arch_spin_unlock(&ftrace_max_lock);
1299
1300         ftrace_start();
1301  out:
1302         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1303 }
1304
1305 static void tracing_start_tr(struct trace_array *tr)
1306 {
1307         struct ring_buffer *buffer;
1308         unsigned long flags;
1309
1310         if (tracing_disabled)
1311                 return;
1312
1313         /* If global, we need to also start the max tracer */
1314         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1315                 return tracing_start();
1316
1317         raw_spin_lock_irqsave(&tr->start_lock, flags);
1318
1319         if (--tr->stop_count) {
1320                 if (tr->stop_count < 0) {
1321                         /* Someone screwed up their debugging */
1322                         WARN_ON_ONCE(1);
1323                         tr->stop_count = 0;
1324                 }
1325                 goto out;
1326         }
1327
1328         buffer = tr->trace_buffer.buffer;
1329         if (buffer)
1330                 ring_buffer_record_enable(buffer);
1331
1332  out:
1333         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1334 }
1335
1336 /**
1337  * tracing_stop - quick stop of the tracer
1338  *
1339  * Light weight way to stop tracing. Use in conjunction with
1340  * tracing_start.
1341  */
1342 void tracing_stop(void)
1343 {
1344         struct ring_buffer *buffer;
1345         unsigned long flags;
1346
1347         ftrace_stop();
1348         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1349         if (global_trace.stop_count++)
1350                 goto out;
1351
1352         /* Prevent the buffers from switching */
1353         arch_spin_lock(&ftrace_max_lock);
1354
1355         buffer = global_trace.trace_buffer.buffer;
1356         if (buffer)
1357                 ring_buffer_record_disable(buffer);
1358
1359 #ifdef CONFIG_TRACER_MAX_TRACE
1360         buffer = global_trace.max_buffer.buffer;
1361         if (buffer)
1362                 ring_buffer_record_disable(buffer);
1363 #endif
1364
1365         arch_spin_unlock(&ftrace_max_lock);
1366
1367  out:
1368         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1369 }
1370
1371 static void tracing_stop_tr(struct trace_array *tr)
1372 {
1373         struct ring_buffer *buffer;
1374         unsigned long flags;
1375
1376         /* If global, we need to also stop the max tracer */
1377         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1378                 return tracing_stop();
1379
1380         raw_spin_lock_irqsave(&tr->start_lock, flags);
1381         if (tr->stop_count++)
1382                 goto out;
1383
1384         buffer = tr->trace_buffer.buffer;
1385         if (buffer)
1386                 ring_buffer_record_disable(buffer);
1387
1388  out:
1389         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1390 }
1391
1392 void trace_stop_cmdline_recording(void);
1393
1394 static void trace_save_cmdline(struct task_struct *tsk)
1395 {
1396         unsigned pid, idx;
1397
1398         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1399                 return;
1400
1401         /*
1402          * It's not the end of the world if we don't get
1403          * the lock, but we also don't want to spin
1404          * nor do we want to disable interrupts,
1405          * so if we miss here, then better luck next time.
1406          */
1407         if (!arch_spin_trylock(&trace_cmdline_lock))
1408                 return;
1409
1410         idx = map_pid_to_cmdline[tsk->pid];
1411         if (idx == NO_CMDLINE_MAP) {
1412                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1413
1414                 /*
1415                  * Check whether the cmdline buffer at idx has a pid
1416                  * mapped. We are going to overwrite that entry so we
1417                  * need to clear the map_pid_to_cmdline. Otherwise we
1418                  * would read the new comm for the old pid.
1419                  */
1420                 pid = map_cmdline_to_pid[idx];
1421                 if (pid != NO_CMDLINE_MAP)
1422                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1423
1424                 map_cmdline_to_pid[idx] = tsk->pid;
1425                 map_pid_to_cmdline[tsk->pid] = idx;
1426
1427                 cmdline_idx = idx;
1428         }
1429
1430         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1431
1432         arch_spin_unlock(&trace_cmdline_lock);
1433 }
1434
1435 void trace_find_cmdline(int pid, char comm[])
1436 {
1437         unsigned map;
1438
1439         if (!pid) {
1440                 strcpy(comm, "<idle>");
1441                 return;
1442         }
1443
1444         if (WARN_ON_ONCE(pid < 0)) {
1445                 strcpy(comm, "<XXX>");
1446                 return;
1447         }
1448
1449         if (pid > PID_MAX_DEFAULT) {
1450                 strcpy(comm, "<...>");
1451                 return;
1452         }
1453
1454         preempt_disable();
1455         arch_spin_lock(&trace_cmdline_lock);
1456         map = map_pid_to_cmdline[pid];
1457         if (map != NO_CMDLINE_MAP)
1458                 strcpy(comm, saved_cmdlines[map]);
1459         else
1460                 strcpy(comm, "<...>");
1461
1462         arch_spin_unlock(&trace_cmdline_lock);
1463         preempt_enable();
1464 }
1465
1466 void tracing_record_cmdline(struct task_struct *tsk)
1467 {
1468         if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1469                 return;
1470
1471         if (!__this_cpu_read(trace_cmdline_save))
1472                 return;
1473
1474         __this_cpu_write(trace_cmdline_save, false);
1475
1476         trace_save_cmdline(tsk);
1477 }
1478
1479 void
1480 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1481                              int pc)
1482 {
1483         struct task_struct *tsk = current;
1484
1485         entry->preempt_count            = pc & 0xff;
1486         entry->pid                      = (tsk) ? tsk->pid : 0;
1487         entry->flags =
1488 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1489                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1490 #else
1491                 TRACE_FLAG_IRQS_NOSUPPORT |
1492 #endif
1493                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1494                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1495                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1496 }
1497 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1498
1499 struct ring_buffer_event *
1500 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1501                           int type,
1502                           unsigned long len,
1503                           unsigned long flags, int pc)
1504 {
1505         struct ring_buffer_event *event;
1506
1507         event = ring_buffer_lock_reserve(buffer, len);
1508         if (event != NULL) {
1509                 struct trace_entry *ent = ring_buffer_event_data(event);
1510
1511                 tracing_generic_entry_update(ent, flags, pc);
1512                 ent->type = type;
1513         }
1514
1515         return event;
1516 }
1517
1518 void
1519 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1520 {
1521         __this_cpu_write(trace_cmdline_save, true);
1522         ring_buffer_unlock_commit(buffer, event);
1523 }
1524
1525 static inline void
1526 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1527                              struct ring_buffer_event *event,
1528                              unsigned long flags, int pc)
1529 {
1530         __buffer_unlock_commit(buffer, event);
1531
1532         ftrace_trace_stack(buffer, flags, 6, pc);
1533         ftrace_trace_userstack(buffer, flags, pc);
1534 }
1535
1536 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1537                                 struct ring_buffer_event *event,
1538                                 unsigned long flags, int pc)
1539 {
1540         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1541 }
1542 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1543
1544 struct ring_buffer_event *
1545 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1546                           struct ftrace_event_file *ftrace_file,
1547                           int type, unsigned long len,
1548                           unsigned long flags, int pc)
1549 {
1550         *current_rb = ftrace_file->tr->trace_buffer.buffer;
1551         return trace_buffer_lock_reserve(*current_rb,
1552                                          type, len, flags, pc);
1553 }
1554 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1555
1556 struct ring_buffer_event *
1557 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1558                                   int type, unsigned long len,
1559                                   unsigned long flags, int pc)
1560 {
1561         *current_rb = global_trace.trace_buffer.buffer;
1562         return trace_buffer_lock_reserve(*current_rb,
1563                                          type, len, flags, pc);
1564 }
1565 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1566
1567 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1568                                         struct ring_buffer_event *event,
1569                                         unsigned long flags, int pc)
1570 {
1571         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1572 }
1573 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1574
1575 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1576                                      struct ring_buffer_event *event,
1577                                      unsigned long flags, int pc,
1578                                      struct pt_regs *regs)
1579 {
1580         __buffer_unlock_commit(buffer, event);
1581
1582         ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1583         ftrace_trace_userstack(buffer, flags, pc);
1584 }
1585 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1586
1587 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1588                                          struct ring_buffer_event *event)
1589 {
1590         ring_buffer_discard_commit(buffer, event);
1591 }
1592 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1593
1594 void
1595 trace_function(struct trace_array *tr,
1596                unsigned long ip, unsigned long parent_ip, unsigned long flags,
1597                int pc)
1598 {
1599         struct ftrace_event_call *call = &event_function;
1600         struct ring_buffer *buffer = tr->trace_buffer.buffer;
1601         struct ring_buffer_event *event;
1602         struct ftrace_entry *entry;
1603
1604         /* If we are reading the ring buffer, don't trace */
1605         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1606                 return;
1607
1608         event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1609                                           flags, pc);
1610         if (!event)
1611                 return;
1612         entry   = ring_buffer_event_data(event);
1613         entry->ip                       = ip;
1614         entry->parent_ip                = parent_ip;
1615
1616         if (!filter_check_discard(call, entry, buffer, event))
1617                 __buffer_unlock_commit(buffer, event);
1618 }
1619
1620 void
1621 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1622        unsigned long ip, unsigned long parent_ip, unsigned long flags,
1623        int pc)
1624 {
1625         if (likely(!atomic_read(&data->disabled)))
1626                 trace_function(tr, ip, parent_ip, flags, pc);
1627 }
1628
1629 #ifdef CONFIG_STACKTRACE
1630
1631 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1632 struct ftrace_stack {
1633         unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
1634 };
1635
1636 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1637 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1638
1639 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1640                                  unsigned long flags,
1641                                  int skip, int pc, struct pt_regs *regs)
1642 {
1643         struct ftrace_event_call *call = &event_kernel_stack;
1644         struct ring_buffer_event *event;
1645         struct stack_entry *entry;
1646         struct stack_trace trace;
1647         int use_stack;
1648         int size = FTRACE_STACK_ENTRIES;
1649
1650         trace.nr_entries        = 0;
1651         trace.skip              = skip;
1652
1653         /*
1654          * Since events can happen in NMIs there's no safe way to
1655          * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1656          * or NMI comes in, it will just have to use the default
1657          * FTRACE_STACK_SIZE.
1658          */
1659         preempt_disable_notrace();
1660
1661         use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1662         /*
1663          * We don't need any atomic variables, just a barrier.
1664          * If an interrupt comes in, we don't care, because it would
1665          * have exited and put the counter back to what we want.
1666          * We just need a barrier to keep gcc from moving things
1667          * around.
1668          */
1669         barrier();
1670         if (use_stack == 1) {
1671                 trace.entries           = &__get_cpu_var(ftrace_stack).calls[0];
1672                 trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
1673
1674                 if (regs)
1675                         save_stack_trace_regs(regs, &trace);
1676                 else
1677                         save_stack_trace(&trace);
1678
1679                 if (trace.nr_entries > size)
1680                         size = trace.nr_entries;
1681         } else
1682                 /* From now on, use_stack is a boolean */
1683                 use_stack = 0;
1684
1685         size *= sizeof(unsigned long);
1686
1687         event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1688                                           sizeof(*entry) + size, flags, pc);
1689         if (!event)
1690                 goto out;
1691         entry = ring_buffer_event_data(event);
1692
1693         memset(&entry->caller, 0, size);
1694
1695         if (use_stack)
1696                 memcpy(&entry->caller, trace.entries,
1697                        trace.nr_entries * sizeof(unsigned long));
1698         else {
1699                 trace.max_entries       = FTRACE_STACK_ENTRIES;
1700                 trace.entries           = entry->caller;
1701                 if (regs)
1702                         save_stack_trace_regs(regs, &trace);
1703                 else
1704                         save_stack_trace(&trace);
1705         }
1706
1707         entry->size = trace.nr_entries;
1708
1709         if (!filter_check_discard(call, entry, buffer, event))
1710                 __buffer_unlock_commit(buffer, event);
1711
1712  out:
1713         /* Again, don't let gcc optimize things here */
1714         barrier();
1715         __this_cpu_dec(ftrace_stack_reserve);
1716         preempt_enable_notrace();
1717
1718 }
1719
1720 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1721                              int skip, int pc, struct pt_regs *regs)
1722 {
1723         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1724                 return;
1725
1726         __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1727 }
1728
1729 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1730                         int skip, int pc)
1731 {
1732         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1733                 return;
1734
1735         __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1736 }
1737
1738 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1739                    int pc)
1740 {
1741         __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1742 }
1743
1744 /**
1745  * trace_dump_stack - record a stack back trace in the trace buffer
1746  * @skip: Number of functions to skip (helper handlers)
1747  */
1748 void trace_dump_stack(int skip)
1749 {
1750         unsigned long flags;
1751
1752         if (tracing_disabled || tracing_selftest_running)
1753                 return;
1754
1755         local_save_flags(flags);
1756
1757         /*
1758          * Skip 3 more, seems to get us at the caller of
1759          * this function.
1760          */
1761         skip += 3;
1762         __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1763                              flags, skip, preempt_count(), NULL);
1764 }
1765
1766 static DEFINE_PER_CPU(int, user_stack_count);
1767
1768 void
1769 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1770 {
1771         struct ftrace_event_call *call = &event_user_stack;
1772         struct ring_buffer_event *event;
1773         struct userstack_entry *entry;
1774         struct stack_trace trace;
1775
1776         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1777                 return;
1778
1779         /*
1780          * NMIs can not handle page faults, even with fix ups.
1781          * The save user stack can (and often does) fault.
1782          */
1783         if (unlikely(in_nmi()))
1784                 return;
1785
1786         /*
1787          * prevent recursion, since the user stack tracing may
1788          * trigger other kernel events.
1789          */
1790         preempt_disable();
1791         if (__this_cpu_read(user_stack_count))
1792                 goto out;
1793
1794         __this_cpu_inc(user_stack_count);
1795
1796         event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1797                                           sizeof(*entry), flags, pc);
1798         if (!event)
1799                 goto out_drop_count;
1800         entry   = ring_buffer_event_data(event);
1801
1802         entry->tgid             = current->tgid;
1803         memset(&entry->caller, 0, sizeof(entry->caller));
1804
1805         trace.nr_entries        = 0;
1806         trace.max_entries       = FTRACE_STACK_ENTRIES;
1807         trace.skip              = 0;
1808         trace.entries           = entry->caller;
1809
1810         save_stack_trace_user(&trace);
1811         if (!filter_check_discard(call, entry, buffer, event))
1812                 __buffer_unlock_commit(buffer, event);
1813
1814  out_drop_count:
1815         __this_cpu_dec(user_stack_count);
1816  out:
1817         preempt_enable();
1818 }
1819
1820 #ifdef UNUSED
1821 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1822 {
1823         ftrace_trace_userstack(tr, flags, preempt_count());
1824 }
1825 #endif /* UNUSED */
1826
1827 #endif /* CONFIG_STACKTRACE */
1828
1829 /* created for use with alloc_percpu */
1830 struct trace_buffer_struct {
1831         char buffer[TRACE_BUF_SIZE];
1832 };
1833
1834 static struct trace_buffer_struct *trace_percpu_buffer;
1835 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1836 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1837 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1838
1839 /*
1840  * The buffer used is dependent on the context. There is a per cpu
1841  * buffer for normal context, softirq contex, hard irq context and
1842  * for NMI context. Thise allows for lockless recording.
1843  *
1844  * Note, if the buffers failed to be allocated, then this returns NULL
1845  */
1846 static char *get_trace_buf(void)
1847 {
1848         struct trace_buffer_struct *percpu_buffer;
1849
1850         /*
1851          * If we have allocated per cpu buffers, then we do not
1852          * need to do any locking.
1853          */
1854         if (in_nmi())
1855                 percpu_buffer = trace_percpu_nmi_buffer;
1856         else if (in_irq())
1857                 percpu_buffer = trace_percpu_irq_buffer;
1858         else if (in_softirq())
1859                 percpu_buffer = trace_percpu_sirq_buffer;
1860         else
1861                 percpu_buffer = trace_percpu_buffer;
1862
1863         if (!percpu_buffer)
1864                 return NULL;
1865
1866         return this_cpu_ptr(&percpu_buffer->buffer[0]);
1867 }
1868
1869 static int alloc_percpu_trace_buffer(void)
1870 {
1871         struct trace_buffer_struct *buffers;
1872         struct trace_buffer_struct *sirq_buffers;
1873         struct trace_buffer_struct *irq_buffers;
1874         struct trace_buffer_struct *nmi_buffers;
1875
1876         buffers = alloc_percpu(struct trace_buffer_struct);
1877         if (!buffers)
1878                 goto err_warn;
1879
1880         sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1881         if (!sirq_buffers)
1882                 goto err_sirq;
1883
1884         irq_buffers = alloc_percpu(struct trace_buffer_struct);
1885         if (!irq_buffers)
1886                 goto err_irq;
1887
1888         nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1889         if (!nmi_buffers)
1890                 goto err_nmi;
1891
1892         trace_percpu_buffer = buffers;
1893         trace_percpu_sirq_buffer = sirq_buffers;
1894         trace_percpu_irq_buffer = irq_buffers;
1895         trace_percpu_nmi_buffer = nmi_buffers;
1896
1897         return 0;
1898
1899  err_nmi:
1900         free_percpu(irq_buffers);
1901  err_irq:
1902         free_percpu(sirq_buffers);
1903  err_sirq:
1904         free_percpu(buffers);
1905  err_warn:
1906         WARN(1, "Could not allocate percpu trace_printk buffer");
1907         return -ENOMEM;
1908 }
1909
1910 static int buffers_allocated;
1911
1912 void trace_printk_init_buffers(void)
1913 {
1914         if (buffers_allocated)
1915                 return;
1916
1917         if (alloc_percpu_trace_buffer())
1918                 return;
1919
1920         pr_info("ftrace: Allocated trace_printk buffers\n");
1921
1922         /* Expand the buffers to set size */
1923         tracing_update_buffers();
1924
1925         buffers_allocated = 1;
1926
1927         /*
1928          * trace_printk_init_buffers() can be called by modules.
1929          * If that happens, then we need to start cmdline recording
1930          * directly here. If the global_trace.buffer is already
1931          * allocated here, then this was called by module code.
1932          */
1933         if (global_trace.trace_buffer.buffer)
1934                 tracing_start_cmdline_record();
1935 }
1936
1937 void trace_printk_start_comm(void)
1938 {
1939         /* Start tracing comms if trace printk is set */
1940         if (!buffers_allocated)
1941                 return;
1942         tracing_start_cmdline_record();
1943 }
1944
1945 static void trace_printk_start_stop_comm(int enabled)
1946 {
1947         if (!buffers_allocated)
1948                 return;
1949
1950         if (enabled)
1951                 tracing_start_cmdline_record();
1952         else
1953                 tracing_stop_cmdline_record();
1954 }
1955
1956 /**
1957  * trace_vbprintk - write binary msg to tracing buffer
1958  *
1959  */
1960 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1961 {
1962         struct ftrace_event_call *call = &event_bprint;
1963         struct ring_buffer_event *event;
1964         struct ring_buffer *buffer;
1965         struct trace_array *tr = &global_trace;
1966         struct bprint_entry *entry;
1967         unsigned long flags;
1968         char *tbuffer;
1969         int len = 0, size, pc;
1970
1971         if (unlikely(tracing_selftest_running || tracing_disabled))
1972                 return 0;
1973
1974         /* Don't pollute graph traces with trace_vprintk internals */
1975         pause_graph_tracing();
1976
1977         pc = preempt_count();
1978         preempt_disable_notrace();
1979
1980         tbuffer = get_trace_buf();
1981         if (!tbuffer) {
1982                 len = 0;
1983                 goto out;
1984         }
1985
1986         len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1987
1988         if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1989                 goto out;
1990
1991         local_save_flags(flags);
1992         size = sizeof(*entry) + sizeof(u32) * len;
1993         buffer = tr->trace_buffer.buffer;
1994         event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1995                                           flags, pc);
1996         if (!event)
1997                 goto out;
1998         entry = ring_buffer_event_data(event);
1999         entry->ip                       = ip;
2000         entry->fmt                      = fmt;
2001
2002         memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2003         if (!filter_check_discard(call, entry, buffer, event)) {
2004                 __buffer_unlock_commit(buffer, event);
2005                 ftrace_trace_stack(buffer, flags, 6, pc);
2006         }
2007
2008 out:
2009         preempt_enable_notrace();
2010         unpause_graph_tracing();
2011
2012         return len;
2013 }
2014 EXPORT_SYMBOL_GPL(trace_vbprintk);
2015
2016 static int
2017 __trace_array_vprintk(struct ring_buffer *buffer,
2018                       unsigned long ip, const char *fmt, va_list args)
2019 {
2020         struct ftrace_event_call *call = &event_print;
2021         struct ring_buffer_event *event;
2022         int len = 0, size, pc;
2023         struct print_entry *entry;
2024         unsigned long flags;
2025         char *tbuffer;
2026
2027         if (tracing_disabled || tracing_selftest_running)
2028                 return 0;
2029
2030         /* Don't pollute graph traces with trace_vprintk internals */
2031         pause_graph_tracing();
2032
2033         pc = preempt_count();
2034         preempt_disable_notrace();
2035
2036
2037         tbuffer = get_trace_buf();
2038         if (!tbuffer) {
2039                 len = 0;
2040                 goto out;
2041         }
2042
2043         len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2044         if (len > TRACE_BUF_SIZE)
2045                 goto out;
2046
2047         local_save_flags(flags);
2048         size = sizeof(*entry) + len + 1;
2049         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2050                                           flags, pc);
2051         if (!event)
2052                 goto out;
2053         entry = ring_buffer_event_data(event);
2054         entry->ip = ip;
2055
2056         memcpy(&entry->buf, tbuffer, len);
2057         entry->buf[len] = '\0';
2058         if (!filter_check_discard(call, entry, buffer, event)) {
2059                 __buffer_unlock_commit(buffer, event);
2060                 ftrace_trace_stack(buffer, flags, 6, pc);
2061         }
2062  out:
2063         preempt_enable_notrace();
2064         unpause_graph_tracing();
2065
2066         return len;
2067 }
2068
2069 int trace_array_vprintk(struct trace_array *tr,
2070                         unsigned long ip, const char *fmt, va_list args)
2071 {
2072         return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2073 }
2074
2075 int trace_array_printk(struct trace_array *tr,
2076                        unsigned long ip, const char *fmt, ...)
2077 {
2078         int ret;
2079         va_list ap;
2080
2081         if (!(trace_flags & TRACE_ITER_PRINTK))
2082                 return 0;
2083
2084         va_start(ap, fmt);
2085         ret = trace_array_vprintk(tr, ip, fmt, ap);
2086         va_end(ap);
2087         return ret;
2088 }
2089
2090 int trace_array_printk_buf(struct ring_buffer *buffer,
2091                            unsigned long ip, const char *fmt, ...)
2092 {
2093         int ret;
2094         va_list ap;
2095
2096         if (!(trace_flags & TRACE_ITER_PRINTK))
2097                 return 0;
2098
2099         va_start(ap, fmt);
2100         ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2101         va_end(ap);
2102         return ret;
2103 }
2104
2105 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2106 {
2107         return trace_array_vprintk(&global_trace, ip, fmt, args);
2108 }
2109 EXPORT_SYMBOL_GPL(trace_vprintk);
2110
2111 static void trace_iterator_increment(struct trace_iterator *iter)
2112 {
2113         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2114
2115         iter->idx++;
2116         if (buf_iter)
2117                 ring_buffer_read(buf_iter, NULL);
2118 }
2119
2120 static struct trace_entry *
2121 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2122                 unsigned long *lost_events)
2123 {
2124         struct ring_buffer_event *event;
2125         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2126
2127         if (buf_iter)
2128                 event = ring_buffer_iter_peek(buf_iter, ts);
2129         else
2130                 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2131                                          lost_events);
2132
2133         if (event) {
2134                 iter->ent_size = ring_buffer_event_length(event);
2135                 return ring_buffer_event_data(event);
2136         }
2137         iter->ent_size = 0;
2138         return NULL;
2139 }
2140
2141 static struct trace_entry *
2142 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2143                   unsigned long *missing_events, u64 *ent_ts)
2144 {
2145         struct ring_buffer *buffer = iter->trace_buffer->buffer;
2146         struct trace_entry *ent, *next = NULL;
2147         unsigned long lost_events = 0, next_lost = 0;
2148         int cpu_file = iter->cpu_file;
2149         u64 next_ts = 0, ts;
2150         int next_cpu = -1;
2151         int next_size = 0;
2152         int cpu;
2153
2154         /*
2155          * If we are in a per_cpu trace file, don't bother by iterating over
2156          * all cpu and peek directly.
2157          */
2158         if (cpu_file > RING_BUFFER_ALL_CPUS) {
2159                 if (ring_buffer_empty_cpu(buffer, cpu_file))
2160                         return NULL;
2161                 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2162                 if (ent_cpu)
2163                         *ent_cpu = cpu_file;
2164
2165                 return ent;
2166         }
2167
2168         for_each_tracing_cpu(cpu) {
2169
2170                 if (ring_buffer_empty_cpu(buffer, cpu))
2171                         continue;
2172
2173                 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2174
2175                 /*
2176                  * Pick the entry with the smallest timestamp:
2177                  */
2178                 if (ent && (!next || ts < next_ts)) {
2179                         next = ent;
2180                         next_cpu = cpu;
2181                         next_ts = ts;
2182                         next_lost = lost_events;
2183                         next_size = iter->ent_size;
2184                 }
2185         }
2186
2187         iter->ent_size = next_size;
2188
2189         if (ent_cpu)
2190                 *ent_cpu = next_cpu;
2191
2192         if (ent_ts)
2193                 *ent_ts = next_ts;
2194
2195         if (missing_events)
2196                 *missing_events = next_lost;
2197
2198         return next;
2199 }
2200
2201 /* Find the next real entry, without updating the iterator itself */
2202 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2203                                           int *ent_cpu, u64 *ent_ts)
2204 {
2205         return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2206 }
2207
2208 /* Find the next real entry, and increment the iterator to the next entry */
2209 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2210 {
2211         iter->ent = __find_next_entry(iter, &iter->cpu,
2212                                       &iter->lost_events, &iter->ts);
2213
2214         if (iter->ent)
2215                 trace_iterator_increment(iter);
2216
2217         return iter->ent ? iter : NULL;
2218 }
2219
2220 static void trace_consume(struct trace_iterator *iter)
2221 {
2222         ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2223                             &iter->lost_events);
2224 }
2225
2226 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2227 {
2228         struct trace_iterator *iter = m->private;
2229         int i = (int)*pos;
2230         void *ent;
2231
2232         WARN_ON_ONCE(iter->leftover);
2233
2234         (*pos)++;
2235
2236         /* can't go backwards */
2237         if (iter->idx > i)
2238                 return NULL;
2239
2240         if (iter->idx < 0)
2241                 ent = trace_find_next_entry_inc(iter);
2242         else
2243                 ent = iter;
2244
2245         while (ent && iter->idx < i)
2246                 ent = trace_find_next_entry_inc(iter);
2247
2248         iter->pos = *pos;
2249
2250         return ent;
2251 }
2252
2253 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2254 {
2255         struct ring_buffer_event *event;
2256         struct ring_buffer_iter *buf_iter;
2257         unsigned long entries = 0;
2258         u64 ts;
2259
2260         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2261
2262         buf_iter = trace_buffer_iter(iter, cpu);
2263         if (!buf_iter)
2264                 return;
2265
2266         ring_buffer_iter_reset(buf_iter);
2267
2268         /*
2269          * We could have the case with the max latency tracers
2270          * that a reset never took place on a cpu. This is evident
2271          * by the timestamp being before the start of the buffer.
2272          */
2273         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2274                 if (ts >= iter->trace_buffer->time_start)
2275                         break;
2276                 entries++;
2277                 ring_buffer_read(buf_iter, NULL);
2278         }
2279
2280         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2281 }
2282
2283 /*
2284  * The current tracer is copied to avoid a global locking
2285  * all around.
2286  */
2287 static void *s_start(struct seq_file *m, loff_t *pos)
2288 {
2289         struct trace_iterator *iter = m->private;
2290         struct trace_array *tr = iter->tr;
2291         int cpu_file = iter->cpu_file;
2292         void *p = NULL;
2293         loff_t l = 0;
2294         int cpu;
2295
2296         /*
2297          * copy the tracer to avoid using a global lock all around.
2298          * iter->trace is a copy of current_trace, the pointer to the
2299          * name may be used instead of a strcmp(), as iter->trace->name
2300          * will point to the same string as current_trace->name.
2301          */
2302         mutex_lock(&trace_types_lock);
2303         if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2304                 *iter->trace = *tr->current_trace;
2305         mutex_unlock(&trace_types_lock);
2306
2307 #ifdef CONFIG_TRACER_MAX_TRACE
2308         if (iter->snapshot && iter->trace->use_max_tr)
2309                 return ERR_PTR(-EBUSY);
2310 #endif
2311
2312         if (!iter->snapshot)
2313                 atomic_inc(&trace_record_cmdline_disabled);
2314
2315         if (*pos != iter->pos) {
2316                 iter->ent = NULL;
2317                 iter->cpu = 0;
2318                 iter->idx = -1;
2319
2320                 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2321                         for_each_tracing_cpu(cpu)
2322                                 tracing_iter_reset(iter, cpu);
2323                 } else
2324                         tracing_iter_reset(iter, cpu_file);
2325
2326                 iter->leftover = 0;
2327                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2328                         ;
2329
2330         } else {
2331                 /*
2332                  * If we overflowed the seq_file before, then we want
2333                  * to just reuse the trace_seq buffer again.
2334                  */
2335                 if (iter->leftover)
2336                         p = iter;
2337                 else {
2338                         l = *pos - 1;
2339                         p = s_next(m, p, &l);
2340                 }
2341         }
2342
2343         trace_event_read_lock();
2344         trace_access_lock(cpu_file);
2345         return p;
2346 }
2347
2348 static void s_stop(struct seq_file *m, void *p)
2349 {
2350         struct trace_iterator *iter = m->private;
2351
2352 #ifdef CONFIG_TRACER_MAX_TRACE
2353         if (iter->snapshot && iter->trace->use_max_tr)
2354                 return;
2355 #endif
2356
2357         if (!iter->snapshot)
2358                 atomic_dec(&trace_record_cmdline_disabled);
2359
2360         trace_access_unlock(iter->cpu_file);
2361         trace_event_read_unlock();
2362 }
2363
2364 static void
2365 get_total_entries(struct trace_buffer *buf,
2366                   unsigned long *total, unsigned long *entries)
2367 {
2368         unsigned long count;
2369         int cpu;
2370
2371         *total = 0;
2372         *entries = 0;
2373
2374         for_each_tracing_cpu(cpu) {
2375                 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2376                 /*
2377                  * If this buffer has skipped entries, then we hold all
2378                  * entries for the trace and we need to ignore the
2379                  * ones before the time stamp.
2380                  */
2381                 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2382                         count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2383                         /* total is the same as the entries */
2384                         *total += count;
2385                 } else
2386                         *total += count +
2387                                 ring_buffer_overrun_cpu(buf->buffer, cpu);
2388                 *entries += count;
2389         }
2390 }
2391
2392 static void print_lat_help_header(struct seq_file *m)
2393 {
2394         seq_puts(m, "#                  _------=> CPU#            \n");
2395         seq_puts(m, "#                 / _-----=> irqs-off        \n");
2396         seq_puts(m, "#                | / _----=> need-resched    \n");
2397         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
2398         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
2399         seq_puts(m, "#                |||| /     delay             \n");
2400         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
2401         seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
2402 }
2403
2404 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2405 {
2406         unsigned long total;
2407         unsigned long entries;
2408
2409         get_total_entries(buf, &total, &entries);
2410         seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
2411                    entries, total, num_online_cpus());
2412         seq_puts(m, "#\n");
2413 }
2414
2415 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2416 {
2417         print_event_info(buf, m);
2418         seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2419         seq_puts(m, "#              | |       |          |         |\n");
2420 }
2421
2422 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2423 {
2424         print_event_info(buf, m);
2425         seq_puts(m, "#                              _-----=> irqs-off\n");
2426         seq_puts(m, "#                             / _----=> need-resched\n");
2427         seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
2428         seq_puts(m, "#                            || / _--=> preempt-depth\n");
2429         seq_puts(m, "#                            ||| /     delay\n");
2430         seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
2431         seq_puts(m, "#              | |       |   ||||       |         |\n");
2432 }
2433
2434 void
2435 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2436 {
2437         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2438         struct trace_buffer *buf = iter->trace_buffer;
2439         struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2440         struct tracer *type = iter->trace;
2441         unsigned long entries;
2442         unsigned long total;
2443         const char *name = "preemption";
2444
2445         name = type->name;
2446
2447         get_total_entries(buf, &total, &entries);
2448
2449         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2450                    name, UTS_RELEASE);
2451         seq_puts(m, "# -----------------------------------"
2452                  "---------------------------------\n");
2453         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2454                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2455                    nsecs_to_usecs(data->saved_latency),
2456                    entries,
2457                    total,
2458                    buf->cpu,
2459 #if defined(CONFIG_PREEMPT_NONE)
2460                    "server",
2461 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2462                    "desktop",
2463 #elif defined(CONFIG_PREEMPT)
2464                    "preempt",
2465 #else
2466                    "unknown",
2467 #endif
2468                    /* These are reserved for later use */
2469                    0, 0, 0, 0);
2470 #ifdef CONFIG_SMP
2471         seq_printf(m, " #P:%d)\n", num_online_cpus());
2472 #else
2473         seq_puts(m, ")\n");
2474 #endif
2475         seq_puts(m, "#    -----------------\n");
2476         seq_printf(m, "#    | task: %.16s-%d "
2477                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2478                    data->comm, data->pid,
2479                    from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2480                    data->policy, data->rt_priority);
2481         seq_puts(m, "#    -----------------\n");
2482
2483         if (data->critical_start) {
2484                 seq_puts(m, "#  => started at: ");
2485                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2486                 trace_print_seq(m, &iter->seq);
2487                 seq_puts(m, "\n#  => ended at:   ");
2488                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2489                 trace_print_seq(m, &iter->seq);
2490                 seq_puts(m, "\n#\n");
2491         }
2492
2493         seq_puts(m, "#\n");
2494 }
2495
2496 static void test_cpu_buff_start(struct trace_iterator *iter)
2497 {
2498         struct trace_seq *s = &iter->seq;
2499
2500         if (!(trace_flags & TRACE_ITER_ANNOTATE))
2501                 return;
2502
2503         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2504                 return;
2505
2506         if (cpumask_test_cpu(iter->cpu, iter->started))
2507                 return;
2508
2509         if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2510                 return;
2511
2512         cpumask_set_cpu(iter->cpu, iter->started);
2513
2514         /* Don't print started cpu buffer for the first entry of the trace */
2515         if (iter->idx > 1)
2516                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2517                                 iter->cpu);
2518 }
2519
2520 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2521 {
2522         struct trace_seq *s = &iter->seq;
2523         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2524         struct trace_entry *entry;
2525         struct trace_event *event;
2526
2527         entry = iter->ent;
2528
2529         test_cpu_buff_start(iter);
2530
2531         event = ftrace_find_event(entry->type);
2532
2533         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2534                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2535                         if (!trace_print_lat_context(iter))
2536                                 goto partial;
2537                 } else {
2538                         if (!trace_print_context(iter))
2539                                 goto partial;
2540                 }
2541         }
2542
2543         if (event)
2544                 return event->funcs->trace(iter, sym_flags, event);
2545
2546         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2547                 goto partial;
2548
2549         return TRACE_TYPE_HANDLED;
2550 partial:
2551         return TRACE_TYPE_PARTIAL_LINE;
2552 }
2553
2554 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2555 {
2556         struct trace_seq *s = &iter->seq;
2557         struct trace_entry *entry;
2558         struct trace_event *event;
2559
2560         entry = iter->ent;
2561
2562         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2563                 if (!trace_seq_printf(s, "%d %d %llu ",
2564                                       entry->pid, iter->cpu, iter->ts))
2565                         goto partial;
2566         }
2567
2568         event = ftrace_find_event(entry->type);
2569         if (event)
2570                 return event->funcs->raw(iter, 0, event);
2571
2572         if (!trace_seq_printf(s, "%d ?\n", entry->type))
2573                 goto partial;
2574
2575         return TRACE_TYPE_HANDLED;
2576 partial:
2577         return TRACE_TYPE_PARTIAL_LINE;
2578 }
2579
2580 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2581 {
2582         struct trace_seq *s = &iter->seq;
2583         unsigned char newline = '\n';
2584         struct trace_entry *entry;
2585         struct trace_event *event;
2586
2587         entry = iter->ent;
2588
2589         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2590                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2591                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2592                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2593         }
2594
2595         event = ftrace_find_event(entry->type);
2596         if (event) {
2597                 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2598                 if (ret != TRACE_TYPE_HANDLED)
2599                         return ret;
2600         }
2601
2602         SEQ_PUT_FIELD_RET(s, newline);
2603
2604         return TRACE_TYPE_HANDLED;
2605 }
2606
2607 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2608 {
2609         struct trace_seq *s = &iter->seq;
2610         struct trace_entry *entry;
2611         struct trace_event *event;
2612
2613         entry = iter->ent;
2614
2615         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2616                 SEQ_PUT_FIELD_RET(s, entry->pid);
2617                 SEQ_PUT_FIELD_RET(s, iter->cpu);
2618                 SEQ_PUT_FIELD_RET(s, iter->ts);
2619         }
2620
2621         event = ftrace_find_event(entry->type);
2622         return event ? event->funcs->binary(iter, 0, event) :
2623                 TRACE_TYPE_HANDLED;
2624 }
2625
2626 int trace_empty(struct trace_iterator *iter)
2627 {
2628         struct ring_buffer_iter *buf_iter;
2629         int cpu;
2630
2631         /* If we are looking at one CPU buffer, only check that one */
2632         if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2633                 cpu = iter->cpu_file;
2634                 buf_iter = trace_buffer_iter(iter, cpu);
2635                 if (buf_iter) {
2636                         if (!ring_buffer_iter_empty(buf_iter))
2637                                 return 0;
2638                 } else {
2639                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2640                                 return 0;
2641                 }
2642                 return 1;
2643         }
2644
2645         for_each_tracing_cpu(cpu) {
2646                 buf_iter = trace_buffer_iter(iter, cpu);
2647                 if (buf_iter) {
2648                         if (!ring_buffer_iter_empty(buf_iter))
2649                                 return 0;
2650                 } else {
2651                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2652                                 return 0;
2653                 }
2654         }
2655
2656         return 1;
2657 }
2658
2659 /*  Called with trace_event_read_lock() held. */
2660 enum print_line_t print_trace_line(struct trace_iterator *iter)
2661 {
2662         enum print_line_t ret;
2663
2664         if (iter->lost_events &&
2665             !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2666                                  iter->cpu, iter->lost_events))
2667                 return TRACE_TYPE_PARTIAL_LINE;
2668
2669         if (iter->trace && iter->trace->print_line) {
2670                 ret = iter->trace->print_line(iter);
2671                 if (ret != TRACE_TYPE_UNHANDLED)
2672                         return ret;
2673         }
2674
2675         if (iter->ent->type == TRACE_BPUTS &&
2676                         trace_flags & TRACE_ITER_PRINTK &&
2677                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2678                 return trace_print_bputs_msg_only(iter);
2679
2680         if (iter->ent->type == TRACE_BPRINT &&
2681                         trace_flags & TRACE_ITER_PRINTK &&
2682                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2683                 return trace_print_bprintk_msg_only(iter);
2684
2685         if (iter->ent->type == TRACE_PRINT &&
2686                         trace_flags & TRACE_ITER_PRINTK &&
2687                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2688                 return trace_print_printk_msg_only(iter);
2689
2690         if (trace_flags & TRACE_ITER_BIN)
2691                 return print_bin_fmt(iter);
2692
2693         if (trace_flags & TRACE_ITER_HEX)
2694                 return print_hex_fmt(iter);
2695
2696         if (trace_flags & TRACE_ITER_RAW)
2697                 return print_raw_fmt(iter);
2698
2699         return print_trace_fmt(iter);
2700 }
2701
2702 void trace_latency_header(struct seq_file *m)
2703 {
2704         struct trace_iterator *iter = m->private;
2705
2706         /* print nothing if the buffers are empty */
2707         if (trace_empty(iter))
2708                 return;
2709
2710         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2711                 print_trace_header(m, iter);
2712
2713         if (!(trace_flags & TRACE_ITER_VERBOSE))
2714                 print_lat_help_header(m);
2715 }
2716
2717 void trace_default_header(struct seq_file *m)
2718 {
2719         struct trace_iterator *iter = m->private;
2720
2721         if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2722                 return;
2723
2724         if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2725                 /* print nothing if the buffers are empty */
2726                 if (trace_empty(iter))
2727                         return;
2728                 print_trace_header(m, iter);
2729                 if (!(trace_flags & TRACE_ITER_VERBOSE))
2730                         print_lat_help_header(m);
2731         } else {
2732                 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2733                         if (trace_flags & TRACE_ITER_IRQ_INFO)
2734                                 print_func_help_header_irq(iter->trace_buffer, m);
2735                         else
2736                                 print_func_help_header(iter->trace_buffer, m);
2737                 }
2738         }
2739 }
2740
2741 static void test_ftrace_alive(struct seq_file *m)
2742 {
2743         if (!ftrace_is_dead())
2744                 return;
2745         seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2746         seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
2747 }
2748
2749 #ifdef CONFIG_TRACER_MAX_TRACE
2750 static void show_snapshot_main_help(struct seq_file *m)
2751 {
2752         seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2753         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2754         seq_printf(m, "#                      Takes a snapshot of the main buffer.\n");
2755         seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2756         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2757         seq_printf(m, "#                       is not a '0' or '1')\n");
2758 }
2759
2760 static void show_snapshot_percpu_help(struct seq_file *m)
2761 {
2762         seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2763 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2764         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2765         seq_printf(m, "#                      Takes a snapshot of the main buffer for this cpu.\n");
2766 #else
2767         seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2768         seq_printf(m, "#                     Must use main snapshot file to allocate.\n");
2769 #endif
2770         seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2771         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2772         seq_printf(m, "#                       is not a '0' or '1')\n");
2773 }
2774
2775 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2776 {
2777         if (iter->tr->allocated_snapshot)
2778                 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2779         else
2780                 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2781
2782         seq_printf(m, "# Snapshot commands:\n");
2783         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2784                 show_snapshot_main_help(m);
2785         else
2786                 show_snapshot_percpu_help(m);
2787 }
2788 #else
2789 /* Should never be called */
2790 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2791 #endif
2792
2793 static int s_show(struct seq_file *m, void *v)
2794 {
2795         struct trace_iterator *iter = v;
2796         int ret;
2797
2798         if (iter->ent == NULL) {
2799                 if (iter->tr) {
2800                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
2801                         seq_puts(m, "#\n");
2802                         test_ftrace_alive(m);
2803                 }
2804                 if (iter->snapshot && trace_empty(iter))
2805                         print_snapshot_help(m, iter);
2806                 else if (iter->trace && iter->trace->print_header)
2807                         iter->trace->print_header(m);
2808                 else
2809                         trace_default_header(m);
2810
2811         } else if (iter->leftover) {
2812                 /*
2813                  * If we filled the seq_file buffer earlier, we
2814                  * want to just show it now.
2815                  */
2816                 ret = trace_print_seq(m, &iter->seq);
2817
2818                 /* ret should this time be zero, but you never know */
2819                 iter->leftover = ret;
2820
2821         } else {
2822                 print_trace_line(iter);
2823                 ret = trace_print_seq(m, &iter->seq);
2824                 /*
2825                  * If we overflow the seq_file buffer, then it will
2826                  * ask us for this data again at start up.
2827                  * Use that instead.
2828                  *  ret is 0 if seq_file write succeeded.
2829                  *        -1 otherwise.
2830                  */
2831                 iter->leftover = ret;
2832         }
2833
2834         return 0;
2835 }
2836
2837 /*
2838  * Should be used after trace_array_get(), trace_types_lock
2839  * ensures that i_cdev was already initialized.
2840  */
2841 static inline int tracing_get_cpu(struct inode *inode)
2842 {
2843         if (inode->i_cdev) /* See trace_create_cpu_file() */
2844                 return (long)inode->i_cdev - 1;
2845         return RING_BUFFER_ALL_CPUS;
2846 }
2847
2848 static const struct seq_operations tracer_seq_ops = {
2849         .start          = s_start,
2850         .next           = s_next,
2851         .stop           = s_stop,
2852         .show           = s_show,
2853 };
2854
2855 static struct trace_iterator *
2856 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2857 {
2858         struct trace_array *tr = inode->i_private;
2859         struct trace_iterator *iter;
2860         int cpu;
2861
2862         if (tracing_disabled)
2863                 return ERR_PTR(-ENODEV);
2864
2865         iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2866         if (!iter)
2867                 return ERR_PTR(-ENOMEM);
2868
2869         iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2870                                     GFP_KERNEL);
2871         if (!iter->buffer_iter)
2872                 goto release;
2873
2874         /*
2875          * We make a copy of the current tracer to avoid concurrent
2876          * changes on it while we are reading.
2877          */
2878         mutex_lock(&trace_types_lock);
2879         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2880         if (!iter->trace)
2881                 goto fail;
2882
2883         *iter->trace = *tr->current_trace;
2884
2885         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2886                 goto fail;
2887
2888         iter->tr = tr;
2889
2890 #ifdef CONFIG_TRACER_MAX_TRACE
2891         /* Currently only the top directory has a snapshot */
2892         if (tr->current_trace->print_max || snapshot)
2893                 iter->trace_buffer = &tr->max_buffer;
2894         else
2895 #endif
2896                 iter->trace_buffer = &tr->trace_buffer;
2897         iter->snapshot = snapshot;
2898         iter->pos = -1;
2899         iter->cpu_file = tracing_get_cpu(inode);
2900         mutex_init(&iter->mutex);
2901
2902         /* Notify the tracer early; before we stop tracing. */
2903         if (iter->trace && iter->trace->open)
2904                 iter->trace->open(iter);
2905
2906         /* Annotate start of buffers if we had overruns */
2907         if (ring_buffer_overruns(iter->trace_buffer->buffer))
2908                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2909
2910         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2911         if (trace_clocks[tr->clock_id].in_ns)
2912                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2913
2914         /* stop the trace while dumping if we are not opening "snapshot" */
2915         if (!iter->snapshot)
2916                 tracing_stop_tr(tr);
2917
2918         if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2919                 for_each_tracing_cpu(cpu) {
2920                         iter->buffer_iter[cpu] =
2921                                 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2922                 }
2923                 ring_buffer_read_prepare_sync();
2924                 for_each_tracing_cpu(cpu) {
2925                         ring_buffer_read_start(iter->buffer_iter[cpu]);
2926                         tracing_iter_reset(iter, cpu);
2927                 }
2928         } else {
2929                 cpu = iter->cpu_file;
2930                 iter->buffer_iter[cpu] =
2931                         ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2932                 ring_buffer_read_prepare_sync();
2933                 ring_buffer_read_start(iter->buffer_iter[cpu]);
2934                 tracing_iter_reset(iter, cpu);
2935         }
2936
2937         mutex_unlock(&trace_types_lock);
2938
2939         return iter;
2940
2941  fail:
2942         mutex_unlock(&trace_types_lock);
2943         kfree(iter->trace);
2944         kfree(iter->buffer_iter);
2945 release:
2946         seq_release_private(inode, file);
2947         return ERR_PTR(-ENOMEM);
2948 }
2949
2950 int tracing_open_generic(struct inode *inode, struct file *filp)
2951 {
2952         if (tracing_disabled)
2953                 return -ENODEV;
2954
2955         filp->private_data = inode->i_private;
2956         return 0;
2957 }
2958
2959 /*
2960  * Open and update trace_array ref count.
2961  * Must have the current trace_array passed to it.
2962  */
2963 int tracing_open_generic_tr(struct inode *inode, struct file *filp)
2964 {
2965         struct trace_array *tr = inode->i_private;
2966
2967         if (tracing_disabled)
2968                 return -ENODEV;
2969
2970         if (trace_array_get(tr) < 0)
2971                 return -ENODEV;
2972
2973         filp->private_data = inode->i_private;
2974
2975         return 0;
2976 }
2977
2978 static int tracing_release(struct inode *inode, struct file *file)
2979 {
2980         struct trace_array *tr = inode->i_private;
2981         struct seq_file *m = file->private_data;
2982         struct trace_iterator *iter;
2983         int cpu;
2984
2985         if (!(file->f_mode & FMODE_READ)) {
2986                 trace_array_put(tr);
2987                 return 0;
2988         }
2989
2990         /* Writes do not use seq_file */
2991         iter = m->private;
2992         mutex_lock(&trace_types_lock);
2993
2994         for_each_tracing_cpu(cpu) {
2995                 if (iter->buffer_iter[cpu])
2996                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2997         }
2998
2999         if (iter->trace && iter->trace->close)
3000                 iter->trace->close(iter);
3001
3002         if (!iter->snapshot)
3003                 /* reenable tracing if it was previously enabled */
3004                 tracing_start_tr(tr);
3005
3006         __trace_array_put(tr);
3007
3008         mutex_unlock(&trace_types_lock);
3009
3010         mutex_destroy(&iter->mutex);
3011         free_cpumask_var(iter->started);
3012         kfree(iter->trace);
3013         kfree(iter->buffer_iter);
3014         seq_release_private(inode, file);
3015
3016         return 0;
3017 }
3018
3019 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3020 {
3021         struct trace_array *tr = inode->i_private;
3022
3023         trace_array_put(tr);
3024         return 0;
3025 }
3026
3027 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3028 {
3029         struct trace_array *tr = inode->i_private;
3030
3031         trace_array_put(tr);
3032
3033         return single_release(inode, file);
3034 }
3035
3036 static int tracing_open(struct inode *inode, struct file *file)
3037 {
3038         struct trace_array *tr = inode->i_private;
3039         struct trace_iterator *iter;
3040         int ret = 0;
3041
3042         if (trace_array_get(tr) < 0)
3043                 return -ENODEV;
3044
3045         /* If this file was open for write, then erase contents */
3046         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3047                 int cpu = tracing_get_cpu(inode);
3048
3049                 if (cpu == RING_BUFFER_ALL_CPUS)
3050                         tracing_reset_online_cpus(&tr->trace_buffer);
3051                 else
3052                         tracing_reset(&tr->trace_buffer, cpu);
3053         }
3054
3055         if (file->f_mode & FMODE_READ) {
3056                 iter = __tracing_open(inode, file, false);
3057                 if (IS_ERR(iter))
3058                         ret = PTR_ERR(iter);
3059                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3060                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
3061         }
3062
3063         if (ret < 0)
3064                 trace_array_put(tr);
3065
3066         return ret;
3067 }
3068
3069 static void *
3070 t_next(struct seq_file *m, void *v, loff_t *pos)
3071 {
3072         struct tracer *t = v;
3073
3074         (*pos)++;
3075
3076         if (t)
3077                 t = t->next;
3078
3079         return t;
3080 }
3081
3082 static void *t_start(struct seq_file *m, loff_t *pos)
3083 {
3084         struct tracer *t;
3085         loff_t l = 0;
3086
3087         mutex_lock(&trace_types_lock);
3088         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
3089                 ;
3090
3091         return t;
3092 }
3093
3094 static void t_stop(struct seq_file *m, void *p)
3095 {
3096         mutex_unlock(&trace_types_lock);
3097 }
3098
3099 static int t_show(struct seq_file *m, void *v)
3100 {
3101         struct tracer *t = v;
3102
3103         if (!t)
3104                 return 0;
3105
3106         seq_printf(m, "%s", t->name);
3107         if (t->next)
3108                 seq_putc(m, ' ');
3109         else
3110                 seq_putc(m, '\n');
3111
3112         return 0;
3113 }
3114
3115 static const struct seq_operations show_traces_seq_ops = {
3116         .start          = t_start,
3117         .next           = t_next,
3118         .stop           = t_stop,
3119         .show           = t_show,
3120 };
3121
3122 static int show_traces_open(struct inode *inode, struct file *file)
3123 {
3124         if (tracing_disabled)
3125                 return -ENODEV;
3126
3127         return seq_open(file, &show_traces_seq_ops);
3128 }
3129
3130 static ssize_t
3131 tracing_write_stub(struct file *filp, const char __user *ubuf,
3132                    size_t count, loff_t *ppos)
3133 {
3134         return count;
3135 }
3136
3137 static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
3138 {
3139         if (file->f_mode & FMODE_READ)
3140                 return seq_lseek(file, offset, origin);
3141         else
3142                 return 0;
3143 }
3144
3145 static const struct file_operations tracing_fops = {
3146         .open           = tracing_open,
3147         .read           = seq_read,
3148         .write          = tracing_write_stub,
3149         .llseek         = tracing_seek,
3150         .release        = tracing_release,
3151 };
3152
3153 static const struct file_operations show_traces_fops = {
3154         .open           = show_traces_open,
3155         .read           = seq_read,
3156         .release        = seq_release,
3157         .llseek         = seq_lseek,
3158 };
3159
3160 /*
3161  * Only trace on a CPU if the bitmask is set:
3162  */
3163 static cpumask_var_t tracing_cpumask;
3164
3165 /*
3166  * The tracer itself will not take this lock, but still we want
3167  * to provide a consistent cpumask to user-space:
3168  */
3169 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3170
3171 /*
3172  * Temporary storage for the character representation of the
3173  * CPU bitmask (and one more byte for the newline):
3174  */
3175 static char mask_str[NR_CPUS + 1];
3176
3177 static ssize_t
3178 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3179                      size_t count, loff_t *ppos)
3180 {
3181         int len;
3182
3183         mutex_lock(&tracing_cpumask_update_lock);
3184
3185         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
3186         if (count - len < 2) {
3187                 count = -EINVAL;
3188                 goto out_err;
3189         }
3190         len += sprintf(mask_str + len, "\n");
3191         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3192
3193 out_err:
3194         mutex_unlock(&tracing_cpumask_update_lock);
3195
3196         return count;
3197 }
3198
3199 static ssize_t
3200 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3201                       size_t count, loff_t *ppos)
3202 {
3203         struct trace_array *tr = filp->private_data;
3204         cpumask_var_t tracing_cpumask_new;
3205         int err, cpu;
3206
3207         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3208                 return -ENOMEM;
3209
3210         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3211         if (err)
3212                 goto err_unlock;
3213
3214         mutex_lock(&tracing_cpumask_update_lock);
3215
3216         local_irq_disable();
3217         arch_spin_lock(&ftrace_max_lock);
3218         for_each_tracing_cpu(cpu) {
3219                 /*
3220                  * Increase/decrease the disabled counter if we are
3221                  * about to flip a bit in the cpumask:
3222                  */
3223                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
3224                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3225                         atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3226                         ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3227                 }
3228                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
3229                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3230                         atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3231                         ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3232                 }
3233         }
3234         arch_spin_unlock(&ftrace_max_lock);
3235         local_irq_enable();
3236
3237         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
3238
3239         mutex_unlock(&tracing_cpumask_update_lock);
3240         free_cpumask_var(tracing_cpumask_new);
3241
3242         return count;
3243
3244 err_unlock:
3245         free_cpumask_var(tracing_cpumask_new);
3246
3247         return err;
3248 }
3249
3250 static const struct file_operations tracing_cpumask_fops = {
3251         .open           = tracing_open_generic,
3252         .read           = tracing_cpumask_read,
3253         .write          = tracing_cpumask_write,
3254         .llseek         = generic_file_llseek,
3255 };
3256
3257 static int tracing_trace_options_show(struct seq_file *m, void *v)
3258 {
3259         struct tracer_opt *trace_opts;
3260         struct trace_array *tr = m->private;
3261         u32 tracer_flags;
3262         int i;
3263
3264         mutex_lock(&trace_types_lock);
3265         tracer_flags = tr->current_trace->flags->val;
3266         trace_opts = tr->current_trace->flags->opts;
3267
3268         for (i = 0; trace_options[i]; i++) {
3269                 if (trace_flags & (1 << i))
3270                         seq_printf(m, "%s\n", trace_options[i]);
3271                 else
3272                         seq_printf(m, "no%s\n", trace_options[i]);
3273         }
3274
3275         for (i = 0; trace_opts[i].name; i++) {
3276                 if (tracer_flags & trace_opts[i].bit)
3277                         seq_printf(m, "%s\n", trace_opts[i].name);
3278                 else
3279                         seq_printf(m, "no%s\n", trace_opts[i].name);
3280         }
3281         mutex_unlock(&trace_types_lock);
3282
3283         return 0;
3284 }
3285
3286 static int __set_tracer_option(struct tracer *trace,
3287                                struct tracer_flags *tracer_flags,
3288                                struct tracer_opt *opts, int neg)
3289 {
3290         int ret;
3291
3292         ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3293         if (ret)
3294                 return ret;
3295
3296         if (neg)
3297                 tracer_flags->val &= ~opts->bit;
3298         else
3299                 tracer_flags->val |= opts->bit;
3300         return 0;
3301 }
3302
3303 /* Try to assign a tracer specific option */
3304 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3305 {
3306         struct tracer_flags *tracer_flags = trace->flags;
3307         struct tracer_opt *opts = NULL;
3308         int i;
3309
3310         for (i = 0; tracer_flags->opts[i].name; i++) {
3311                 opts = &tracer_flags->opts[i];
3312
3313                 if (strcmp(cmp, opts->name) == 0)
3314                         return __set_tracer_option(trace, trace->flags,
3315                                                    opts, neg);
3316         }
3317
3318         return -EINVAL;
3319 }
3320
3321 /* Some tracers require overwrite to stay enabled */
3322 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3323 {
3324         if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3325                 return -1;
3326
3327         return 0;
3328 }
3329
3330 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3331 {
3332         /* do nothing if flag is already set */
3333         if (!!(trace_flags & mask) == !!enabled)
3334                 return 0;
3335
3336         /* Give the tracer a chance to approve the change */
3337         if (tr->current_trace->flag_changed)
3338                 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
3339                         return -EINVAL;
3340
3341         if (enabled)
3342                 trace_flags |= mask;
3343         else
3344                 trace_flags &= ~mask;
3345
3346         if (mask == TRACE_ITER_RECORD_CMD)
3347                 trace_event_enable_cmd_record(enabled);
3348
3349         if (mask == TRACE_ITER_OVERWRITE) {
3350                 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3351 #ifdef CONFIG_TRACER_MAX_TRACE
3352                 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3353 #endif
3354         }
3355
3356         if (mask == TRACE_ITER_PRINTK)
3357                 trace_printk_start_stop_comm(enabled);
3358
3359         return 0;
3360 }
3361
3362 static int trace_set_options(struct trace_array *tr, char *option)
3363 {
3364         char *cmp;
3365         int neg = 0;
3366         int ret = -ENODEV;
3367         int i;
3368
3369         cmp = strstrip(option);
3370
3371         if (strncmp(cmp, "no", 2) == 0) {
3372                 neg = 1;
3373                 cmp += 2;
3374         }
3375
3376         mutex_lock(&trace_types_lock);
3377
3378         for (i = 0; trace_options[i]; i++) {
3379                 if (strcmp(cmp, trace_options[i]) == 0) {
3380                         ret = set_tracer_flag(tr, 1 << i, !neg);
3381                         break;
3382                 }
3383         }
3384
3385         /* If no option could be set, test the specific tracer options */
3386         if (!trace_options[i])
3387                 ret = set_tracer_option(tr->current_trace, cmp, neg);
3388
3389         mutex_unlock(&trace_types_lock);
3390
3391         return ret;
3392 }
3393
3394 static ssize_t
3395 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3396                         size_t cnt, loff_t *ppos)
3397 {
3398         struct seq_file *m = filp->private_data;
3399         struct trace_array *tr = m->private;
3400         char buf[64];
3401         int ret;
3402
3403         if (cnt >= sizeof(buf))
3404                 return -EINVAL;
3405
3406         if (copy_from_user(&buf, ubuf, cnt))
3407                 return -EFAULT;
3408
3409         buf[cnt] = 0;
3410
3411         ret = trace_set_options(tr, buf);
3412         if (ret < 0)
3413                 return ret;
3414
3415         *ppos += cnt;
3416
3417         return cnt;
3418 }
3419
3420 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3421 {
3422         struct trace_array *tr = inode->i_private;
3423         int ret;
3424
3425         if (tracing_disabled)
3426                 return -ENODEV;
3427
3428         if (trace_array_get(tr) < 0)
3429                 return -ENODEV;
3430
3431         ret = single_open(file, tracing_trace_options_show, inode->i_private);
3432         if (ret < 0)
3433                 trace_array_put(tr);
3434
3435         return ret;
3436 }
3437
3438 static const struct file_operations tracing_iter_fops = {
3439         .open           = tracing_trace_options_open,
3440         .read           = seq_read,
3441         .llseek         = seq_lseek,
3442         .release        = tracing_single_release_tr,
3443         .write          = tracing_trace_options_write,
3444 };
3445
3446 static const char readme_msg[] =
3447         "tracing mini-HOWTO:\n\n"
3448         "# echo 0 > tracing_on : quick way to disable tracing\n"
3449         "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3450         " Important files:\n"
3451         "  trace\t\t\t- The static contents of the buffer\n"
3452         "\t\t\t  To clear the buffer write into this file: echo > trace\n"
3453         "  trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3454         "  current_tracer\t- function and latency tracers\n"
3455         "  available_tracers\t- list of configured tracers for current_tracer\n"
3456         "  buffer_size_kb\t- view and modify size of per cpu buffer\n"
3457         "  buffer_total_size_kb  - view total size of all cpu buffers\n\n"
3458         "  trace_clock\t\t-change the clock used to order events\n"
3459         "       local:   Per cpu clock but may not be synced across CPUs\n"
3460         "      global:   Synced across CPUs but slows tracing down.\n"
3461         "     counter:   Not a clock, but just an increment\n"
3462         "      uptime:   Jiffy counter from time of boot\n"
3463         "        perf:   Same clock that perf events use\n"
3464 #ifdef CONFIG_X86_64
3465         "     x86-tsc:   TSC cycle counter\n"
3466 #endif
3467         "\n  trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3468         "  tracing_cpumask\t- Limit which CPUs to trace\n"
3469         "  instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3470         "\t\t\t  Remove sub-buffer with rmdir\n"
3471         "  trace_options\t\t- Set format or modify how tracing happens\n"
3472         "\t\t\t  Disable an option by adding a suffix 'no' to the option name\n"
3473 #ifdef CONFIG_DYNAMIC_FTRACE
3474         "\n  available_filter_functions - list of functions that can be filtered on\n"
3475         "  set_ftrace_filter\t- echo function name in here to only trace these functions\n"
3476         "            accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3477         "            modules: Can select a group via module\n"
3478         "             Format: :mod:<module-name>\n"
3479         "             example: echo :mod:ext3 > set_ftrace_filter\n"
3480         "            triggers: a command to perform when function is hit\n"
3481         "              Format: <function>:<trigger>[:count]\n"
3482         "             trigger: traceon, traceoff\n"
3483         "                      enable_event:<system>:<event>\n"
3484         "                      disable_event:<system>:<event>\n"
3485 #ifdef CONFIG_STACKTRACE
3486         "                      stacktrace\n"
3487 #endif
3488 #ifdef CONFIG_TRACER_SNAPSHOT
3489         "                      snapshot\n"
3490 #endif
3491         "             example: echo do_fault:traceoff > set_ftrace_filter\n"
3492         "                      echo do_trap:traceoff:3 > set_ftrace_filter\n"
3493         "             The first one will disable tracing every time do_fault is hit\n"
3494         "             The second will disable tracing at most 3 times when do_trap is hit\n"
3495         "               The first time do trap is hit and it disables tracing, the counter\n"
3496         "               will decrement to 2. If tracing is already disabled, the counter\n"
3497         "               will not decrement. It only decrements when the trigger did work\n"
3498         "             To remove trigger without count:\n"
3499         "               echo '!<function>:<trigger> > set_ftrace_filter\n"
3500         "             To remove trigger with a count:\n"
3501         "               echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3502         "  set_ftrace_notrace\t- echo function name in here to never trace.\n"
3503         "            accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3504         "            modules: Can select a group via module command :mod:\n"
3505         "            Does not accept triggers\n"
3506 #endif /* CONFIG_DYNAMIC_FTRACE */
3507 #ifdef CONFIG_FUNCTION_TRACER
3508         "  set_ftrace_pid\t- Write pid(s) to only function trace those pids (function)\n"
3509 #endif
3510 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3511         "  set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3512         "  max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3513 #endif
3514 #ifdef CONFIG_TRACER_SNAPSHOT
3515         "\n  snapshot\t\t- Like 'trace' but shows the content of the static snapshot buffer\n"
3516         "\t\t\t  Read the contents for more information\n"
3517 #endif
3518 #ifdef CONFIG_STACKTRACE
3519         "  stack_trace\t\t- Shows the max stack trace when active\n"
3520         "  stack_max_size\t- Shows current max stack size that was traced\n"
3521         "\t\t\t  Write into this file to reset the max size (trigger a new trace)\n"
3522 #ifdef CONFIG_DYNAMIC_FTRACE
3523         "  stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace traces\n"
3524 #endif
3525 #endif /* CONFIG_STACKTRACE */
3526 ;
3527
3528 static ssize_t
3529 tracing_readme_read(struct file *filp, char __user *ubuf,
3530                        size_t cnt, loff_t *ppos)
3531 {
3532         return simple_read_from_buffer(ubuf, cnt, ppos,
3533                                         readme_msg, strlen(readme_msg));
3534 }
3535
3536 static const struct file_operations tracing_readme_fops = {
3537         .open           = tracing_open_generic,
3538         .read           = tracing_readme_read,
3539         .llseek         = generic_file_llseek,
3540 };
3541
3542 static ssize_t
3543 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3544                                 size_t cnt, loff_t *ppos)
3545 {
3546         char *buf_comm;
3547         char *file_buf;
3548         char *buf;
3549         int len = 0;
3550         int pid;
3551         int i;
3552
3553         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3554         if (!file_buf)
3555                 return -ENOMEM;
3556
3557         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3558         if (!buf_comm) {
3559                 kfree(file_buf);
3560                 return -ENOMEM;
3561         }
3562
3563         buf = file_buf;
3564
3565         for (i = 0; i < SAVED_CMDLINES; i++) {
3566                 int r;
3567
3568                 pid = map_cmdline_to_pid[i];
3569                 if (pid == -1 || pid == NO_CMDLINE_MAP)
3570                         continue;
3571
3572                 trace_find_cmdline(pid, buf_comm);
3573                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3574                 buf += r;
3575                 len += r;
3576         }
3577
3578         len = simple_read_from_buffer(ubuf, cnt, ppos,
3579                                       file_buf, len);
3580
3581         kfree(file_buf);
3582         kfree(buf_comm);
3583
3584         return len;
3585 }
3586
3587 static const struct file_operations tracing_saved_cmdlines_fops = {
3588     .open       = tracing_open_generic,
3589     .read       = tracing_saved_cmdlines_read,
3590     .llseek     = generic_file_llseek,
3591 };
3592
3593 static ssize_t
3594 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3595                        size_t cnt, loff_t *ppos)
3596 {
3597         struct trace_array *tr = filp->private_data;
3598         char buf[MAX_TRACER_SIZE+2];
3599         int r;
3600
3601         mutex_lock(&trace_types_lock);
3602         r = sprintf(buf, "%s\n", tr->current_trace->name);
3603         mutex_unlock(&trace_types_lock);
3604
3605         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3606 }
3607
3608 int tracer_init(struct tracer *t, struct trace_array *tr)
3609 {
3610         tracing_reset_online_cpus(&tr->trace_buffer);
3611         return t->init(tr);
3612 }
3613
3614 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3615 {
3616         int cpu;
3617
3618         for_each_tracing_cpu(cpu)
3619                 per_cpu_ptr(buf->data, cpu)->entries = val;
3620 }
3621
3622 #ifdef CONFIG_TRACER_MAX_TRACE
3623 /* resize @tr's buffer to the size of @size_tr's entries */
3624 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3625                                         struct trace_buffer *size_buf, int cpu_id)
3626 {
3627         int cpu, ret = 0;
3628
3629         if (cpu_id == RING_BUFFER_ALL_CPUS) {
3630                 for_each_tracing_cpu(cpu) {
3631                         ret = ring_buffer_resize(trace_buf->buffer,
3632                                  per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3633                         if (ret < 0)
3634                                 break;
3635                         per_cpu_ptr(trace_buf->data, cpu)->entries =
3636                                 per_cpu_ptr(size_buf->data, cpu)->entries;
3637                 }
3638         } else {
3639                 ret = ring_buffer_resize(trace_buf->buffer,
3640                                  per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3641                 if (ret == 0)
3642                         per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3643                                 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3644         }
3645
3646         return ret;
3647 }
3648 #endif /* CONFIG_TRACER_MAX_TRACE */
3649
3650 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3651                                         unsigned long size, int cpu)
3652 {
3653         int ret;
3654
3655         /*
3656          * If kernel or user changes the size of the ring buffer
3657          * we use the size that was given, and we can forget about
3658          * expanding it later.
3659          */
3660         ring_buffer_expanded = true;
3661
3662         /* May be called before buffers are initialized */
3663         if (!tr->trace_buffer.buffer)
3664                 return 0;
3665
3666         ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3667         if (ret < 0)
3668                 return ret;
3669
3670 #ifdef CONFIG_TRACER_MAX_TRACE
3671         if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3672             !tr->current_trace->use_max_tr)
3673                 goto out;
3674
3675         ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3676         if (ret < 0) {
3677                 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3678                                                      &tr->trace_buffer, cpu);
3679                 if (r < 0) {
3680                         /*
3681                          * AARGH! We are left with different
3682                          * size max buffer!!!!
3683                          * The max buffer is our "snapshot" buffer.
3684                          * When a tracer needs a snapshot (one of the
3685                          * latency tracers), it swaps the max buffer
3686                          * with the saved snap shot. We succeeded to
3687                          * update the size of the main buffer, but failed to
3688                          * update the size of the max buffer. But when we tried
3689                          * to reset the main buffer to the original size, we
3690                          * failed there too. This is very unlikely to
3691                          * happen, but if it does, warn and kill all
3692                          * tracing.
3693                          */
3694                         WARN_ON(1);
3695                         tracing_disabled = 1;
3696                 }
3697                 return ret;
3698         }
3699
3700         if (cpu == RING_BUFFER_ALL_CPUS)
3701                 set_buffer_entries(&tr->max_buffer, size);
3702         else
3703                 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3704
3705  out:
3706 #endif /* CONFIG_TRACER_MAX_TRACE */
3707
3708         if (cpu == RING_BUFFER_ALL_CPUS)
3709                 set_buffer_entries(&tr->trace_buffer, size);
3710         else
3711                 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
3712
3713         return ret;
3714 }
3715
3716 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3717                                           unsigned long size, int cpu_id)
3718 {
3719         int ret = size;
3720
3721         mutex_lock(&trace_types_lock);
3722
3723         if (cpu_id != RING_BUFFER_ALL_CPUS) {
3724                 /* make sure, this cpu is enabled in the mask */
3725                 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3726                         ret = -EINVAL;
3727                         goto out;
3728                 }
3729         }
3730
3731         ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3732         if (ret < 0)
3733                 ret = -ENOMEM;
3734
3735 out:
3736         mutex_unlock(&trace_types_lock);
3737
3738         return ret;
3739 }
3740
3741
3742 /**
3743  * tracing_update_buffers - used by tracing facility to expand ring buffers
3744  *
3745  * To save on memory when the tracing is never used on a system with it
3746  * configured in. The ring buffers are set to a minimum size. But once
3747  * a user starts to use the tracing facility, then they need to grow
3748  * to their default size.
3749  *
3750  * This function is to be called when a tracer is about to be used.
3751  */
3752 int tracing_update_buffers(void)
3753 {
3754         int ret = 0;
3755
3756         mutex_lock(&trace_types_lock);
3757         if (!ring_buffer_expanded)
3758                 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3759                                                 RING_BUFFER_ALL_CPUS);
3760         mutex_unlock(&trace_types_lock);
3761
3762         return ret;
3763 }
3764
3765 struct trace_option_dentry;
3766
3767 static struct trace_option_dentry *
3768 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
3769
3770 static void
3771 destroy_trace_option_files(struct trace_option_dentry *topts);
3772
3773 static int tracing_set_tracer(const char *buf)
3774 {
3775         static struct trace_option_dentry *topts;
3776         struct trace_array *tr = &global_trace;
3777         struct tracer *t;
3778 #ifdef CONFIG_TRACER_MAX_TRACE
3779         bool had_max_tr;
3780 #endif
3781         int ret = 0;
3782
3783         mutex_lock(&trace_types_lock);
3784
3785         if (!ring_buffer_expanded) {
3786                 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3787                                                 RING_BUFFER_ALL_CPUS);
3788                 if (ret < 0)
3789                         goto out;
3790                 ret = 0;
3791         }
3792
3793         for (t = trace_types; t; t = t->next) {
3794                 if (strcmp(t->name, buf) == 0)
3795                         break;
3796         }
3797         if (!t) {
3798                 ret = -EINVAL;
3799                 goto out;
3800         }
3801         if (t == tr->current_trace)
3802                 goto out;
3803
3804         trace_branch_disable();
3805
3806         tr->current_trace->enabled = false;
3807
3808         if (tr->current_trace->reset)
3809                 tr->current_trace->reset(tr);
3810
3811         /* Current trace needs to be nop_trace before synchronize_sched */
3812         tr->current_trace = &nop_trace;
3813
3814 #ifdef CONFIG_TRACER_MAX_TRACE
3815         had_max_tr = tr->allocated_snapshot;
3816
3817         if (had_max_tr && !t->use_max_tr) {
3818                 /*
3819                  * We need to make sure that the update_max_tr sees that
3820                  * current_trace changed to nop_trace to keep it from
3821                  * swapping the buffers after we resize it.
3822                  * The update_max_tr is called from interrupts disabled
3823                  * so a synchronized_sched() is sufficient.
3824                  */
3825                 synchronize_sched();
3826                 free_snapshot(tr);
3827         }
3828 #endif
3829         destroy_trace_option_files(topts);
3830
3831         topts = create_trace_option_files(tr, t);
3832
3833 #ifdef CONFIG_TRACER_MAX_TRACE
3834         if (t->use_max_tr && !had_max_tr) {
3835                 ret = alloc_snapshot(tr);
3836                 if (ret < 0)
3837                         goto out;
3838         }
3839 #endif
3840
3841         if (t->init) {
3842                 ret = tracer_init(t, tr);
3843                 if (ret)
3844                         goto out;
3845         }
3846
3847         tr->current_trace = t;
3848         tr->current_trace->enabled = true;
3849         trace_branch_enable(tr);
3850  out:
3851         mutex_unlock(&trace_types_lock);
3852
3853         return ret;
3854 }
3855
3856 static ssize_t
3857 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3858                         size_t cnt, loff_t *ppos)
3859 {
3860         char buf[MAX_TRACER_SIZE+1];
3861         int i;
3862         size_t ret;
3863         int err;
3864
3865         ret = cnt;
3866
3867         if (cnt > MAX_TRACER_SIZE)
3868                 cnt = MAX_TRACER_SIZE;
3869
3870         if (copy_from_user(&buf, ubuf, cnt))
3871                 return -EFAULT;
3872
3873         buf[cnt] = 0;
3874
3875         /* strip ending whitespace. */
3876         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3877                 buf[i] = 0;
3878
3879         err = tracing_set_tracer(buf);
3880         if (err)
3881                 return err;
3882
3883         *ppos += ret;
3884
3885         return ret;
3886 }
3887
3888 static ssize_t
3889 tracing_max_lat_read(struct file *filp, char __user *ubuf,
3890                      size_t cnt, loff_t *ppos)
3891 {
3892         unsigned long *ptr = filp->private_data;
3893         char buf[64];
3894         int r;
3895
3896         r = snprintf(buf, sizeof(buf), "%ld\n",
3897                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3898         if (r > sizeof(buf))
3899                 r = sizeof(buf);
3900         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3901 }
3902
3903 static ssize_t
3904 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3905                       size_t cnt, loff_t *ppos)
3906 {
3907         unsigned long *ptr = filp->private_data;
3908         unsigned long val;
3909         int ret;
3910
3911         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3912         if (ret)
3913                 return ret;
3914
3915         *ptr = val * 1000;
3916
3917         return cnt;
3918 }
3919
3920 static int tracing_open_pipe(struct inode *inode, struct file *filp)
3921 {
3922         struct trace_array *tr = inode->i_private;
3923         struct trace_iterator *iter;
3924         int ret = 0;
3925
3926         if (tracing_disabled)
3927                 return -ENODEV;
3928
3929         if (trace_array_get(tr) < 0)
3930                 return -ENODEV;
3931
3932         mutex_lock(&trace_types_lock);
3933
3934         /* create a buffer to store the information to pass to userspace */
3935         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3936         if (!iter) {
3937                 ret = -ENOMEM;
3938                 __trace_array_put(tr);
3939                 goto out;
3940         }
3941
3942         /*
3943          * We make a copy of the current tracer to avoid concurrent
3944          * changes on it while we are reading.
3945          */
3946         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3947         if (!iter->trace) {
3948                 ret = -ENOMEM;
3949                 goto fail;
3950         }
3951         *iter->trace = *tr->current_trace;
3952
3953         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3954                 ret = -ENOMEM;
3955                 goto fail;
3956         }
3957
3958         /* trace pipe does not show start of buffer */
3959         cpumask_setall(iter->started);
3960
3961         if (trace_flags & TRACE_ITER_LATENCY_FMT)
3962                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3963
3964         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3965         if (trace_clocks[tr->clock_id].in_ns)
3966                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3967
3968         iter->tr = tr;
3969         iter->trace_buffer = &tr->trace_buffer;
3970         iter->cpu_file = tracing_get_cpu(inode);
3971         mutex_init(&iter->mutex);
3972         filp->private_data = iter;
3973
3974         if (iter->trace->pipe_open)
3975                 iter->trace->pipe_open(iter);
3976
3977         nonseekable_open(inode, filp);
3978 out:
3979         mutex_unlock(&trace_types_lock);
3980         return ret;
3981
3982 fail:
3983         kfree(iter->trace);
3984         kfree(iter);
3985         __trace_array_put(tr);
3986         mutex_unlock(&trace_types_lock);
3987         return ret;
3988 }
3989
3990 static int tracing_release_pipe(struct inode *inode, struct file *file)
3991 {
3992         struct trace_iterator *iter = file->private_data;
3993         struct trace_array *tr = inode->i_private;
3994
3995         mutex_lock(&trace_types_lock);
3996
3997         if (iter->trace->pipe_close)
3998                 iter->trace->pipe_close(iter);
3999
4000         mutex_unlock(&trace_types_lock);
4001
4002         free_cpumask_var(iter->started);
4003         mutex_destroy(&iter->mutex);
4004         kfree(iter->trace);
4005         kfree(iter);
4006
4007         trace_array_put(tr);
4008
4009         return 0;
4010 }
4011
4012 static unsigned int
4013 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4014 {
4015         /* Iterators are static, they should be filled or empty */
4016         if (trace_buffer_iter(iter, iter->cpu_file))
4017                 return POLLIN | POLLRDNORM;
4018
4019         if (trace_flags & TRACE_ITER_BLOCK)
4020                 /*
4021                  * Always select as readable when in blocking mode
4022                  */
4023                 return POLLIN | POLLRDNORM;
4024         else
4025                 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4026                                              filp, poll_table);
4027 }
4028
4029 static unsigned int
4030 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4031 {
4032         struct trace_iterator *iter = filp->private_data;
4033
4034         return trace_poll(iter, filp, poll_table);
4035 }
4036
4037 /*
4038  * This is a make-shift waitqueue.
4039  * A tracer might use this callback on some rare cases:
4040  *
4041  *  1) the current tracer might hold the runqueue lock when it wakes up
4042  *     a reader, hence a deadlock (sched, function, and function graph tracers)
4043  *  2) the function tracers, trace all functions, we don't want
4044  *     the overhead of calling wake_up and friends
4045  *     (and tracing them too)
4046  *
4047  *     Anyway, this is really very primitive wakeup.
4048  */
4049 void poll_wait_pipe(struct trace_iterator *iter)
4050 {
4051         set_current_state(TASK_INTERRUPTIBLE);
4052         /* sleep for 100 msecs, and try again. */
4053         schedule_timeout(HZ / 10);
4054 }
4055
4056 /* Must be called with trace_types_lock mutex held. */
4057 static int tracing_wait_pipe(struct file *filp)
4058 {
4059         struct trace_iterator *iter = filp->private_data;
4060
4061         while (trace_empty(iter)) {
4062
4063                 if ((filp->f_flags & O_NONBLOCK)) {
4064                         return -EAGAIN;
4065                 }
4066
4067                 mutex_unlock(&iter->mutex);
4068
4069                 iter->trace->wait_pipe(iter);
4070
4071                 mutex_lock(&iter->mutex);
4072
4073                 if (signal_pending(current))
4074                         return -EINTR;
4075
4076                 /*
4077                  * We block until we read something and tracing is disabled.
4078                  * We still block if tracing is disabled, but we have never
4079                  * read anything. This allows a user to cat this file, and
4080                  * then enable tracing. But after we have read something,
4081                  * we give an EOF when tracing is again disabled.
4082                  *
4083                  * iter->pos will be 0 if we haven't read anything.
4084                  */
4085                 if (!tracing_is_on() && iter->pos)
4086                         break;
4087         }
4088
4089         return 1;
4090 }
4091
4092 /*
4093  * Consumer reader.
4094  */
4095 static ssize_t
4096 tracing_read_pipe(struct file *filp, char __user *ubuf,
4097                   size_t cnt, loff_t *ppos)
4098 {
4099         struct trace_iterator *iter = filp->private_data;
4100         struct trace_array *tr = iter->tr;
4101         ssize_t sret;
4102
4103         /* return any leftover data */
4104         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4105         if (sret != -EBUSY)
4106                 return sret;
4107
4108         trace_seq_init(&iter->seq);
4109
4110         /* copy the tracer to avoid using a global lock all around */
4111         mutex_lock(&trace_types_lock);
4112         if (unlikely(iter->trace->name != tr->current_trace->name))
4113                 *iter->trace = *tr->current_trace;
4114         mutex_unlock(&trace_types_lock);
4115
4116         /*
4117          * Avoid more than one consumer on a single file descriptor
4118          * This is just a matter of traces coherency, the ring buffer itself
4119          * is protected.
4120          */
4121         mutex_lock(&iter->mutex);
4122         if (iter->trace->read) {
4123                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4124                 if (sret)
4125                         goto out;
4126         }
4127
4128 waitagain:
4129         sret = tracing_wait_pipe(filp);
4130         if (sret <= 0)
4131                 goto out;
4132
4133         /* stop when tracing is finished */
4134         if (trace_empty(iter)) {
4135                 sret = 0;
4136                 goto out;
4137         }
4138
4139         if (cnt >= PAGE_SIZE)
4140                 cnt = PAGE_SIZE - 1;
4141
4142         /* reset all but tr, trace, and overruns */
4143         memset(&iter->seq, 0,
4144                sizeof(struct trace_iterator) -
4145                offsetof(struct trace_iterator, seq));
4146         cpumask_clear(iter->started);
4147         iter->pos = -1;
4148
4149         trace_event_read_lock();
4150         trace_access_lock(iter->cpu_file);
4151         while (trace_find_next_entry_inc(iter) != NULL) {
4152                 enum print_line_t ret;
4153                 int len = iter->seq.len;
4154
4155                 ret = print_trace_line(iter);
4156                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4157                         /* don't print partial lines */
4158                         iter->seq.len = len;
4159                         break;
4160                 }
4161                 if (ret != TRACE_TYPE_NO_CONSUME)
4162                         trace_consume(iter);
4163
4164                 if (iter->seq.len >= cnt)
4165                         break;
4166
4167                 /*
4168                  * Setting the full flag means we reached the trace_seq buffer
4169                  * size and we should leave by partial output condition above.
4170                  * One of the trace_seq_* functions is not used properly.
4171                  */
4172                 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4173                           iter->ent->type);
4174         }
4175         trace_access_unlock(iter->cpu_file);
4176         trace_event_read_unlock();
4177
4178         /* Now copy what we have to the user */
4179         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4180         if (iter->seq.readpos >= iter->seq.len)
4181                 trace_seq_init(&iter->seq);
4182
4183         /*
4184          * If there was nothing to send to user, in spite of consuming trace
4185          * entries, go back to wait for more entries.
4186          */
4187         if (sret == -EBUSY)
4188                 goto waitagain;
4189
4190 out:
4191         mutex_unlock(&iter->mutex);
4192
4193         return sret;
4194 }
4195
4196 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
4197                                      struct pipe_buffer *buf)
4198 {
4199         __free_page(buf->page);
4200 }
4201
4202 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4203                                      unsigned int idx)
4204 {
4205         __free_page(spd->pages[idx]);
4206 }
4207
4208 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4209         .can_merge              = 0,
4210         .map                    = generic_pipe_buf_map,
4211         .unmap                  = generic_pipe_buf_unmap,
4212         .confirm                = generic_pipe_buf_confirm,
4213         .release                = tracing_pipe_buf_release,
4214         .steal                  = generic_pipe_buf_steal,
4215         .get                    = generic_pipe_buf_get,
4216 };
4217
4218 static size_t
4219 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4220 {
4221         size_t count;
4222         int ret;
4223
4224         /* Seq buffer is page-sized, exactly what we need. */
4225         for (;;) {
4226                 count = iter->seq.len;
4227                 ret = print_trace_line(iter);
4228                 count = iter->seq.len - count;
4229                 if (rem < count) {
4230                         rem = 0;
4231                         iter->seq.len -= count;
4232                         break;
4233                 }
4234                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4235                         iter->seq.len -= count;
4236                         break;
4237                 }
4238
4239                 if (ret != TRACE_TYPE_NO_CONSUME)
4240                         trace_consume(iter);
4241                 rem -= count;
4242                 if (!trace_find_next_entry_inc(iter))   {
4243                         rem = 0;
4244                         iter->ent = NULL;
4245                         break;
4246                 }
4247         }
4248
4249         return rem;
4250 }
4251
4252 static ssize_t tracing_splice_read_pipe(struct file *filp,
4253                                         loff_t *ppos,
4254                                         struct pipe_inode_info *pipe,
4255                                         size_t len,
4256                                         unsigned int flags)
4257 {
4258         struct page *pages_def[PIPE_DEF_BUFFERS];
4259         struct partial_page partial_def[PIPE_DEF_BUFFERS];
4260         struct trace_iterator *iter = filp->private_data;
4261         struct splice_pipe_desc spd = {
4262                 .pages          = pages_def,
4263                 .partial        = partial_def,
4264                 .nr_pages       = 0, /* This gets updated below. */
4265                 .nr_pages_max   = PIPE_DEF_BUFFERS,
4266                 .flags          = flags,
4267                 .ops            = &tracing_pipe_buf_ops,
4268                 .spd_release    = tracing_spd_release_pipe,
4269         };
4270         struct trace_array *tr = iter->tr;
4271         ssize_t ret;
4272         size_t rem;
4273         unsigned int i;
4274
4275         if (splice_grow_spd(pipe, &spd))
4276                 return -ENOMEM;
4277
4278         /* copy the tracer to avoid using a global lock all around */
4279         mutex_lock(&trace_types_lock);
4280         if (unlikely(iter->trace->name != tr->current_trace->name))
4281                 *iter->trace = *tr->current_trace;
4282         mutex_unlock(&trace_types_lock);
4283
4284         mutex_lock(&iter->mutex);
4285
4286         if (iter->trace->splice_read) {
4287                 ret = iter->trace->splice_read(iter, filp,
4288                                                ppos, pipe, len, flags);
4289                 if (ret)
4290                         goto out_err;
4291         }
4292
4293         ret = tracing_wait_pipe(filp);
4294         if (ret <= 0)
4295                 goto out_err;
4296
4297         if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4298                 ret = -EFAULT;
4299                 goto out_err;
4300         }
4301
4302         trace_event_read_lock();
4303         trace_access_lock(iter->cpu_file);
4304
4305         /* Fill as many pages as possible. */
4306         for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4307                 spd.pages[i] = alloc_page(GFP_KERNEL);
4308                 if (!spd.pages[i])
4309                         break;
4310
4311                 rem = tracing_fill_pipe_page(rem, iter);
4312
4313                 /* Copy the data into the page, so we can start over. */
4314                 ret = trace_seq_to_buffer(&iter->seq,
4315                                           page_address(spd.pages[i]),
4316                                           iter->seq.len);
4317                 if (ret < 0) {
4318                         __free_page(spd.pages[i]);
4319                         break;
4320                 }
4321                 spd.partial[i].offset = 0;
4322                 spd.partial[i].len = iter->seq.len;
4323
4324                 trace_seq_init(&iter->seq);
4325         }
4326
4327         trace_access_unlock(iter->cpu_file);
4328         trace_event_read_unlock();
4329         mutex_unlock(&iter->mutex);
4330
4331         spd.nr_pages = i;
4332
4333         ret = splice_to_pipe(pipe, &spd);
4334 out:
4335         splice_shrink_spd(&spd);
4336         return ret;
4337
4338 out_err:
4339         mutex_unlock(&iter->mutex);
4340         goto out;
4341 }
4342
4343 static ssize_t
4344 tracing_entries_read(struct file *filp, char __user *ubuf,
4345                      size_t cnt, loff_t *ppos)
4346 {
4347         struct inode *inode = file_inode(filp);
4348         struct trace_array *tr = inode->i_private;
4349         int cpu = tracing_get_cpu(inode);
4350         char buf[64];
4351         int r = 0;
4352         ssize_t ret;
4353
4354         mutex_lock(&trace_types_lock);
4355
4356         if (cpu == RING_BUFFER_ALL_CPUS) {
4357                 int cpu, buf_size_same;
4358                 unsigned long size;
4359
4360                 size = 0;
4361                 buf_size_same = 1;
4362                 /* check if all cpu sizes are same */
4363                 for_each_tracing_cpu(cpu) {
4364                         /* fill in the size from first enabled cpu */
4365                         if (size == 0)
4366                                 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4367                         if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4368                                 buf_size_same = 0;
4369                                 break;
4370                         }
4371                 }
4372
4373                 if (buf_size_same) {
4374                         if (!ring_buffer_expanded)
4375                                 r = sprintf(buf, "%lu (expanded: %lu)\n",
4376                                             size >> 10,
4377                                             trace_buf_size >> 10);
4378                         else
4379                                 r = sprintf(buf, "%lu\n", size >> 10);
4380                 } else
4381                         r = sprintf(buf, "X\n");
4382         } else
4383                 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4384
4385         mutex_unlock(&trace_types_lock);
4386
4387         ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4388         return ret;
4389 }
4390
4391 static ssize_t
4392 tracing_entries_write(struct file *filp, const char __user *ubuf,
4393                       size_t cnt, loff_t *ppos)
4394 {
4395         struct inode *inode = file_inode(filp);
4396         struct trace_array *tr = inode->i_private;
4397         unsigned long val;
4398         int ret;
4399
4400         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4401         if (ret)
4402                 return ret;
4403
4404         /* must have at least 1 entry */
4405         if (!val)
4406                 return -EINVAL;
4407
4408         /* value is in KB */
4409         val <<= 10;
4410         ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4411         if (ret < 0)
4412                 return ret;
4413
4414         *ppos += cnt;
4415
4416         return cnt;
4417 }
4418
4419 static ssize_t
4420 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4421                                 size_t cnt, loff_t *ppos)
4422 {
4423         struct trace_array *tr = filp->private_data;
4424         char buf[64];
4425         int r, cpu;
4426         unsigned long size = 0, expanded_size = 0;
4427
4428         mutex_lock(&trace_types_lock);
4429         for_each_tracing_cpu(cpu) {
4430                 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4431                 if (!ring_buffer_expanded)
4432                         expanded_size += trace_buf_size >> 10;
4433         }
4434         if (ring_buffer_expanded)
4435                 r = sprintf(buf, "%lu\n", size);
4436         else
4437                 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4438         mutex_unlock(&trace_types_lock);
4439
4440         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4441 }
4442
4443 static ssize_t
4444 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4445                           size_t cnt, loff_t *ppos)
4446 {
4447         /*
4448          * There is no need to read what the user has written, this function
4449          * is just to make sure that there is no error when "echo" is used
4450          */
4451
4452         *ppos += cnt;
4453
4454         return cnt;
4455 }
4456
4457 static int
4458 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4459 {
4460         struct trace_array *tr = inode->i_private;
4461
4462         /* disable tracing ? */
4463         if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4464                 tracer_tracing_off(tr);
4465         /* resize the ring buffer to 0 */
4466         tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4467
4468         trace_array_put(tr);
4469
4470         return 0;
4471 }
4472
4473 static ssize_t
4474 tracing_mark_write(struct file *filp, const char __user *ubuf,
4475                                         size_t cnt, loff_t *fpos)
4476 {
4477         unsigned long addr = (unsigned long)ubuf;
4478         struct trace_array *tr = filp->private_data;
4479         struct ring_buffer_event *event;
4480         struct ring_buffer *buffer;
4481         struct print_entry *entry;
4482         unsigned long irq_flags;
4483         struct page *pages[2];
4484         void *map_page[2];
4485         int nr_pages = 1;
4486         ssize_t written;
4487         int offset;
4488         int size;
4489         int len;
4490         int ret;
4491         int i;
4492
4493         if (tracing_disabled)
4494                 return -EINVAL;
4495
4496         if (!(trace_flags & TRACE_ITER_MARKERS))
4497                 return -EINVAL;
4498
4499         if (cnt > TRACE_BUF_SIZE)
4500                 cnt = TRACE_BUF_SIZE;
4501
4502         /*
4503          * Userspace is injecting traces into the kernel trace buffer.
4504          * We want to be as non intrusive as possible.
4505          * To do so, we do not want to allocate any special buffers
4506          * or take any locks, but instead write the userspace data
4507          * straight into the ring buffer.
4508          *
4509          * First we need to pin the userspace buffer into memory,
4510          * which, most likely it is, because it just referenced it.
4511          * But there's no guarantee that it is. By using get_user_pages_fast()
4512          * and kmap_atomic/kunmap_atomic() we can get access to the
4513          * pages directly. We then write the data directly into the
4514          * ring buffer.
4515          */
4516         BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4517
4518         /* check if we cross pages */
4519         if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4520                 nr_pages = 2;
4521
4522         offset = addr & (PAGE_SIZE - 1);
4523         addr &= PAGE_MASK;
4524
4525         ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4526         if (ret < nr_pages) {
4527                 while (--ret >= 0)
4528                         put_page(pages[ret]);
4529                 written = -EFAULT;
4530                 goto out;
4531         }
4532
4533         for (i = 0; i < nr_pages; i++)
4534                 map_page[i] = kmap_atomic(pages[i]);
4535
4536         local_save_flags(irq_flags);
4537         size = sizeof(*entry) + cnt + 2; /* possible \n added */
4538         buffer = tr->trace_buffer.buffer;
4539         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4540                                           irq_flags, preempt_count());
4541         if (!event) {
4542                 /* Ring buffer disabled, return as if not open for write */
4543                 written = -EBADF;
4544                 goto out_unlock;
4545         }
4546
4547         entry = ring_buffer_event_data(event);
4548         entry->ip = _THIS_IP_;
4549
4550         if (nr_pages == 2) {
4551                 len = PAGE_SIZE - offset;
4552                 memcpy(&entry->buf, map_page[0] + offset, len);
4553                 memcpy(&entry->buf[len], map_page[1], cnt - len);
4554         } else
4555                 memcpy(&entry->buf, map_page[0] + offset, cnt);
4556
4557         if (entry->buf[cnt - 1] != '\n') {
4558                 entry->buf[cnt] = '\n';
4559                 entry->buf[cnt + 1] = '\0';
4560         } else
4561                 entry->buf[cnt] = '\0';
4562
4563         __buffer_unlock_commit(buffer, event);
4564
4565         written = cnt;
4566
4567         *fpos += written;
4568
4569  out_unlock:
4570         for (i = 0; i < nr_pages; i++){
4571                 kunmap_atomic(map_page[i]);
4572                 put_page(pages[i]);
4573         }
4574  out:
4575         return written;
4576 }
4577
4578 static int tracing_clock_show(struct seq_file *m, void *v)
4579 {
4580         struct trace_array *tr = m->private;
4581         int i;
4582
4583         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4584                 seq_printf(m,
4585                         "%s%s%s%s", i ? " " : "",
4586                         i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4587                         i == tr->clock_id ? "]" : "");
4588         seq_putc(m, '\n');
4589
4590         return 0;
4591 }
4592
4593 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4594                                    size_t cnt, loff_t *fpos)
4595 {
4596         struct seq_file *m = filp->private_data;
4597         struct trace_array *tr = m->private;
4598         char buf[64];
4599         const char *clockstr;
4600         int i;
4601
4602         if (cnt >= sizeof(buf))
4603                 return -EINVAL;
4604
4605         if (copy_from_user(&buf, ubuf, cnt))
4606                 return -EFAULT;
4607
4608         buf[cnt] = 0;
4609
4610         clockstr = strstrip(buf);
4611
4612         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4613                 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4614                         break;
4615         }
4616         if (i == ARRAY_SIZE(trace_clocks))
4617                 return -EINVAL;
4618
4619         mutex_lock(&trace_types_lock);
4620
4621         tr->clock_id = i;
4622
4623         ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4624
4625         /*
4626          * New clock may not be consistent with the previous clock.
4627          * Reset the buffer so that it doesn't have incomparable timestamps.
4628          */
4629         tracing_reset_online_cpus(&tr->trace_buffer);
4630
4631 #ifdef CONFIG_TRACER_MAX_TRACE
4632         if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4633                 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4634         tracing_reset_online_cpus(&tr->max_buffer);
4635 #endif
4636
4637         mutex_unlock(&trace_types_lock);
4638
4639         *fpos += cnt;
4640
4641         return cnt;
4642 }
4643
4644 static int tracing_clock_open(struct inode *inode, struct file *file)
4645 {
4646         struct trace_array *tr = inode->i_private;
4647         int ret;
4648
4649         if (tracing_disabled)
4650                 return -ENODEV;
4651
4652         if (trace_array_get(tr))
4653                 return -ENODEV;
4654
4655         ret = single_open(file, tracing_clock_show, inode->i_private);
4656         if (ret < 0)
4657                 trace_array_put(tr);
4658
4659         return ret;
4660 }
4661
4662 struct ftrace_buffer_info {
4663         struct trace_iterator   iter;
4664         void                    *spare;
4665         unsigned int            read;
4666 };
4667
4668 #ifdef CONFIG_TRACER_SNAPSHOT
4669 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4670 {
4671         struct trace_array *tr = inode->i_private;
4672         struct trace_iterator *iter;
4673         struct seq_file *m;
4674         int ret = 0;
4675
4676         if (trace_array_get(tr) < 0)
4677                 return -ENODEV;
4678
4679         if (file->f_mode & FMODE_READ) {
4680                 iter = __tracing_open(inode, file, true);
4681                 if (IS_ERR(iter))
4682                         ret = PTR_ERR(iter);
4683         } else {
4684                 /* Writes still need the seq_file to hold the private data */
4685                 ret = -ENOMEM;
4686                 m = kzalloc(sizeof(*m), GFP_KERNEL);
4687                 if (!m)
4688                         goto out;
4689                 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4690                 if (!iter) {
4691                         kfree(m);
4692                         goto out;
4693                 }
4694                 ret = 0;
4695
4696                 iter->tr = tr;
4697                 iter->trace_buffer = &tr->max_buffer;
4698                 iter->cpu_file = tracing_get_cpu(inode);
4699                 m->private = iter;
4700                 file->private_data = m;
4701         }
4702 out:
4703         if (ret < 0)
4704                 trace_array_put(tr);
4705
4706         return ret;
4707 }
4708
4709 static ssize_t
4710 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4711                        loff_t *ppos)
4712 {
4713         struct seq_file *m = filp->private_data;
4714         struct trace_iterator *iter = m->private;
4715         struct trace_array *tr = iter->tr;
4716         unsigned long val;
4717         int ret;
4718
4719         ret = tracing_update_buffers();
4720         if (ret < 0)
4721                 return ret;
4722
4723         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4724         if (ret)
4725                 return ret;
4726
4727         mutex_lock(&trace_types_lock);
4728
4729         if (tr->current_trace->use_max_tr) {
4730                 ret = -EBUSY;
4731                 goto out;
4732         }
4733
4734         switch (val) {
4735         case 0:
4736                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4737                         ret = -EINVAL;
4738                         break;
4739                 }
4740                 if (tr->allocated_snapshot)
4741                         free_snapshot(tr);
4742                 break;
4743         case 1:
4744 /* Only allow per-cpu swap if the ring buffer supports it */
4745 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4746                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4747                         ret = -EINVAL;
4748                         break;
4749                 }
4750 #endif
4751                 if (!tr->allocated_snapshot) {
4752                         ret = alloc_snapshot(tr);
4753                         if (ret < 0)
4754                                 break;
4755                 }
4756                 local_irq_disable();
4757                 /* Now, we're going to swap */
4758                 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4759                         update_max_tr(tr, current, smp_processor_id());
4760                 else
4761                         update_max_tr_single(tr, current, iter->cpu_file);
4762                 local_irq_enable();
4763                 break;
4764         default:
4765                 if (tr->allocated_snapshot) {
4766                         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4767                                 tracing_reset_online_cpus(&tr->max_buffer);
4768                         else
4769                                 tracing_reset(&tr->max_buffer, iter->cpu_file);
4770                 }
4771                 break;
4772         }
4773
4774         if (ret >= 0) {
4775                 *ppos += cnt;
4776                 ret = cnt;
4777         }
4778 out:
4779         mutex_unlock(&trace_types_lock);
4780         return ret;
4781 }
4782
4783 static int tracing_snapshot_release(struct inode *inode, struct file *file)
4784 {
4785         struct seq_file *m = file->private_data;
4786         int ret;
4787
4788         ret = tracing_release(inode, file);
4789
4790         if (file->f_mode & FMODE_READ)
4791                 return ret;
4792
4793         /* If write only, the seq_file is just a stub */
4794         if (m)
4795                 kfree(m->private);
4796         kfree(m);
4797
4798         return 0;
4799 }
4800
4801 static int tracing_buffers_open(struct inode *inode, struct file *filp);
4802 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4803                                     size_t count, loff_t *ppos);
4804 static int tracing_buffers_release(struct inode *inode, struct file *file);
4805 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4806                    struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4807
4808 static int snapshot_raw_open(struct inode *inode, struct file *filp)
4809 {
4810         struct ftrace_buffer_info *info;
4811         int ret;
4812
4813         ret = tracing_buffers_open(inode, filp);
4814         if (ret < 0)
4815                 return ret;
4816
4817         info = filp->private_data;
4818
4819         if (info->iter.trace->use_max_tr) {
4820                 tracing_buffers_release(inode, filp);
4821                 return -EBUSY;
4822         }
4823
4824         info->iter.snapshot = true;
4825         info->iter.trace_buffer = &info->iter.tr->max_buffer;
4826
4827         return ret;
4828 }
4829
4830 #endif /* CONFIG_TRACER_SNAPSHOT */
4831
4832
4833 static const struct file_operations tracing_max_lat_fops = {
4834         .open           = tracing_open_generic,
4835         .read           = tracing_max_lat_read,
4836         .write          = tracing_max_lat_write,
4837         .llseek         = generic_file_llseek,
4838 };
4839
4840 static const struct file_operations set_tracer_fops = {
4841         .open           = tracing_open_generic,
4842         .read           = tracing_set_trace_read,
4843         .write          = tracing_set_trace_write,
4844         .llseek         = generic_file_llseek,
4845 };
4846
4847 static const struct file_operations tracing_pipe_fops = {
4848         .open           = tracing_open_pipe,
4849         .poll           = tracing_poll_pipe,
4850         .read           = tracing_read_pipe,
4851         .splice_read    = tracing_splice_read_pipe,
4852         .release        = tracing_release_pipe,
4853         .llseek         = no_llseek,
4854 };
4855
4856 static const struct file_operations tracing_entries_fops = {
4857         .open           = tracing_open_generic_tr,
4858         .read           = tracing_entries_read,
4859         .write          = tracing_entries_write,
4860         .llseek         = generic_file_llseek,
4861         .release        = tracing_release_generic_tr,
4862 };
4863
4864 static const struct file_operations tracing_total_entries_fops = {
4865         .open           = tracing_open_generic_tr,
4866         .read           = tracing_total_entries_read,
4867         .llseek         = generic_file_llseek,
4868         .release        = tracing_release_generic_tr,
4869 };
4870
4871 static const struct file_operations tracing_free_buffer_fops = {
4872         .open           = tracing_open_generic_tr,
4873         .write          = tracing_free_buffer_write,
4874         .release        = tracing_free_buffer_release,
4875 };
4876
4877 static const struct file_operations tracing_mark_fops = {
4878         .open           = tracing_open_generic_tr,
4879         .write          = tracing_mark_write,
4880         .llseek         = generic_file_llseek,
4881         .release        = tracing_release_generic_tr,
4882 };
4883
4884 static const struct file_operations trace_clock_fops = {
4885         .open           = tracing_clock_open,
4886         .read           = seq_read,
4887         .llseek         = seq_lseek,
4888         .release        = tracing_single_release_tr,
4889         .write          = tracing_clock_write,
4890 };
4891
4892 #ifdef CONFIG_TRACER_SNAPSHOT
4893 static const struct file_operations snapshot_fops = {
4894         .open           = tracing_snapshot_open,
4895         .read           = seq_read,
4896         .write          = tracing_snapshot_write,
4897         .llseek         = tracing_seek,
4898         .release        = tracing_snapshot_release,
4899 };
4900
4901 static const struct file_operations snapshot_raw_fops = {
4902         .open           = snapshot_raw_open,
4903         .read           = tracing_buffers_read,
4904         .release        = tracing_buffers_release,
4905         .splice_read    = tracing_buffers_splice_read,
4906         .llseek         = no_llseek,
4907 };
4908
4909 #endif /* CONFIG_TRACER_SNAPSHOT */
4910
4911 static int tracing_buffers_open(struct inode *inode, struct file *filp)
4912 {
4913         struct trace_array *tr = inode->i_private;
4914         struct ftrace_buffer_info *info;
4915         int ret;
4916
4917         if (tracing_disabled)
4918                 return -ENODEV;
4919
4920         if (trace_array_get(tr) < 0)
4921                 return -ENODEV;
4922
4923         info = kzalloc(sizeof(*info), GFP_KERNEL);
4924         if (!info) {
4925                 trace_array_put(tr);
4926                 return -ENOMEM;
4927         }
4928
4929         mutex_lock(&trace_types_lock);
4930
4931         info->iter.tr           = tr;
4932         info->iter.cpu_file     = tracing_get_cpu(inode);
4933         info->iter.trace        = tr->current_trace;
4934         info->iter.trace_buffer = &tr->trace_buffer;
4935         info->spare             = NULL;
4936         /* Force reading ring buffer for first read */
4937         info->read              = (unsigned int)-1;
4938
4939         filp->private_data = info;
4940
4941         mutex_unlock(&trace_types_lock);
4942
4943         ret = nonseekable_open(inode, filp);
4944         if (ret < 0)
4945                 trace_array_put(tr);
4946
4947         return ret;
4948 }
4949
4950 static unsigned int
4951 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4952 {
4953         struct ftrace_buffer_info *info = filp->private_data;
4954         struct trace_iterator *iter = &info->iter;
4955
4956         return trace_poll(iter, filp, poll_table);
4957 }
4958
4959 static ssize_t
4960 tracing_buffers_read(struct file *filp, char __user *ubuf,
4961                      size_t count, loff_t *ppos)
4962 {
4963         struct ftrace_buffer_info *info = filp->private_data;
4964         struct trace_iterator *iter = &info->iter;
4965         ssize_t ret;
4966         ssize_t size;
4967
4968         if (!count)
4969                 return 0;
4970
4971         mutex_lock(&trace_types_lock);
4972
4973 #ifdef CONFIG_TRACER_MAX_TRACE
4974         if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4975                 size = -EBUSY;
4976                 goto out_unlock;
4977         }
4978 #endif
4979
4980         if (!info->spare)
4981                 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4982                                                           iter->cpu_file);
4983         size = -ENOMEM;
4984         if (!info->spare)
4985                 goto out_unlock;
4986
4987         /* Do we have previous read data to read? */
4988         if (info->read < PAGE_SIZE)
4989                 goto read;
4990
4991  again:
4992         trace_access_lock(iter->cpu_file);
4993         ret = ring_buffer_read_page(iter->trace_buffer->buffer,
4994                                     &info->spare,
4995                                     count,
4996                                     iter->cpu_file, 0);
4997         trace_access_unlock(iter->cpu_file);
4998
4999         if (ret < 0) {
5000                 if (trace_empty(iter)) {
5001                         if ((filp->f_flags & O_NONBLOCK)) {
5002                                 size = -EAGAIN;
5003                                 goto out_unlock;
5004                         }
5005                         mutex_unlock(&trace_types_lock);
5006                         iter->trace->wait_pipe(iter);
5007                         mutex_lock(&trace_types_lock);
5008                         if (signal_pending(current)) {
5009                                 size = -EINTR;
5010                                 goto out_unlock;
5011                         }
5012                         goto again;
5013                 }
5014                 size = 0;
5015                 goto out_unlock;
5016         }
5017
5018         info->read = 0;
5019  read:
5020         size = PAGE_SIZE - info->read;
5021         if (size > count)
5022                 size = count;
5023
5024         ret = copy_to_user(ubuf, info->spare + info->read, size);
5025         if (ret == size) {
5026                 size = -EFAULT;
5027                 goto out_unlock;
5028         }
5029         size -= ret;
5030
5031         *ppos += size;
5032         info->read += size;
5033
5034  out_unlock:
5035         mutex_unlock(&trace_types_lock);
5036
5037         return size;
5038 }
5039
5040 static int tracing_buffers_release(struct inode *inode, struct file *file)
5041 {
5042         struct ftrace_buffer_info *info = file->private_data;
5043         struct trace_iterator *iter = &info->iter;
5044
5045         mutex_lock(&trace_types_lock);
5046
5047         __trace_array_put(iter->tr);
5048
5049         if (info->spare)
5050                 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5051         kfree(info);
5052
5053         mutex_unlock(&trace_types_lock);
5054
5055         return 0;
5056 }
5057
5058 struct buffer_ref {
5059         struct ring_buffer      *buffer;
5060         void                    *page;
5061         int                     ref;
5062 };
5063
5064 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5065                                     struct pipe_buffer *buf)
5066 {
5067         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5068
5069         if (--ref->ref)
5070                 return;
5071
5072         ring_buffer_free_read_page(ref->buffer, ref->page);
5073         kfree(ref);
5074         buf->private = 0;
5075 }
5076
5077 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5078                                 struct pipe_buffer *buf)
5079 {
5080         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5081
5082         ref->ref++;
5083 }
5084
5085 /* Pipe buffer operations for a buffer. */
5086 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5087         .can_merge              = 0,
5088         .map                    = generic_pipe_buf_map,
5089         .unmap                  = generic_pipe_buf_unmap,
5090         .confirm                = generic_pipe_buf_confirm,
5091         .release                = buffer_pipe_buf_release,
5092         .steal                  = generic_pipe_buf_steal,
5093         .get                    = buffer_pipe_buf_get,
5094 };
5095
5096 /*
5097  * Callback from splice_to_pipe(), if we need to release some pages
5098  * at the end of the spd in case we error'ed out in filling the pipe.
5099  */
5100 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5101 {
5102         struct buffer_ref *ref =
5103                 (struct buffer_ref *)spd->partial[i].private;
5104
5105         if (--ref->ref)
5106                 return;
5107
5108         ring_buffer_free_read_page(ref->buffer, ref->page);
5109         kfree(ref);
5110         spd->partial[i].private = 0;
5111 }
5112
5113 static ssize_t
5114 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5115                             struct pipe_inode_info *pipe, size_t len,
5116                             unsigned int flags)
5117 {
5118         struct ftrace_buffer_info *info = file->private_data;
5119         struct trace_iterator *iter = &info->iter;
5120         struct partial_page partial_def[PIPE_DEF_BUFFERS];
5121         struct page *pages_def[PIPE_DEF_BUFFERS];
5122         struct splice_pipe_desc spd = {
5123                 .pages          = pages_def,
5124                 .partial        = partial_def,
5125                 .nr_pages_max   = PIPE_DEF_BUFFERS,
5126                 .flags          = flags,
5127                 .ops            = &buffer_pipe_buf_ops,
5128                 .spd_release    = buffer_spd_release,
5129         };
5130         struct buffer_ref *ref;
5131         int entries, size, i;
5132         ssize_t ret;
5133
5134         mutex_lock(&trace_types_lock);
5135
5136 #ifdef CONFIG_TRACER_MAX_TRACE
5137         if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5138                 ret = -EBUSY;
5139                 goto out;
5140         }
5141 #endif
5142
5143         if (splice_grow_spd(pipe, &spd)) {
5144                 ret = -ENOMEM;
5145                 goto out;
5146         }
5147
5148         if (*ppos & (PAGE_SIZE - 1)) {
5149                 ret = -EINVAL;
5150                 goto out;
5151         }
5152
5153         if (len & (PAGE_SIZE - 1)) {
5154                 if (len < PAGE_SIZE) {
5155                         ret = -EINVAL;
5156                         goto out;
5157                 }
5158                 len &= PAGE_MASK;
5159         }
5160
5161  again:
5162         trace_access_lock(iter->cpu_file);
5163         entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5164
5165         for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
5166                 struct page *page;
5167                 int r;
5168
5169                 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5170                 if (!ref)
5171                         break;
5172
5173                 ref->ref = 1;
5174                 ref->buffer = iter->trace_buffer->buffer;
5175                 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5176                 if (!ref->page) {
5177                         kfree(ref);
5178                         break;
5179                 }
5180
5181                 r = ring_buffer_read_page(ref->buffer, &ref->page,
5182                                           len, iter->cpu_file, 1);
5183                 if (r < 0) {
5184                         ring_buffer_free_read_page(ref->buffer, ref->page);
5185                         kfree(ref);
5186                         break;
5187                 }
5188
5189                 /*
5190                  * zero out any left over data, this is going to
5191                  * user land.
5192                  */
5193                 size = ring_buffer_page_len(ref->page);
5194                 if (size < PAGE_SIZE)
5195                         memset(ref->page + size, 0, PAGE_SIZE - size);
5196
5197                 page = virt_to_page(ref->page);
5198
5199                 spd.pages[i] = page;
5200                 spd.partial[i].len = PAGE_SIZE;
5201                 spd.partial[i].offset = 0;
5202                 spd.partial[i].private = (unsigned long)ref;
5203                 spd.nr_pages++;
5204                 *ppos += PAGE_SIZE;
5205
5206                 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5207         }
5208
5209         trace_access_unlock(iter->cpu_file);
5210         spd.nr_pages = i;
5211
5212         /* did we read anything? */
5213         if (!spd.nr_pages) {
5214                 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5215                         ret = -EAGAIN;
5216                         goto out;
5217                 }
5218                 mutex_unlock(&trace_types_lock);
5219                 iter->trace->wait_pipe(iter);
5220                 mutex_lock(&trace_types_lock);
5221                 if (signal_pending(current)) {
5222                         ret = -EINTR;
5223                         goto out;
5224                 }
5225                 goto again;
5226         }
5227
5228         ret = splice_to_pipe(pipe, &spd);
5229         splice_shrink_spd(&spd);
5230 out:
5231         mutex_unlock(&trace_types_lock);
5232
5233         return ret;
5234 }
5235
5236 static const struct file_operations tracing_buffers_fops = {
5237         .open           = tracing_buffers_open,
5238         .read           = tracing_buffers_read,
5239         .poll           = tracing_buffers_poll,
5240         .release        = tracing_buffers_release,
5241         .splice_read    = tracing_buffers_splice_read,
5242         .llseek         = no_llseek,
5243 };
5244
5245 static ssize_t
5246 tracing_stats_read(struct file *filp, char __user *ubuf,
5247                    size_t count, loff_t *ppos)
5248 {
5249         struct inode *inode = file_inode(filp);
5250         struct trace_array *tr = inode->i_private;
5251         struct trace_buffer *trace_buf = &tr->trace_buffer;
5252         int cpu = tracing_get_cpu(inode);
5253         struct trace_seq *s;
5254         unsigned long cnt;
5255         unsigned long long t;
5256         unsigned long usec_rem;
5257
5258         s = kmalloc(sizeof(*s), GFP_KERNEL);
5259         if (!s)
5260                 return -ENOMEM;
5261
5262         trace_seq_init(s);
5263
5264         cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5265         trace_seq_printf(s, "entries: %ld\n", cnt);
5266
5267         cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5268         trace_seq_printf(s, "overrun: %ld\n", cnt);
5269
5270         cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5271         trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5272
5273         cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5274         trace_seq_printf(s, "bytes: %ld\n", cnt);
5275
5276         if (trace_clocks[tr->clock_id].in_ns) {
5277                 /* local or global for trace_clock */
5278                 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5279                 usec_rem = do_div(t, USEC_PER_SEC);
5280                 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5281                                                                 t, usec_rem);
5282
5283                 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5284                 usec_rem = do_div(t, USEC_PER_SEC);
5285                 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5286         } else {
5287                 /* counter or tsc mode for trace_clock */
5288                 trace_seq_printf(s, "oldest event ts: %llu\n",
5289                                 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5290
5291                 trace_seq_printf(s, "now ts: %llu\n",
5292                                 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5293         }
5294
5295         cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5296         trace_seq_printf(s, "dropped events: %ld\n", cnt);
5297
5298         cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5299         trace_seq_printf(s, "read events: %ld\n", cnt);
5300
5301         count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5302
5303         kfree(s);
5304
5305         return count;
5306 }
5307
5308 static const struct file_operations tracing_stats_fops = {
5309         .open           = tracing_open_generic_tr,
5310         .read           = tracing_stats_read,
5311         .llseek         = generic_file_llseek,
5312         .release        = tracing_release_generic_tr,
5313 };
5314
5315 #ifdef CONFIG_DYNAMIC_FTRACE
5316
5317 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5318 {
5319         return 0;
5320 }
5321
5322 static ssize_t
5323 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5324                   size_t cnt, loff_t *ppos)
5325 {
5326         static char ftrace_dyn_info_buffer[1024];
5327         static DEFINE_MUTEX(dyn_info_mutex);
5328         unsigned long *p = filp->private_data;
5329         char *buf = ftrace_dyn_info_buffer;
5330         int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5331         int r;
5332
5333         mutex_lock(&dyn_info_mutex);
5334         r = sprintf(buf, "%ld ", *p);
5335
5336         r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5337         buf[r++] = '\n';
5338
5339         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5340
5341         mutex_unlock(&dyn_info_mutex);
5342
5343         return r;
5344 }
5345
5346 static const struct file_operations tracing_dyn_info_fops = {
5347         .open           = tracing_open_generic,
5348         .read           = tracing_read_dyn_info,
5349         .llseek         = generic_file_llseek,
5350 };
5351 #endif /* CONFIG_DYNAMIC_FTRACE */
5352
5353 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5354 static void
5355 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5356 {
5357         tracing_snapshot();
5358 }
5359
5360 static void
5361 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5362 {
5363         unsigned long *count = (long *)data;
5364
5365         if (!*count)
5366                 return;
5367
5368         if (*count != -1)
5369                 (*count)--;
5370
5371         tracing_snapshot();
5372 }
5373
5374 static int
5375 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5376                       struct ftrace_probe_ops *ops, void *data)
5377 {
5378         long count = (long)data;
5379
5380         seq_printf(m, "%ps:", (void *)ip);
5381
5382         seq_printf(m, "snapshot");
5383
5384         if (count == -1)
5385                 seq_printf(m, ":unlimited\n");
5386         else
5387                 seq_printf(m, ":count=%ld\n", count);
5388
5389         return 0;
5390 }
5391
5392 static struct ftrace_probe_ops snapshot_probe_ops = {
5393         .func                   = ftrace_snapshot,
5394         .print                  = ftrace_snapshot_print,
5395 };
5396
5397 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5398         .func                   = ftrace_count_snapshot,
5399         .print                  = ftrace_snapshot_print,
5400 };
5401
5402 static int
5403 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5404                                char *glob, char *cmd, char *param, int enable)
5405 {
5406         struct ftrace_probe_ops *ops;
5407         void *count = (void *)-1;
5408         char *number;
5409         int ret;
5410
5411         /* hash funcs only work with set_ftrace_filter */
5412         if (!enable)
5413                 return -EINVAL;
5414
5415         ops = param ? &snapshot_count_probe_ops :  &snapshot_probe_ops;
5416
5417         if (glob[0] == '!') {
5418                 unregister_ftrace_function_probe_func(glob+1, ops);
5419                 return 0;
5420         }
5421
5422         if (!param)
5423                 goto out_reg;
5424
5425         number = strsep(&param, ":");
5426
5427         if (!strlen(number))
5428                 goto out_reg;
5429
5430         /*
5431          * We use the callback data field (which is a pointer)
5432          * as our counter.
5433          */
5434         ret = kstrtoul(number, 0, (unsigned long *)&count);
5435         if (ret)
5436                 return ret;
5437
5438  out_reg:
5439         ret = register_ftrace_function_probe(glob, ops, count);
5440
5441         if (ret >= 0)
5442                 alloc_snapshot(&global_trace);
5443
5444         return ret < 0 ? ret : 0;
5445 }
5446
5447 static struct ftrace_func_command ftrace_snapshot_cmd = {
5448         .name                   = "snapshot",
5449         .func                   = ftrace_trace_snapshot_callback,
5450 };
5451
5452 static int register_snapshot_cmd(void)
5453 {
5454         return register_ftrace_command(&ftrace_snapshot_cmd);
5455 }
5456 #else
5457 static inline int register_snapshot_cmd(void) { return 0; }
5458 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5459
5460 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5461 {
5462         if (tr->dir)
5463                 return tr->dir;
5464
5465         if (!debugfs_initialized())
5466                 return NULL;
5467
5468         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5469                 tr->dir = debugfs_create_dir("tracing", NULL);
5470
5471         if (!tr->dir)
5472                 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5473
5474         return tr->dir;
5475 }
5476
5477 struct dentry *tracing_init_dentry(void)
5478 {
5479         return tracing_init_dentry_tr(&global_trace);
5480 }
5481
5482 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5483 {
5484         struct dentry *d_tracer;
5485
5486         if (tr->percpu_dir)
5487                 return tr->percpu_dir;
5488
5489         d_tracer = tracing_init_dentry_tr(tr);
5490         if (!d_tracer)
5491                 return NULL;
5492
5493         tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5494
5495         WARN_ONCE(!tr->percpu_dir,
5496                   "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5497
5498         return tr->percpu_dir;
5499 }
5500
5501 static struct dentry *
5502 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5503                       void *data, long cpu, const struct file_operations *fops)
5504 {
5505         struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5506
5507         if (ret) /* See tracing_get_cpu() */
5508                 ret->d_inode->i_cdev = (void *)(cpu + 1);
5509         return ret;
5510 }
5511
5512 static void
5513 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5514 {
5515         struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5516         struct dentry *d_cpu;
5517         char cpu_dir[30]; /* 30 characters should be more than enough */
5518
5519         if (!d_percpu)
5520                 return;
5521
5522         snprintf(cpu_dir, 30, "cpu%ld", cpu);
5523         d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5524         if (!d_cpu) {
5525                 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5526                 return;
5527         }
5528
5529         /* per cpu trace_pipe */
5530         trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5531                                 tr, cpu, &tracing_pipe_fops);
5532
5533         /* per cpu trace */
5534         trace_create_cpu_file("trace", 0644, d_cpu,
5535                                 tr, cpu, &tracing_fops);
5536
5537         trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5538                                 tr, cpu, &tracing_buffers_fops);
5539
5540         trace_create_cpu_file("stats", 0444, d_cpu,
5541                                 tr, cpu, &tracing_stats_fops);
5542
5543         trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5544                                 tr, cpu, &tracing_entries_fops);
5545
5546 #ifdef CONFIG_TRACER_SNAPSHOT
5547         trace_create_cpu_file("snapshot", 0644, d_cpu,
5548                                 tr, cpu, &snapshot_fops);
5549
5550         trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5551                                 tr, cpu, &snapshot_raw_fops);
5552 #endif
5553 }
5554
5555 #ifdef CONFIG_FTRACE_SELFTEST
5556 /* Let selftest have access to static functions in this file */
5557 #include "trace_selftest.c"
5558 #endif
5559
5560 struct trace_option_dentry {
5561         struct tracer_opt               *opt;
5562         struct tracer_flags             *flags;
5563         struct trace_array              *tr;
5564         struct dentry                   *entry;
5565 };
5566
5567 static ssize_t
5568 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5569                         loff_t *ppos)
5570 {
5571         struct trace_option_dentry *topt = filp->private_data;
5572         char *buf;
5573
5574         if (topt->flags->val & topt->opt->bit)
5575                 buf = "1\n";
5576         else
5577                 buf = "0\n";
5578
5579         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5580 }
5581
5582 static ssize_t
5583 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5584                          loff_t *ppos)
5585 {
5586         struct trace_option_dentry *topt = filp->private_data;
5587         unsigned long val;
5588         int ret;
5589
5590         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5591         if (ret)
5592                 return ret;
5593
5594         if (val != 0 && val != 1)
5595                 return -EINVAL;
5596
5597         if (!!(topt->flags->val & topt->opt->bit) != val) {
5598                 mutex_lock(&trace_types_lock);
5599                 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
5600                                           topt->opt, !val);
5601                 mutex_unlock(&trace_types_lock);
5602                 if (ret)
5603                         return ret;
5604         }
5605
5606         *ppos += cnt;
5607
5608         return cnt;
5609 }
5610
5611
5612 static const struct file_operations trace_options_fops = {
5613         .open = tracing_open_generic,
5614         .read = trace_options_read,
5615         .write = trace_options_write,
5616         .llseek = generic_file_llseek,
5617 };
5618
5619 static ssize_t
5620 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5621                         loff_t *ppos)
5622 {
5623         long index = (long)filp->private_data;
5624         char *buf;
5625
5626         if (trace_flags & (1 << index))
5627                 buf = "1\n";
5628         else
5629                 buf = "0\n";
5630
5631         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5632 }
5633
5634 static ssize_t
5635 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5636                          loff_t *ppos)
5637 {
5638         struct trace_array *tr = &global_trace;
5639         long index = (long)filp->private_data;
5640         unsigned long val;
5641         int ret;
5642
5643         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5644         if (ret)
5645                 return ret;
5646
5647         if (val != 0 && val != 1)
5648                 return -EINVAL;
5649
5650         mutex_lock(&trace_types_lock);
5651         ret = set_tracer_flag(tr, 1 << index, val);
5652         mutex_unlock(&trace_types_lock);
5653
5654         if (ret < 0)
5655                 return ret;
5656
5657         *ppos += cnt;
5658
5659         return cnt;
5660 }
5661
5662 static const struct file_operations trace_options_core_fops = {
5663         .open = tracing_open_generic,
5664         .read = trace_options_core_read,
5665         .write = trace_options_core_write,
5666         .llseek = generic_file_llseek,
5667 };
5668
5669 struct dentry *trace_create_file(const char *name,
5670                                  umode_t mode,
5671                                  struct dentry *parent,
5672                                  void *data,
5673                                  const struct file_operations *fops)
5674 {
5675         struct dentry *ret;
5676
5677         ret = debugfs_create_file(name, mode, parent, data, fops);
5678         if (!ret)
5679                 pr_warning("Could not create debugfs '%s' entry\n", name);
5680
5681         return ret;
5682 }
5683
5684
5685 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
5686 {
5687         struct dentry *d_tracer;
5688
5689         if (tr->options)
5690                 return tr->options;
5691
5692         d_tracer = tracing_init_dentry_tr(tr);
5693         if (!d_tracer)
5694                 return NULL;
5695
5696         tr->options = debugfs_create_dir("options", d_tracer);
5697         if (!tr->options) {
5698                 pr_warning("Could not create debugfs directory 'options'\n");
5699                 return NULL;
5700         }
5701
5702         return tr->options;
5703 }
5704
5705 static void
5706 create_trace_option_file(struct trace_array *tr,
5707                          struct trace_option_dentry *topt,
5708                          struct tracer_flags *flags,
5709                          struct tracer_opt *opt)
5710 {
5711         struct dentry *t_options;
5712
5713         t_options = trace_options_init_dentry(tr);
5714         if (!t_options)
5715                 return;
5716
5717         topt->flags = flags;
5718         topt->opt = opt;
5719         topt->tr = tr;
5720
5721         topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
5722                                     &trace_options_fops);
5723
5724 }
5725
5726 static struct trace_option_dentry *
5727 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
5728 {
5729         struct trace_option_dentry *topts;
5730         struct tracer_flags *flags;
5731         struct tracer_opt *opts;
5732         int cnt;
5733
5734         if (!tracer)
5735                 return NULL;
5736
5737         flags = tracer->flags;
5738
5739         if (!flags || !flags->opts)
5740                 return NULL;
5741
5742         opts = flags->opts;
5743
5744         for (cnt = 0; opts[cnt].name; cnt++)
5745                 ;
5746
5747         topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
5748         if (!topts)
5749                 return NULL;
5750
5751         for (cnt = 0; opts[cnt].name; cnt++)
5752                 create_trace_option_file(tr, &topts[cnt], flags,
5753                                          &opts[cnt]);
5754
5755         return topts;
5756 }
5757
5758 static void
5759 destroy_trace_option_files(struct trace_option_dentry *topts)
5760 {
5761         int cnt;
5762
5763         if (!topts)
5764                 return;
5765
5766         for (cnt = 0; topts[cnt].opt; cnt++) {
5767                 if (topts[cnt].entry)
5768                         debugfs_remove(topts[cnt].entry);
5769         }
5770
5771         kfree(topts);
5772 }
5773
5774 static struct dentry *
5775 create_trace_option_core_file(struct trace_array *tr,
5776                               const char *option, long index)
5777 {
5778         struct dentry *t_options;
5779
5780         t_options = trace_options_init_dentry(tr);
5781         if (!t_options)
5782                 return NULL;
5783
5784         return trace_create_file(option, 0644, t_options, (void *)index,
5785                                     &trace_options_core_fops);
5786 }
5787
5788 static __init void create_trace_options_dir(struct trace_array *tr)
5789 {
5790         struct dentry *t_options;
5791         int i;
5792
5793         t_options = trace_options_init_dentry(tr);
5794         if (!t_options)
5795                 return;
5796
5797         for (i = 0; trace_options[i]; i++)
5798                 create_trace_option_core_file(tr, trace_options[i], i);
5799 }
5800
5801 static ssize_t
5802 rb_simple_read(struct file *filp, char __user *ubuf,
5803                size_t cnt, loff_t *ppos)
5804 {
5805         struct trace_array *tr = filp->private_data;
5806         char buf[64];
5807         int r;
5808
5809         r = tracer_tracing_is_on(tr);
5810         r = sprintf(buf, "%d\n", r);
5811
5812         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5813 }
5814
5815 static ssize_t
5816 rb_simple_write(struct file *filp, const char __user *ubuf,
5817                 size_t cnt, loff_t *ppos)
5818 {
5819         struct trace_array *tr = filp->private_data;
5820         struct ring_buffer *buffer = tr->trace_buffer.buffer;
5821         unsigned long val;
5822         int ret;
5823
5824         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5825         if (ret)
5826                 return ret;
5827
5828         if (buffer) {
5829                 mutex_lock(&trace_types_lock);
5830                 if (val) {
5831                         tracer_tracing_on(tr);
5832                         if (tr->current_trace->start)
5833                                 tr->current_trace->start(tr);
5834                 } else {
5835                         tracer_tracing_off(tr);
5836                         if (tr->current_trace->stop)
5837                                 tr->current_trace->stop(tr);
5838                 }
5839                 mutex_unlock(&trace_types_lock);
5840         }
5841
5842         (*ppos)++;
5843
5844         return cnt;
5845 }
5846
5847 static const struct file_operations rb_simple_fops = {
5848         .open           = tracing_open_generic_tr,
5849         .read           = rb_simple_read,
5850         .write          = rb_simple_write,
5851         .release        = tracing_release_generic_tr,
5852         .llseek         = default_llseek,
5853 };
5854
5855 struct dentry *trace_instance_dir;
5856
5857 static void
5858 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5859
5860 static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
5861 {
5862         int cpu;
5863
5864         for_each_tracing_cpu(cpu) {
5865                 memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
5866                 per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
5867                 per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
5868         }
5869 }
5870
5871 static int
5872 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
5873 {
5874         enum ring_buffer_flags rb_flags;
5875
5876         rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5877
5878         buf->buffer = ring_buffer_alloc(size, rb_flags);
5879         if (!buf->buffer)
5880                 return -ENOMEM;
5881
5882         buf->data = alloc_percpu(struct trace_array_cpu);
5883         if (!buf->data) {
5884                 ring_buffer_free(buf->buffer);
5885                 return -ENOMEM;
5886         }
5887
5888         init_trace_buffers(tr, buf);
5889
5890         /* Allocate the first page for all buffers */
5891         set_buffer_entries(&tr->trace_buffer,
5892                            ring_buffer_size(tr->trace_buffer.buffer, 0));
5893
5894         return 0;
5895 }
5896
5897 static int allocate_trace_buffers(struct trace_array *tr, int size)
5898 {
5899         int ret;
5900
5901         ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5902         if (ret)
5903                 return ret;
5904
5905 #ifdef CONFIG_TRACER_MAX_TRACE
5906         ret = allocate_trace_buffer(tr, &tr->max_buffer,
5907                                     allocate_snapshot ? size : 1);
5908         if (WARN_ON(ret)) {
5909                 ring_buffer_free(tr->trace_buffer.buffer);
5910                 free_percpu(tr->trace_buffer.data);
5911                 return -ENOMEM;
5912         }
5913         tr->allocated_snapshot = allocate_snapshot;
5914
5915         /*
5916          * Only the top level trace array gets its snapshot allocated
5917          * from the kernel command line.
5918          */
5919         allocate_snapshot = false;
5920 #endif
5921         return 0;
5922 }
5923
5924 static int new_instance_create(const char *name)
5925 {
5926         struct trace_array *tr;
5927         int ret;
5928
5929         mutex_lock(&trace_types_lock);
5930
5931         ret = -EEXIST;
5932         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5933                 if (tr->name && strcmp(tr->name, name) == 0)
5934                         goto out_unlock;
5935         }
5936
5937         ret = -ENOMEM;
5938         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5939         if (!tr)
5940                 goto out_unlock;
5941
5942         tr->name = kstrdup(name, GFP_KERNEL);
5943         if (!tr->name)
5944                 goto out_free_tr;
5945
5946         raw_spin_lock_init(&tr->start_lock);
5947
5948         tr->current_trace = &nop_trace;
5949
5950         INIT_LIST_HEAD(&tr->systems);
5951         INIT_LIST_HEAD(&tr->events);
5952
5953         if (allocate_trace_buffers(tr, trace_buf_size) < 0)
5954                 goto out_free_tr;
5955
5956         /* Holder for file callbacks */
5957         tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5958         tr->trace_cpu.tr = tr;
5959
5960         tr->dir = debugfs_create_dir(name, trace_instance_dir);
5961         if (!tr->dir)
5962                 goto out_free_tr;
5963
5964         ret = event_trace_add_tracer(tr->dir, tr);
5965         if (ret) {
5966                 debugfs_remove_recursive(tr->dir);
5967                 goto out_free_tr;
5968         }
5969
5970         init_tracer_debugfs(tr, tr->dir);
5971
5972         list_add(&tr->list, &ftrace_trace_arrays);
5973
5974         mutex_unlock(&trace_types_lock);
5975
5976         return 0;
5977
5978  out_free_tr:
5979         if (tr->trace_buffer.buffer)
5980                 ring_buffer_free(tr->trace_buffer.buffer);
5981         kfree(tr->name);
5982         kfree(tr);
5983
5984  out_unlock:
5985         mutex_unlock(&trace_types_lock);
5986
5987         return ret;
5988
5989 }
5990
5991 static int instance_delete(const char *name)
5992 {
5993         struct trace_array *tr;
5994         int found = 0;
5995         int ret;
5996
5997         mutex_lock(&trace_types_lock);
5998
5999         ret = -ENODEV;
6000         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6001                 if (tr->name && strcmp(tr->name, name) == 0) {
6002                         found = 1;
6003                         break;
6004                 }
6005         }
6006         if (!found)
6007                 goto out_unlock;
6008
6009         ret = -EBUSY;
6010         if (tr->ref)
6011                 goto out_unlock;
6012
6013         list_del(&tr->list);
6014
6015         event_trace_del_tracer(tr);
6016         debugfs_remove_recursive(tr->dir);
6017         free_percpu(tr->trace_buffer.data);
6018         ring_buffer_free(tr->trace_buffer.buffer);
6019
6020         kfree(tr->name);
6021         kfree(tr);
6022
6023         ret = 0;
6024
6025  out_unlock:
6026         mutex_unlock(&trace_types_lock);
6027
6028         return ret;
6029 }
6030
6031 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6032 {
6033         struct dentry *parent;
6034         int ret;
6035
6036         /* Paranoid: Make sure the parent is the "instances" directory */
6037         parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6038         if (WARN_ON_ONCE(parent != trace_instance_dir))
6039                 return -ENOENT;
6040
6041         /*
6042          * The inode mutex is locked, but debugfs_create_dir() will also
6043          * take the mutex. As the instances directory can not be destroyed
6044          * or changed in any other way, it is safe to unlock it, and
6045          * let the dentry try. If two users try to make the same dir at
6046          * the same time, then the new_instance_create() will determine the
6047          * winner.
6048          */
6049         mutex_unlock(&inode->i_mutex);
6050
6051         ret = new_instance_create(dentry->d_iname);
6052
6053         mutex_lock(&inode->i_mutex);
6054
6055         return ret;
6056 }
6057
6058 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6059 {
6060         struct dentry *parent;
6061         int ret;
6062
6063         /* Paranoid: Make sure the parent is the "instances" directory */
6064         parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6065         if (WARN_ON_ONCE(parent != trace_instance_dir))
6066                 return -ENOENT;
6067
6068         /* The caller did a dget() on dentry */
6069         mutex_unlock(&dentry->d_inode->i_mutex);
6070
6071         /*
6072          * The inode mutex is locked, but debugfs_create_dir() will also
6073          * take the mutex. As the instances directory can not be destroyed
6074          * or changed in any other way, it is safe to unlock it, and
6075          * let the dentry try. If two users try to make the same dir at
6076          * the same time, then the instance_delete() will determine the
6077          * winner.
6078          */
6079         mutex_unlock(&inode->i_mutex);
6080
6081         ret = instance_delete(dentry->d_iname);
6082
6083         mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6084         mutex_lock(&dentry->d_inode->i_mutex);
6085
6086         return ret;
6087 }
6088
6089 static const struct inode_operations instance_dir_inode_operations = {
6090         .lookup         = simple_lookup,
6091         .mkdir          = instance_mkdir,
6092         .rmdir          = instance_rmdir,
6093 };
6094
6095 static __init void create_trace_instances(struct dentry *d_tracer)
6096 {
6097         trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6098         if (WARN_ON(!trace_instance_dir))
6099                 return;
6100
6101         /* Hijack the dir inode operations, to allow mkdir */
6102         trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6103 }
6104
6105 static void
6106 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6107 {
6108         int cpu;
6109
6110         trace_create_file("trace_options", 0644, d_tracer,
6111                           tr, &tracing_iter_fops);
6112
6113         trace_create_file("trace", 0644, d_tracer,
6114                           tr, &tracing_fops);
6115
6116         trace_create_file("trace_pipe", 0444, d_tracer,
6117                           tr, &tracing_pipe_fops);
6118
6119         trace_create_file("buffer_size_kb", 0644, d_tracer,
6120                           tr, &tracing_entries_fops);
6121
6122         trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6123                           tr, &tracing_total_entries_fops);
6124
6125         trace_create_file("free_buffer", 0644, d_tracer,
6126                           tr, &tracing_free_buffer_fops);
6127
6128         trace_create_file("trace_marker", 0220, d_tracer,
6129                           tr, &tracing_mark_fops);
6130
6131         trace_create_file("trace_clock", 0644, d_tracer, tr,
6132                           &trace_clock_fops);
6133
6134         trace_create_file("tracing_on", 0644, d_tracer,
6135                           tr, &rb_simple_fops);
6136
6137 #ifdef CONFIG_TRACER_SNAPSHOT
6138         trace_create_file("snapshot", 0644, d_tracer,
6139                           tr, &snapshot_fops);
6140 #endif
6141
6142         for_each_tracing_cpu(cpu)
6143                 tracing_init_debugfs_percpu(tr, cpu);
6144
6145 }
6146
6147 static __init int tracer_init_debugfs(void)
6148 {
6149         struct dentry *d_tracer;
6150
6151         trace_access_lock_init();
6152
6153         d_tracer = tracing_init_dentry();
6154         if (!d_tracer)
6155                 return 0;
6156
6157         init_tracer_debugfs(&global_trace, d_tracer);
6158
6159         trace_create_file("tracing_cpumask", 0644, d_tracer,
6160                         &global_trace, &tracing_cpumask_fops);
6161
6162         trace_create_file("available_tracers", 0444, d_tracer,
6163                         &global_trace, &show_traces_fops);
6164
6165         trace_create_file("current_tracer", 0644, d_tracer,
6166                         &global_trace, &set_tracer_fops);
6167
6168 #ifdef CONFIG_TRACER_MAX_TRACE
6169         trace_create_file("tracing_max_latency", 0644, d_tracer,
6170                         &tracing_max_latency, &tracing_max_lat_fops);
6171 #endif
6172
6173         trace_create_file("tracing_thresh", 0644, d_tracer,
6174                         &tracing_thresh, &tracing_max_lat_fops);
6175
6176         trace_create_file("README", 0444, d_tracer,
6177                         NULL, &tracing_readme_fops);
6178
6179         trace_create_file("saved_cmdlines", 0444, d_tracer,
6180                         NULL, &tracing_saved_cmdlines_fops);
6181
6182 #ifdef CONFIG_DYNAMIC_FTRACE
6183         trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6184                         &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6185 #endif
6186
6187         create_trace_instances(d_tracer);
6188
6189         create_trace_options_dir(&global_trace);
6190
6191         return 0;
6192 }
6193
6194 static int trace_panic_handler(struct notifier_block *this,
6195                                unsigned long event, void *unused)
6196 {
6197         if (ftrace_dump_on_oops)
6198                 ftrace_dump(ftrace_dump_on_oops);
6199         return NOTIFY_OK;
6200 }
6201
6202 static struct notifier_block trace_panic_notifier = {
6203         .notifier_call  = trace_panic_handler,
6204         .next           = NULL,
6205         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
6206 };
6207
6208 static int trace_die_handler(struct notifier_block *self,
6209                              unsigned long val,
6210                              void *data)
6211 {
6212         switch (val) {
6213         case DIE_OOPS:
6214                 if (ftrace_dump_on_oops)
6215                         ftrace_dump(ftrace_dump_on_oops);
6216                 break;
6217         default:
6218                 break;
6219         }
6220         return NOTIFY_OK;
6221 }
6222
6223 static struct notifier_block trace_die_notifier = {
6224         .notifier_call = trace_die_handler,
6225         .priority = 200
6226 };
6227
6228 /*
6229  * printk is set to max of 1024, we really don't need it that big.
6230  * Nothing should be printing 1000 characters anyway.
6231  */
6232 #define TRACE_MAX_PRINT         1000
6233
6234 /*
6235  * Define here KERN_TRACE so that we have one place to modify
6236  * it if we decide to change what log level the ftrace dump
6237  * should be at.
6238  */
6239 #define KERN_TRACE              KERN_EMERG
6240
6241 void
6242 trace_printk_seq(struct trace_seq *s)
6243 {
6244         /* Probably should print a warning here. */
6245         if (s->len >= TRACE_MAX_PRINT)
6246                 s->len = TRACE_MAX_PRINT;
6247
6248         /* should be zero ended, but we are paranoid. */
6249         s->buffer[s->len] = 0;
6250
6251         printk(KERN_TRACE "%s", s->buffer);
6252
6253         trace_seq_init(s);
6254 }
6255
6256 void trace_init_global_iter(struct trace_iterator *iter)
6257 {
6258         iter->tr = &global_trace;
6259         iter->trace = iter->tr->current_trace;
6260         iter->cpu_file = RING_BUFFER_ALL_CPUS;
6261         iter->trace_buffer = &global_trace.trace_buffer;
6262 }
6263
6264 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6265 {
6266         /* use static because iter can be a bit big for the stack */
6267         static struct trace_iterator iter;
6268         static atomic_t dump_running;
6269         unsigned int old_userobj;
6270         unsigned long flags;
6271         int cnt = 0, cpu;
6272
6273         /* Only allow one dump user at a time. */
6274         if (atomic_inc_return(&dump_running) != 1) {
6275                 atomic_dec(&dump_running);
6276                 return;
6277         }
6278
6279         /*
6280          * Always turn off tracing when we dump.
6281          * We don't need to show trace output of what happens
6282          * between multiple crashes.
6283          *
6284          * If the user does a sysrq-z, then they can re-enable
6285          * tracing with echo 1 > tracing_on.
6286          */
6287         tracing_off();
6288
6289         local_irq_save(flags);
6290
6291         /* Simulate the iterator */
6292         trace_init_global_iter(&iter);
6293
6294         for_each_tracing_cpu(cpu) {
6295                 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6296         }
6297
6298         old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6299
6300         /* don't look at user memory in panic mode */
6301         trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6302
6303         switch (oops_dump_mode) {
6304         case DUMP_ALL:
6305                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6306                 break;
6307         case DUMP_ORIG:
6308                 iter.cpu_file = raw_smp_processor_id();
6309                 break;
6310         case DUMP_NONE:
6311                 goto out_enable;
6312         default:
6313                 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6314                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6315         }
6316
6317         printk(KERN_TRACE "Dumping ftrace buffer:\n");
6318
6319         /* Did function tracer already get disabled? */
6320         if (ftrace_is_dead()) {
6321                 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6322                 printk("#          MAY BE MISSING FUNCTION EVENTS\n");
6323         }
6324
6325         /*
6326          * We need to stop all tracing on all CPUS to read the
6327          * the next buffer. This is a bit expensive, but is
6328          * not done often. We fill all what we can read,
6329          * and then release the locks again.
6330          */
6331
6332         while (!trace_empty(&iter)) {
6333
6334                 if (!cnt)
6335                         printk(KERN_TRACE "---------------------------------\n");
6336
6337                 cnt++;
6338
6339                 /* reset all but tr, trace, and overruns */
6340                 memset(&iter.seq, 0,
6341                        sizeof(struct trace_iterator) -
6342                        offsetof(struct trace_iterator, seq));
6343                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6344                 iter.pos = -1;
6345
6346                 if (trace_find_next_entry_inc(&iter) != NULL) {
6347                         int ret;
6348
6349                         ret = print_trace_line(&iter);
6350                         if (ret != TRACE_TYPE_NO_CONSUME)
6351                                 trace_consume(&iter);
6352                 }
6353                 touch_nmi_watchdog();
6354
6355                 trace_printk_seq(&iter.seq);
6356         }
6357
6358         if (!cnt)
6359                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
6360         else
6361                 printk(KERN_TRACE "---------------------------------\n");
6362
6363  out_enable:
6364         trace_flags |= old_userobj;
6365
6366         for_each_tracing_cpu(cpu) {
6367                 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6368         }
6369         atomic_dec(&dump_running);
6370         local_irq_restore(flags);
6371 }
6372 EXPORT_SYMBOL_GPL(ftrace_dump);
6373
6374 __init static int tracer_alloc_buffers(void)
6375 {
6376         int ring_buf_size;
6377         int ret = -ENOMEM;
6378
6379
6380         if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6381                 goto out;
6382
6383         if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
6384                 goto out_free_buffer_mask;
6385
6386         /* Only allocate trace_printk buffers if a trace_printk exists */
6387         if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6388                 /* Must be called before global_trace.buffer is allocated */
6389                 trace_printk_init_buffers();
6390
6391         /* To save memory, keep the ring buffer size to its minimum */
6392         if (ring_buffer_expanded)
6393                 ring_buf_size = trace_buf_size;
6394         else
6395                 ring_buf_size = 1;
6396
6397         cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6398         cpumask_copy(tracing_cpumask, cpu_all_mask);
6399
6400         raw_spin_lock_init(&global_trace.start_lock);
6401
6402         /* TODO: make the number of buffers hot pluggable with CPUS */
6403         if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6404                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6405                 WARN_ON(1);
6406                 goto out_free_cpumask;
6407         }
6408
6409         if (global_trace.buffer_disabled)
6410                 tracing_off();
6411
6412         trace_init_cmdlines();
6413
6414         /*
6415          * register_tracer() might reference current_trace, so it
6416          * needs to be set before we register anything. This is
6417          * just a bootstrap of current_trace anyway.
6418          */
6419         global_trace.current_trace = &nop_trace;
6420
6421         register_tracer(&nop_trace);
6422
6423         /* All seems OK, enable tracing */
6424         tracing_disabled = 0;
6425
6426         atomic_notifier_chain_register(&panic_notifier_list,
6427                                        &trace_panic_notifier);
6428
6429         register_die_notifier(&trace_die_notifier);
6430
6431         global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6432
6433         /* Holder for file callbacks */
6434         global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6435         global_trace.trace_cpu.tr = &global_trace;
6436
6437         INIT_LIST_HEAD(&global_trace.systems);
6438         INIT_LIST_HEAD(&global_trace.events);
6439         list_add(&global_trace.list, &ftrace_trace_arrays);
6440
6441         while (trace_boot_options) {
6442                 char *option;
6443
6444                 option = strsep(&trace_boot_options, ",");
6445                 trace_set_options(&global_trace, option);
6446         }
6447
6448         register_snapshot_cmd();
6449
6450         return 0;
6451
6452 out_free_cpumask:
6453         free_percpu(global_trace.trace_buffer.data);
6454 #ifdef CONFIG_TRACER_MAX_TRACE
6455         free_percpu(global_trace.max_buffer.data);
6456 #endif
6457         free_cpumask_var(tracing_cpumask);
6458 out_free_buffer_mask:
6459         free_cpumask_var(tracing_buffer_mask);
6460 out:
6461         return ret;
6462 }
6463
6464 __init static int clear_boot_tracer(void)
6465 {
6466         /*
6467          * The default tracer at boot buffer is an init section.
6468          * This function is called in lateinit. If we did not
6469          * find the boot tracer, then clear it out, to prevent
6470          * later registration from accessing the buffer that is
6471          * about to be freed.
6472          */
6473         if (!default_bootup_tracer)
6474                 return 0;
6475
6476         printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6477                default_bootup_tracer);
6478         default_bootup_tracer = NULL;
6479
6480         return 0;
6481 }
6482
6483 early_initcall(tracer_alloc_buffers);
6484 fs_initcall(tracer_init_debugfs);
6485 late_initcall(clear_boot_tracer);