tracing: Fix potential out-of-bounds in trace_get_user()
[linux-3.10.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 Nadia Yvette Chambers
13  */
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
40 #include <linux/fs.h>
41 #include <linux/sched/rt.h>
42
43 #include "trace.h"
44 #include "trace_output.h"
45
46 /*
47  * On boot up, the ring buffer is set to the minimum size, so that
48  * we do not waste memory on systems that are not using tracing.
49  */
50 bool ring_buffer_expanded;
51
52 /*
53  * We need to change this state when a selftest is running.
54  * A selftest will lurk into the ring-buffer to count the
55  * entries inserted during the selftest although some concurrent
56  * insertions into the ring-buffer such as trace_printk could occurred
57  * at the same time, giving false positive or negative results.
58  */
59 static bool __read_mostly tracing_selftest_running;
60
61 /*
62  * If a tracer is running, we do not want to run SELFTEST.
63  */
64 bool __read_mostly tracing_selftest_disabled;
65
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
68         { }
69 };
70
71 static struct tracer_flags dummy_tracer_flags = {
72         .val = 0,
73         .opts = dummy_tracer_opt
74 };
75
76 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77 {
78         return 0;
79 }
80
81 /*
82  * To prevent the comm cache from being overwritten when no
83  * tracing is active, only save the comm when a trace event
84  * occurred.
85  */
86 static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88 /*
89  * Kill all tracing for good (never come back).
90  * It is initialized to 1 but will turn to zero if the initialization
91  * of the tracer is successful. But that is the only place that sets
92  * this back to zero.
93  */
94 static int tracing_disabled = 1;
95
96 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
97
98 cpumask_var_t __read_mostly     tracing_buffer_mask;
99
100 /*
101  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102  *
103  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104  * is set, then ftrace_dump is called. This will output the contents
105  * of the ftrace buffers to the console.  This is very useful for
106  * capturing traces that lead to crashes and outputing it to a
107  * serial console.
108  *
109  * It is default off, but you can enable it with either specifying
110  * "ftrace_dump_on_oops" in the kernel command line, or setting
111  * /proc/sys/kernel/ftrace_dump_on_oops
112  * Set 1 if you want to dump buffers of all CPUs
113  * Set 2 if you want to dump the buffer of the CPU that triggered oops
114  */
115
116 enum ftrace_dump_mode ftrace_dump_on_oops;
117
118 static int tracing_set_tracer(const char *buf);
119
120 #define MAX_TRACER_SIZE         100
121 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
122 static char *default_bootup_tracer;
123
124 static bool allocate_snapshot;
125
126 static int __init set_cmdline_ftrace(char *str)
127 {
128         strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
129         default_bootup_tracer = bootup_tracer_buf;
130         /* We are using ftrace early, expand it */
131         ring_buffer_expanded = true;
132         return 1;
133 }
134 __setup("ftrace=", set_cmdline_ftrace);
135
136 static int __init set_ftrace_dump_on_oops(char *str)
137 {
138         if (*str++ != '=' || !*str) {
139                 ftrace_dump_on_oops = DUMP_ALL;
140                 return 1;
141         }
142
143         if (!strcmp("orig_cpu", str)) {
144                 ftrace_dump_on_oops = DUMP_ORIG;
145                 return 1;
146         }
147
148         return 0;
149 }
150 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
151
152 static int __init boot_alloc_snapshot(char *str)
153 {
154         allocate_snapshot = true;
155         /* We also need the main ring buffer expanded */
156         ring_buffer_expanded = true;
157         return 1;
158 }
159 __setup("alloc_snapshot", boot_alloc_snapshot);
160
161
162 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
163 static char *trace_boot_options __initdata;
164
165 static int __init set_trace_boot_options(char *str)
166 {
167         strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
168         trace_boot_options = trace_boot_options_buf;
169         return 0;
170 }
171 __setup("trace_options=", set_trace_boot_options);
172
173 unsigned long long ns2usecs(cycle_t nsec)
174 {
175         nsec += 500;
176         do_div(nsec, 1000);
177         return nsec;
178 }
179
180 /*
181  * The global_trace is the descriptor that holds the tracing
182  * buffers for the live tracing. For each CPU, it contains
183  * a link list of pages that will store trace entries. The
184  * page descriptor of the pages in the memory is used to hold
185  * the link list by linking the lru item in the page descriptor
186  * to each of the pages in the buffer per CPU.
187  *
188  * For each active CPU there is a data field that holds the
189  * pages for the buffer for that CPU. Each CPU has the same number
190  * of pages allocated for its buffer.
191  */
192 static struct trace_array       global_trace;
193
194 LIST_HEAD(ftrace_trace_arrays);
195
196 int trace_array_get(struct trace_array *this_tr)
197 {
198         struct trace_array *tr;
199         int ret = -ENODEV;
200
201         mutex_lock(&trace_types_lock);
202         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
203                 if (tr == this_tr) {
204                         tr->ref++;
205                         ret = 0;
206                         break;
207                 }
208         }
209         mutex_unlock(&trace_types_lock);
210
211         return ret;
212 }
213
214 static void __trace_array_put(struct trace_array *this_tr)
215 {
216         WARN_ON(!this_tr->ref);
217         this_tr->ref--;
218 }
219
220 void trace_array_put(struct trace_array *this_tr)
221 {
222         mutex_lock(&trace_types_lock);
223         __trace_array_put(this_tr);
224         mutex_unlock(&trace_types_lock);
225 }
226
227 int filter_current_check_discard(struct ring_buffer *buffer,
228                                  struct ftrace_event_call *call, void *rec,
229                                  struct ring_buffer_event *event)
230 {
231         return filter_check_discard(call, rec, buffer, event);
232 }
233 EXPORT_SYMBOL_GPL(filter_current_check_discard);
234
235 cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
236 {
237         u64 ts;
238
239         /* Early boot up does not have a buffer yet */
240         if (!buf->buffer)
241                 return trace_clock_local();
242
243         ts = ring_buffer_time_stamp(buf->buffer, cpu);
244         ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
245
246         return ts;
247 }
248
249 cycle_t ftrace_now(int cpu)
250 {
251         return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
252 }
253
254 /**
255  * tracing_is_enabled - Show if global_trace has been disabled
256  *
257  * Shows if the global trace has been enabled or not. It uses the
258  * mirror flag "buffer_disabled" to be used in fast paths such as for
259  * the irqsoff tracer. But it may be inaccurate due to races. If you
260  * need to know the accurate state, use tracing_is_on() which is a little
261  * slower, but accurate.
262  */
263 int tracing_is_enabled(void)
264 {
265         /*
266          * For quick access (irqsoff uses this in fast path), just
267          * return the mirror variable of the state of the ring buffer.
268          * It's a little racy, but we don't really care.
269          */
270         smp_rmb();
271         return !global_trace.buffer_disabled;
272 }
273
274 /*
275  * trace_buf_size is the size in bytes that is allocated
276  * for a buffer. Note, the number of bytes is always rounded
277  * to page size.
278  *
279  * This number is purposely set to a low number of 16384.
280  * If the dump on oops happens, it will be much appreciated
281  * to not have to wait for all that output. Anyway this can be
282  * boot time and run time configurable.
283  */
284 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
285
286 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
287
288 /* trace_types holds a link list of available tracers. */
289 static struct tracer            *trace_types __read_mostly;
290
291 /*
292  * trace_types_lock is used to protect the trace_types list.
293  */
294 DEFINE_MUTEX(trace_types_lock);
295
296 /*
297  * serialize the access of the ring buffer
298  *
299  * ring buffer serializes readers, but it is low level protection.
300  * The validity of the events (which returns by ring_buffer_peek() ..etc)
301  * are not protected by ring buffer.
302  *
303  * The content of events may become garbage if we allow other process consumes
304  * these events concurrently:
305  *   A) the page of the consumed events may become a normal page
306  *      (not reader page) in ring buffer, and this page will be rewrited
307  *      by events producer.
308  *   B) The page of the consumed events may become a page for splice_read,
309  *      and this page will be returned to system.
310  *
311  * These primitives allow multi process access to different cpu ring buffer
312  * concurrently.
313  *
314  * These primitives don't distinguish read-only and read-consume access.
315  * Multi read-only access are also serialized.
316  */
317
318 #ifdef CONFIG_SMP
319 static DECLARE_RWSEM(all_cpu_access_lock);
320 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
321
322 static inline void trace_access_lock(int cpu)
323 {
324         if (cpu == RING_BUFFER_ALL_CPUS) {
325                 /* gain it for accessing the whole ring buffer. */
326                 down_write(&all_cpu_access_lock);
327         } else {
328                 /* gain it for accessing a cpu ring buffer. */
329
330                 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
331                 down_read(&all_cpu_access_lock);
332
333                 /* Secondly block other access to this @cpu ring buffer. */
334                 mutex_lock(&per_cpu(cpu_access_lock, cpu));
335         }
336 }
337
338 static inline void trace_access_unlock(int cpu)
339 {
340         if (cpu == RING_BUFFER_ALL_CPUS) {
341                 up_write(&all_cpu_access_lock);
342         } else {
343                 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
344                 up_read(&all_cpu_access_lock);
345         }
346 }
347
348 static inline void trace_access_lock_init(void)
349 {
350         int cpu;
351
352         for_each_possible_cpu(cpu)
353                 mutex_init(&per_cpu(cpu_access_lock, cpu));
354 }
355
356 #else
357
358 static DEFINE_MUTEX(access_lock);
359
360 static inline void trace_access_lock(int cpu)
361 {
362         (void)cpu;
363         mutex_lock(&access_lock);
364 }
365
366 static inline void trace_access_unlock(int cpu)
367 {
368         (void)cpu;
369         mutex_unlock(&access_lock);
370 }
371
372 static inline void trace_access_lock_init(void)
373 {
374 }
375
376 #endif
377
378 /* trace_flags holds trace_options default values */
379 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
380         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
381         TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
382         TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
383
384 void tracer_tracing_on(struct trace_array *tr)
385 {
386         if (tr->trace_buffer.buffer)
387                 ring_buffer_record_on(tr->trace_buffer.buffer);
388         /*
389          * This flag is looked at when buffers haven't been allocated
390          * yet, or by some tracers (like irqsoff), that just want to
391          * know if the ring buffer has been disabled, but it can handle
392          * races of where it gets disabled but we still do a record.
393          * As the check is in the fast path of the tracers, it is more
394          * important to be fast than accurate.
395          */
396         tr->buffer_disabled = 0;
397         /* Make the flag seen by readers */
398         smp_wmb();
399 }
400
401 /**
402  * tracing_on - enable tracing buffers
403  *
404  * This function enables tracing buffers that may have been
405  * disabled with tracing_off.
406  */
407 void tracing_on(void)
408 {
409         tracer_tracing_on(&global_trace);
410 }
411 EXPORT_SYMBOL_GPL(tracing_on);
412
413 /**
414  * __trace_puts - write a constant string into the trace buffer.
415  * @ip:    The address of the caller
416  * @str:   The constant string to write
417  * @size:  The size of the string.
418  */
419 int __trace_puts(unsigned long ip, const char *str, int size)
420 {
421         struct ring_buffer_event *event;
422         struct ring_buffer *buffer;
423         struct print_entry *entry;
424         unsigned long irq_flags;
425         int alloc;
426
427         alloc = sizeof(*entry) + size + 2; /* possible \n added */
428
429         local_save_flags(irq_flags);
430         buffer = global_trace.trace_buffer.buffer;
431         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
432                                           irq_flags, preempt_count());
433         if (!event)
434                 return 0;
435
436         entry = ring_buffer_event_data(event);
437         entry->ip = ip;
438
439         memcpy(&entry->buf, str, size);
440
441         /* Add a newline if necessary */
442         if (entry->buf[size - 1] != '\n') {
443                 entry->buf[size] = '\n';
444                 entry->buf[size + 1] = '\0';
445         } else
446                 entry->buf[size] = '\0';
447
448         __buffer_unlock_commit(buffer, event);
449
450         return size;
451 }
452 EXPORT_SYMBOL_GPL(__trace_puts);
453
454 /**
455  * __trace_bputs - write the pointer to a constant string into trace buffer
456  * @ip:    The address of the caller
457  * @str:   The constant string to write to the buffer to
458  */
459 int __trace_bputs(unsigned long ip, const char *str)
460 {
461         struct ring_buffer_event *event;
462         struct ring_buffer *buffer;
463         struct bputs_entry *entry;
464         unsigned long irq_flags;
465         int size = sizeof(struct bputs_entry);
466
467         local_save_flags(irq_flags);
468         buffer = global_trace.trace_buffer.buffer;
469         event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
470                                           irq_flags, preempt_count());
471         if (!event)
472                 return 0;
473
474         entry = ring_buffer_event_data(event);
475         entry->ip                       = ip;
476         entry->str                      = str;
477
478         __buffer_unlock_commit(buffer, event);
479
480         return 1;
481 }
482 EXPORT_SYMBOL_GPL(__trace_bputs);
483
484 #ifdef CONFIG_TRACER_SNAPSHOT
485 /**
486  * trace_snapshot - take a snapshot of the current buffer.
487  *
488  * This causes a swap between the snapshot buffer and the current live
489  * tracing buffer. You can use this to take snapshots of the live
490  * trace when some condition is triggered, but continue to trace.
491  *
492  * Note, make sure to allocate the snapshot with either
493  * a tracing_snapshot_alloc(), or by doing it manually
494  * with: echo 1 > /sys/kernel/debug/tracing/snapshot
495  *
496  * If the snapshot buffer is not allocated, it will stop tracing.
497  * Basically making a permanent snapshot.
498  */
499 void tracing_snapshot(void)
500 {
501         struct trace_array *tr = &global_trace;
502         struct tracer *tracer = tr->current_trace;
503         unsigned long flags;
504
505         if (in_nmi()) {
506                 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
507                 internal_trace_puts("*** snapshot is being ignored        ***\n");
508                 return;
509         }
510
511         if (!tr->allocated_snapshot) {
512                 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
513                 internal_trace_puts("*** stopping trace here!   ***\n");
514                 tracing_off();
515                 return;
516         }
517
518         /* Note, snapshot can not be used when the tracer uses it */
519         if (tracer->use_max_tr) {
520                 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
521                 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
522                 return;
523         }
524
525         local_irq_save(flags);
526         update_max_tr(tr, current, smp_processor_id());
527         local_irq_restore(flags);
528 }
529 EXPORT_SYMBOL_GPL(tracing_snapshot);
530
531 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
532                                         struct trace_buffer *size_buf, int cpu_id);
533 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
534
535 static int alloc_snapshot(struct trace_array *tr)
536 {
537         int ret;
538
539         if (!tr->allocated_snapshot) {
540
541                 /* allocate spare buffer */
542                 ret = resize_buffer_duplicate_size(&tr->max_buffer,
543                                    &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
544                 if (ret < 0)
545                         return ret;
546
547                 tr->allocated_snapshot = true;
548         }
549
550         return 0;
551 }
552
553 void free_snapshot(struct trace_array *tr)
554 {
555         /*
556          * We don't free the ring buffer. instead, resize it because
557          * The max_tr ring buffer has some state (e.g. ring->clock) and
558          * we want preserve it.
559          */
560         ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
561         set_buffer_entries(&tr->max_buffer, 1);
562         tracing_reset_online_cpus(&tr->max_buffer);
563         tr->allocated_snapshot = false;
564 }
565
566 /**
567  * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
568  *
569  * This is similar to trace_snapshot(), but it will allocate the
570  * snapshot buffer if it isn't already allocated. Use this only
571  * where it is safe to sleep, as the allocation may sleep.
572  *
573  * This causes a swap between the snapshot buffer and the current live
574  * tracing buffer. You can use this to take snapshots of the live
575  * trace when some condition is triggered, but continue to trace.
576  */
577 void tracing_snapshot_alloc(void)
578 {
579         struct trace_array *tr = &global_trace;
580         int ret;
581
582         ret = alloc_snapshot(tr);
583         if (WARN_ON(ret < 0))
584                 return;
585
586         tracing_snapshot();
587 }
588 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
589 #else
590 void tracing_snapshot(void)
591 {
592         WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
593 }
594 EXPORT_SYMBOL_GPL(tracing_snapshot);
595 void tracing_snapshot_alloc(void)
596 {
597         /* Give warning */
598         tracing_snapshot();
599 }
600 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
601 #endif /* CONFIG_TRACER_SNAPSHOT */
602
603 void tracer_tracing_off(struct trace_array *tr)
604 {
605         if (tr->trace_buffer.buffer)
606                 ring_buffer_record_off(tr->trace_buffer.buffer);
607         /*
608          * This flag is looked at when buffers haven't been allocated
609          * yet, or by some tracers (like irqsoff), that just want to
610          * know if the ring buffer has been disabled, but it can handle
611          * races of where it gets disabled but we still do a record.
612          * As the check is in the fast path of the tracers, it is more
613          * important to be fast than accurate.
614          */
615         tr->buffer_disabled = 1;
616         /* Make the flag seen by readers */
617         smp_wmb();
618 }
619
620 /**
621  * tracing_off - turn off tracing buffers
622  *
623  * This function stops the tracing buffers from recording data.
624  * It does not disable any overhead the tracers themselves may
625  * be causing. This function simply causes all recording to
626  * the ring buffers to fail.
627  */
628 void tracing_off(void)
629 {
630         tracer_tracing_off(&global_trace);
631 }
632 EXPORT_SYMBOL_GPL(tracing_off);
633
634 /**
635  * tracer_tracing_is_on - show real state of ring buffer enabled
636  * @tr : the trace array to know if ring buffer is enabled
637  *
638  * Shows real state of the ring buffer if it is enabled or not.
639  */
640 int tracer_tracing_is_on(struct trace_array *tr)
641 {
642         if (tr->trace_buffer.buffer)
643                 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
644         return !tr->buffer_disabled;
645 }
646
647 /**
648  * tracing_is_on - show state of ring buffers enabled
649  */
650 int tracing_is_on(void)
651 {
652         return tracer_tracing_is_on(&global_trace);
653 }
654 EXPORT_SYMBOL_GPL(tracing_is_on);
655
656 static int __init set_buf_size(char *str)
657 {
658         unsigned long buf_size;
659
660         if (!str)
661                 return 0;
662         buf_size = memparse(str, &str);
663         /* nr_entries can not be zero */
664         if (buf_size == 0)
665                 return 0;
666         trace_buf_size = buf_size;
667         return 1;
668 }
669 __setup("trace_buf_size=", set_buf_size);
670
671 static int __init set_tracing_thresh(char *str)
672 {
673         unsigned long threshold;
674         int ret;
675
676         if (!str)
677                 return 0;
678         ret = kstrtoul(str, 0, &threshold);
679         if (ret < 0)
680                 return 0;
681         tracing_thresh = threshold * 1000;
682         return 1;
683 }
684 __setup("tracing_thresh=", set_tracing_thresh);
685
686 unsigned long nsecs_to_usecs(unsigned long nsecs)
687 {
688         return nsecs / 1000;
689 }
690
691 /* These must match the bit postions in trace_iterator_flags */
692 static const char *trace_options[] = {
693         "print-parent",
694         "sym-offset",
695         "sym-addr",
696         "verbose",
697         "raw",
698         "hex",
699         "bin",
700         "block",
701         "stacktrace",
702         "trace_printk",
703         "ftrace_preempt",
704         "branch",
705         "annotate",
706         "userstacktrace",
707         "sym-userobj",
708         "printk-msg-only",
709         "context-info",
710         "latency-format",
711         "sleep-time",
712         "graph-time",
713         "record-cmd",
714         "overwrite",
715         "disable_on_free",
716         "irq-info",
717         "markers",
718         "function-trace",
719         NULL
720 };
721
722 static struct {
723         u64 (*func)(void);
724         const char *name;
725         int in_ns;              /* is this clock in nanoseconds? */
726 } trace_clocks[] = {
727         { trace_clock_local,    "local",        1 },
728         { trace_clock_global,   "global",       1 },
729         { trace_clock_counter,  "counter",      0 },
730         { trace_clock_jiffies,  "uptime",       1 },
731         { trace_clock,          "perf",         1 },
732         ARCH_TRACE_CLOCKS
733 };
734
735 /*
736  * trace_parser_get_init - gets the buffer for trace parser
737  */
738 int trace_parser_get_init(struct trace_parser *parser, int size)
739 {
740         memset(parser, 0, sizeof(*parser));
741
742         parser->buffer = kmalloc(size, GFP_KERNEL);
743         if (!parser->buffer)
744                 return 1;
745
746         parser->size = size;
747         return 0;
748 }
749
750 /*
751  * trace_parser_put - frees the buffer for trace parser
752  */
753 void trace_parser_put(struct trace_parser *parser)
754 {
755         kfree(parser->buffer);
756 }
757
758 /*
759  * trace_get_user - reads the user input string separated by  space
760  * (matched by isspace(ch))
761  *
762  * For each string found the 'struct trace_parser' is updated,
763  * and the function returns.
764  *
765  * Returns number of bytes read.
766  *
767  * See kernel/trace/trace.h for 'struct trace_parser' details.
768  */
769 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
770         size_t cnt, loff_t *ppos)
771 {
772         char ch;
773         size_t read = 0;
774         ssize_t ret;
775
776         if (!*ppos)
777                 trace_parser_clear(parser);
778
779         ret = get_user(ch, ubuf++);
780         if (ret)
781                 goto out;
782
783         read++;
784         cnt--;
785
786         /*
787          * The parser is not finished with the last write,
788          * continue reading the user input without skipping spaces.
789          */
790         if (!parser->cont) {
791                 /* skip white space */
792                 while (cnt && isspace(ch)) {
793                         ret = get_user(ch, ubuf++);
794                         if (ret)
795                                 goto out;
796                         read++;
797                         cnt--;
798                 }
799
800                 /* only spaces were written */
801                 if (isspace(ch)) {
802                         *ppos += read;
803                         ret = read;
804                         goto out;
805                 }
806
807                 parser->idx = 0;
808         }
809
810         /* read the non-space input */
811         while (cnt && !isspace(ch)) {
812                 if (parser->idx < parser->size - 1)
813                         parser->buffer[parser->idx++] = ch;
814                 else {
815                         ret = -EINVAL;
816                         goto out;
817                 }
818                 ret = get_user(ch, ubuf++);
819                 if (ret)
820                         goto out;
821                 read++;
822                 cnt--;
823         }
824
825         /* We either got finished input or we have to wait for another call. */
826         if (isspace(ch)) {
827                 parser->buffer[parser->idx] = 0;
828                 parser->cont = false;
829         } else if (parser->idx < parser->size - 1) {
830                 parser->cont = true;
831                 parser->buffer[parser->idx++] = ch;
832         } else {
833                 ret = -EINVAL;
834                 goto out;
835         }
836
837         *ppos += read;
838         ret = read;
839
840 out:
841         return ret;
842 }
843
844 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
845 {
846         int len;
847         int ret;
848
849         if (!cnt)
850                 return 0;
851
852         if (s->len <= s->readpos)
853                 return -EBUSY;
854
855         len = s->len - s->readpos;
856         if (cnt > len)
857                 cnt = len;
858         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
859         if (ret == cnt)
860                 return -EFAULT;
861
862         cnt -= ret;
863
864         s->readpos += cnt;
865         return cnt;
866 }
867
868 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
869 {
870         int len;
871
872         if (s->len <= s->readpos)
873                 return -EBUSY;
874
875         len = s->len - s->readpos;
876         if (cnt > len)
877                 cnt = len;
878         memcpy(buf, s->buffer + s->readpos, cnt);
879
880         s->readpos += cnt;
881         return cnt;
882 }
883
884 /*
885  * ftrace_max_lock is used to protect the swapping of buffers
886  * when taking a max snapshot. The buffers themselves are
887  * protected by per_cpu spinlocks. But the action of the swap
888  * needs its own lock.
889  *
890  * This is defined as a arch_spinlock_t in order to help
891  * with performance when lockdep debugging is enabled.
892  *
893  * It is also used in other places outside the update_max_tr
894  * so it needs to be defined outside of the
895  * CONFIG_TRACER_MAX_TRACE.
896  */
897 static arch_spinlock_t ftrace_max_lock =
898         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
899
900 unsigned long __read_mostly     tracing_thresh;
901
902 #ifdef CONFIG_TRACER_MAX_TRACE
903 unsigned long __read_mostly     tracing_max_latency;
904
905 /*
906  * Copy the new maximum trace into the separate maximum-trace
907  * structure. (this way the maximum trace is permanently saved,
908  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
909  */
910 static void
911 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
912 {
913         struct trace_buffer *trace_buf = &tr->trace_buffer;
914         struct trace_buffer *max_buf = &tr->max_buffer;
915         struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
916         struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
917
918         max_buf->cpu = cpu;
919         max_buf->time_start = data->preempt_timestamp;
920
921         max_data->saved_latency = tracing_max_latency;
922         max_data->critical_start = data->critical_start;
923         max_data->critical_end = data->critical_end;
924
925         memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
926         max_data->pid = tsk->pid;
927         /*
928          * If tsk == current, then use current_uid(), as that does not use
929          * RCU. The irq tracer can be called out of RCU scope.
930          */
931         if (tsk == current)
932                 max_data->uid = current_uid();
933         else
934                 max_data->uid = task_uid(tsk);
935
936         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
937         max_data->policy = tsk->policy;
938         max_data->rt_priority = tsk->rt_priority;
939
940         /* record this tasks comm */
941         tracing_record_cmdline(tsk);
942 }
943
944 /**
945  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
946  * @tr: tracer
947  * @tsk: the task with the latency
948  * @cpu: The cpu that initiated the trace.
949  *
950  * Flip the buffers between the @tr and the max_tr and record information
951  * about which task was the cause of this latency.
952  */
953 void
954 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
955 {
956         struct ring_buffer *buf;
957
958         if (tr->stop_count)
959                 return;
960
961         WARN_ON_ONCE(!irqs_disabled());
962
963         if (!tr->allocated_snapshot) {
964                 /* Only the nop tracer should hit this when disabling */
965                 WARN_ON_ONCE(tr->current_trace != &nop_trace);
966                 return;
967         }
968
969         arch_spin_lock(&ftrace_max_lock);
970
971         buf = tr->trace_buffer.buffer;
972         tr->trace_buffer.buffer = tr->max_buffer.buffer;
973         tr->max_buffer.buffer = buf;
974
975         __update_max_tr(tr, tsk, cpu);
976         arch_spin_unlock(&ftrace_max_lock);
977 }
978
979 /**
980  * update_max_tr_single - only copy one trace over, and reset the rest
981  * @tr - tracer
982  * @tsk - task with the latency
983  * @cpu - the cpu of the buffer to copy.
984  *
985  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
986  */
987 void
988 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
989 {
990         int ret;
991
992         if (tr->stop_count)
993                 return;
994
995         WARN_ON_ONCE(!irqs_disabled());
996         if (!tr->allocated_snapshot) {
997                 /* Only the nop tracer should hit this when disabling */
998                 WARN_ON_ONCE(tr->current_trace != &nop_trace);
999                 return;
1000         }
1001
1002         arch_spin_lock(&ftrace_max_lock);
1003
1004         ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1005
1006         if (ret == -EBUSY) {
1007                 /*
1008                  * We failed to swap the buffer due to a commit taking
1009                  * place on this CPU. We fail to record, but we reset
1010                  * the max trace buffer (no one writes directly to it)
1011                  * and flag that it failed.
1012                  */
1013                 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1014                         "Failed to swap buffers due to commit in progress\n");
1015         }
1016
1017         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1018
1019         __update_max_tr(tr, tsk, cpu);
1020         arch_spin_unlock(&ftrace_max_lock);
1021 }
1022 #endif /* CONFIG_TRACER_MAX_TRACE */
1023
1024 static void default_wait_pipe(struct trace_iterator *iter)
1025 {
1026         /* Iterators are static, they should be filled or empty */
1027         if (trace_buffer_iter(iter, iter->cpu_file))
1028                 return;
1029
1030         ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1031 }
1032
1033 #ifdef CONFIG_FTRACE_STARTUP_TEST
1034 static int run_tracer_selftest(struct tracer *type)
1035 {
1036         struct trace_array *tr = &global_trace;
1037         struct tracer *saved_tracer = tr->current_trace;
1038         int ret;
1039
1040         if (!type->selftest || tracing_selftest_disabled)
1041                 return 0;
1042
1043         /*
1044          * Run a selftest on this tracer.
1045          * Here we reset the trace buffer, and set the current
1046          * tracer to be this tracer. The tracer can then run some
1047          * internal tracing to verify that everything is in order.
1048          * If we fail, we do not register this tracer.
1049          */
1050         tracing_reset_online_cpus(&tr->trace_buffer);
1051
1052         tr->current_trace = type;
1053
1054 #ifdef CONFIG_TRACER_MAX_TRACE
1055         if (type->use_max_tr) {
1056                 /* If we expanded the buffers, make sure the max is expanded too */
1057                 if (ring_buffer_expanded)
1058                         ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1059                                            RING_BUFFER_ALL_CPUS);
1060                 tr->allocated_snapshot = true;
1061         }
1062 #endif
1063
1064         /* the test is responsible for initializing and enabling */
1065         pr_info("Testing tracer %s: ", type->name);
1066         ret = type->selftest(type, tr);
1067         /* the test is responsible for resetting too */
1068         tr->current_trace = saved_tracer;
1069         if (ret) {
1070                 printk(KERN_CONT "FAILED!\n");
1071                 /* Add the warning after printing 'FAILED' */
1072                 WARN_ON(1);
1073                 return -1;
1074         }
1075         /* Only reset on passing, to avoid touching corrupted buffers */
1076         tracing_reset_online_cpus(&tr->trace_buffer);
1077
1078 #ifdef CONFIG_TRACER_MAX_TRACE
1079         if (type->use_max_tr) {
1080                 tr->allocated_snapshot = false;
1081
1082                 /* Shrink the max buffer again */
1083                 if (ring_buffer_expanded)
1084                         ring_buffer_resize(tr->max_buffer.buffer, 1,
1085                                            RING_BUFFER_ALL_CPUS);
1086         }
1087 #endif
1088
1089         printk(KERN_CONT "PASSED\n");
1090         return 0;
1091 }
1092 #else
1093 static inline int run_tracer_selftest(struct tracer *type)
1094 {
1095         return 0;
1096 }
1097 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1098
1099 /**
1100  * register_tracer - register a tracer with the ftrace system.
1101  * @type - the plugin for the tracer
1102  *
1103  * Register a new plugin tracer.
1104  */
1105 int register_tracer(struct tracer *type)
1106 {
1107         struct tracer *t;
1108         int ret = 0;
1109
1110         if (!type->name) {
1111                 pr_info("Tracer must have a name\n");
1112                 return -1;
1113         }
1114
1115         if (strlen(type->name) >= MAX_TRACER_SIZE) {
1116                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1117                 return -1;
1118         }
1119
1120         mutex_lock(&trace_types_lock);
1121
1122         tracing_selftest_running = true;
1123
1124         for (t = trace_types; t; t = t->next) {
1125                 if (strcmp(type->name, t->name) == 0) {
1126                         /* already found */
1127                         pr_info("Tracer %s already registered\n",
1128                                 type->name);
1129                         ret = -1;
1130                         goto out;
1131                 }
1132         }
1133
1134         if (!type->set_flag)
1135                 type->set_flag = &dummy_set_flag;
1136         if (!type->flags)
1137                 type->flags = &dummy_tracer_flags;
1138         else
1139                 if (!type->flags->opts)
1140                         type->flags->opts = dummy_tracer_opt;
1141         if (!type->wait_pipe)
1142                 type->wait_pipe = default_wait_pipe;
1143
1144         ret = run_tracer_selftest(type);
1145         if (ret < 0)
1146                 goto out;
1147
1148         type->next = trace_types;
1149         trace_types = type;
1150
1151  out:
1152         tracing_selftest_running = false;
1153         mutex_unlock(&trace_types_lock);
1154
1155         if (ret || !default_bootup_tracer)
1156                 goto out_unlock;
1157
1158         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1159                 goto out_unlock;
1160
1161         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1162         /* Do we want this tracer to start on bootup? */
1163         tracing_set_tracer(type->name);
1164         default_bootup_tracer = NULL;
1165         /* disable other selftests, since this will break it. */
1166         tracing_selftest_disabled = true;
1167 #ifdef CONFIG_FTRACE_STARTUP_TEST
1168         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1169                type->name);
1170 #endif
1171
1172  out_unlock:
1173         return ret;
1174 }
1175
1176 void tracing_reset(struct trace_buffer *buf, int cpu)
1177 {
1178         struct ring_buffer *buffer = buf->buffer;
1179
1180         if (!buffer)
1181                 return;
1182
1183         ring_buffer_record_disable(buffer);
1184
1185         /* Make sure all commits have finished */
1186         synchronize_sched();
1187         ring_buffer_reset_cpu(buffer, cpu);
1188
1189         ring_buffer_record_enable(buffer);
1190 }
1191
1192 void tracing_reset_online_cpus(struct trace_buffer *buf)
1193 {
1194         struct ring_buffer *buffer = buf->buffer;
1195         int cpu;
1196
1197         if (!buffer)
1198                 return;
1199
1200         ring_buffer_record_disable(buffer);
1201
1202         /* Make sure all commits have finished */
1203         synchronize_sched();
1204
1205         buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1206
1207         for_each_online_cpu(cpu)
1208                 ring_buffer_reset_cpu(buffer, cpu);
1209
1210         ring_buffer_record_enable(buffer);
1211 }
1212
1213 /* Must have trace_types_lock held */
1214 void tracing_reset_all_online_cpus(void)
1215 {
1216         struct trace_array *tr;
1217
1218         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1219                 tracing_reset_online_cpus(&tr->trace_buffer);
1220 #ifdef CONFIG_TRACER_MAX_TRACE
1221                 tracing_reset_online_cpus(&tr->max_buffer);
1222 #endif
1223         }
1224 }
1225
1226 #define SAVED_CMDLINES 128
1227 #define NO_CMDLINE_MAP UINT_MAX
1228 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1229 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1230 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1231 static int cmdline_idx;
1232 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1233
1234 /* temporary disable recording */
1235 static atomic_t trace_record_cmdline_disabled __read_mostly;
1236
1237 static void trace_init_cmdlines(void)
1238 {
1239         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1240         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
1241         cmdline_idx = 0;
1242 }
1243
1244 int is_tracing_stopped(void)
1245 {
1246         return global_trace.stop_count;
1247 }
1248
1249 /**
1250  * ftrace_off_permanent - disable all ftrace code permanently
1251  *
1252  * This should only be called when a serious anomally has
1253  * been detected.  This will turn off the function tracing,
1254  * ring buffers, and other tracing utilites. It takes no
1255  * locks and can be called from any context.
1256  */
1257 void ftrace_off_permanent(void)
1258 {
1259         tracing_disabled = 1;
1260         ftrace_stop();
1261         tracing_off_permanent();
1262 }
1263
1264 /**
1265  * tracing_start - quick start of the tracer
1266  *
1267  * If tracing is enabled but was stopped by tracing_stop,
1268  * this will start the tracer back up.
1269  */
1270 void tracing_start(void)
1271 {
1272         struct ring_buffer *buffer;
1273         unsigned long flags;
1274
1275         if (tracing_disabled)
1276                 return;
1277
1278         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1279         if (--global_trace.stop_count) {
1280                 if (global_trace.stop_count < 0) {
1281                         /* Someone screwed up their debugging */
1282                         WARN_ON_ONCE(1);
1283                         global_trace.stop_count = 0;
1284                 }
1285                 goto out;
1286         }
1287
1288         /* Prevent the buffers from switching */
1289         arch_spin_lock(&ftrace_max_lock);
1290
1291         buffer = global_trace.trace_buffer.buffer;
1292         if (buffer)
1293                 ring_buffer_record_enable(buffer);
1294
1295 #ifdef CONFIG_TRACER_MAX_TRACE
1296         buffer = global_trace.max_buffer.buffer;
1297         if (buffer)
1298                 ring_buffer_record_enable(buffer);
1299 #endif
1300
1301         arch_spin_unlock(&ftrace_max_lock);
1302
1303         ftrace_start();
1304  out:
1305         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1306 }
1307
1308 static void tracing_start_tr(struct trace_array *tr)
1309 {
1310         struct ring_buffer *buffer;
1311         unsigned long flags;
1312
1313         if (tracing_disabled)
1314                 return;
1315
1316         /* If global, we need to also start the max tracer */
1317         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1318                 return tracing_start();
1319
1320         raw_spin_lock_irqsave(&tr->start_lock, flags);
1321
1322         if (--tr->stop_count) {
1323                 if (tr->stop_count < 0) {
1324                         /* Someone screwed up their debugging */
1325                         WARN_ON_ONCE(1);
1326                         tr->stop_count = 0;
1327                 }
1328                 goto out;
1329         }
1330
1331         buffer = tr->trace_buffer.buffer;
1332         if (buffer)
1333                 ring_buffer_record_enable(buffer);
1334
1335  out:
1336         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1337 }
1338
1339 /**
1340  * tracing_stop - quick stop of the tracer
1341  *
1342  * Light weight way to stop tracing. Use in conjunction with
1343  * tracing_start.
1344  */
1345 void tracing_stop(void)
1346 {
1347         struct ring_buffer *buffer;
1348         unsigned long flags;
1349
1350         ftrace_stop();
1351         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1352         if (global_trace.stop_count++)
1353                 goto out;
1354
1355         /* Prevent the buffers from switching */
1356         arch_spin_lock(&ftrace_max_lock);
1357
1358         buffer = global_trace.trace_buffer.buffer;
1359         if (buffer)
1360                 ring_buffer_record_disable(buffer);
1361
1362 #ifdef CONFIG_TRACER_MAX_TRACE
1363         buffer = global_trace.max_buffer.buffer;
1364         if (buffer)
1365                 ring_buffer_record_disable(buffer);
1366 #endif
1367
1368         arch_spin_unlock(&ftrace_max_lock);
1369
1370  out:
1371         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1372 }
1373
1374 static void tracing_stop_tr(struct trace_array *tr)
1375 {
1376         struct ring_buffer *buffer;
1377         unsigned long flags;
1378
1379         /* If global, we need to also stop the max tracer */
1380         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1381                 return tracing_stop();
1382
1383         raw_spin_lock_irqsave(&tr->start_lock, flags);
1384         if (tr->stop_count++)
1385                 goto out;
1386
1387         buffer = tr->trace_buffer.buffer;
1388         if (buffer)
1389                 ring_buffer_record_disable(buffer);
1390
1391  out:
1392         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1393 }
1394
1395 void trace_stop_cmdline_recording(void);
1396
1397 static void trace_save_cmdline(struct task_struct *tsk)
1398 {
1399         unsigned pid, idx;
1400
1401         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1402                 return;
1403
1404         /*
1405          * It's not the end of the world if we don't get
1406          * the lock, but we also don't want to spin
1407          * nor do we want to disable interrupts,
1408          * so if we miss here, then better luck next time.
1409          */
1410         if (!arch_spin_trylock(&trace_cmdline_lock))
1411                 return;
1412
1413         idx = map_pid_to_cmdline[tsk->pid];
1414         if (idx == NO_CMDLINE_MAP) {
1415                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1416
1417                 /*
1418                  * Check whether the cmdline buffer at idx has a pid
1419                  * mapped. We are going to overwrite that entry so we
1420                  * need to clear the map_pid_to_cmdline. Otherwise we
1421                  * would read the new comm for the old pid.
1422                  */
1423                 pid = map_cmdline_to_pid[idx];
1424                 if (pid != NO_CMDLINE_MAP)
1425                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1426
1427                 map_cmdline_to_pid[idx] = tsk->pid;
1428                 map_pid_to_cmdline[tsk->pid] = idx;
1429
1430                 cmdline_idx = idx;
1431         }
1432
1433         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1434
1435         arch_spin_unlock(&trace_cmdline_lock);
1436 }
1437
1438 void trace_find_cmdline(int pid, char comm[])
1439 {
1440         unsigned map;
1441
1442         if (!pid) {
1443                 strcpy(comm, "<idle>");
1444                 return;
1445         }
1446
1447         if (WARN_ON_ONCE(pid < 0)) {
1448                 strcpy(comm, "<XXX>");
1449                 return;
1450         }
1451
1452         if (pid > PID_MAX_DEFAULT) {
1453                 strcpy(comm, "<...>");
1454                 return;
1455         }
1456
1457         preempt_disable();
1458         arch_spin_lock(&trace_cmdline_lock);
1459         map = map_pid_to_cmdline[pid];
1460         if (map != NO_CMDLINE_MAP)
1461                 strcpy(comm, saved_cmdlines[map]);
1462         else
1463                 strcpy(comm, "<...>");
1464
1465         arch_spin_unlock(&trace_cmdline_lock);
1466         preempt_enable();
1467 }
1468
1469 void tracing_record_cmdline(struct task_struct *tsk)
1470 {
1471         if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1472                 return;
1473
1474         if (!__this_cpu_read(trace_cmdline_save))
1475                 return;
1476
1477         __this_cpu_write(trace_cmdline_save, false);
1478
1479         trace_save_cmdline(tsk);
1480 }
1481
1482 void
1483 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1484                              int pc)
1485 {
1486         struct task_struct *tsk = current;
1487
1488         entry->preempt_count            = pc & 0xff;
1489         entry->pid                      = (tsk) ? tsk->pid : 0;
1490         entry->flags =
1491 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1492                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1493 #else
1494                 TRACE_FLAG_IRQS_NOSUPPORT |
1495 #endif
1496                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1497                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1498                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1499 }
1500 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1501
1502 struct ring_buffer_event *
1503 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1504                           int type,
1505                           unsigned long len,
1506                           unsigned long flags, int pc)
1507 {
1508         struct ring_buffer_event *event;
1509
1510         event = ring_buffer_lock_reserve(buffer, len);
1511         if (event != NULL) {
1512                 struct trace_entry *ent = ring_buffer_event_data(event);
1513
1514                 tracing_generic_entry_update(ent, flags, pc);
1515                 ent->type = type;
1516         }
1517
1518         return event;
1519 }
1520
1521 void
1522 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1523 {
1524         __this_cpu_write(trace_cmdline_save, true);
1525         ring_buffer_unlock_commit(buffer, event);
1526 }
1527
1528 static inline void
1529 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1530                              struct ring_buffer_event *event,
1531                              unsigned long flags, int pc)
1532 {
1533         __buffer_unlock_commit(buffer, event);
1534
1535         ftrace_trace_stack(buffer, flags, 6, pc);
1536         ftrace_trace_userstack(buffer, flags, pc);
1537 }
1538
1539 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1540                                 struct ring_buffer_event *event,
1541                                 unsigned long flags, int pc)
1542 {
1543         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1544 }
1545 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1546
1547 struct ring_buffer_event *
1548 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1549                           struct ftrace_event_file *ftrace_file,
1550                           int type, unsigned long len,
1551                           unsigned long flags, int pc)
1552 {
1553         *current_rb = ftrace_file->tr->trace_buffer.buffer;
1554         return trace_buffer_lock_reserve(*current_rb,
1555                                          type, len, flags, pc);
1556 }
1557 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1558
1559 struct ring_buffer_event *
1560 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1561                                   int type, unsigned long len,
1562                                   unsigned long flags, int pc)
1563 {
1564         *current_rb = global_trace.trace_buffer.buffer;
1565         return trace_buffer_lock_reserve(*current_rb,
1566                                          type, len, flags, pc);
1567 }
1568 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1569
1570 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1571                                         struct ring_buffer_event *event,
1572                                         unsigned long flags, int pc)
1573 {
1574         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1575 }
1576 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1577
1578 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1579                                      struct ring_buffer_event *event,
1580                                      unsigned long flags, int pc,
1581                                      struct pt_regs *regs)
1582 {
1583         __buffer_unlock_commit(buffer, event);
1584
1585         ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1586         ftrace_trace_userstack(buffer, flags, pc);
1587 }
1588 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1589
1590 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1591                                          struct ring_buffer_event *event)
1592 {
1593         ring_buffer_discard_commit(buffer, event);
1594 }
1595 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1596
1597 void
1598 trace_function(struct trace_array *tr,
1599                unsigned long ip, unsigned long parent_ip, unsigned long flags,
1600                int pc)
1601 {
1602         struct ftrace_event_call *call = &event_function;
1603         struct ring_buffer *buffer = tr->trace_buffer.buffer;
1604         struct ring_buffer_event *event;
1605         struct ftrace_entry *entry;
1606
1607         /* If we are reading the ring buffer, don't trace */
1608         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1609                 return;
1610
1611         event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1612                                           flags, pc);
1613         if (!event)
1614                 return;
1615         entry   = ring_buffer_event_data(event);
1616         entry->ip                       = ip;
1617         entry->parent_ip                = parent_ip;
1618
1619         if (!filter_check_discard(call, entry, buffer, event))
1620                 __buffer_unlock_commit(buffer, event);
1621 }
1622
1623 void
1624 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1625        unsigned long ip, unsigned long parent_ip, unsigned long flags,
1626        int pc)
1627 {
1628         if (likely(!atomic_read(&data->disabled)))
1629                 trace_function(tr, ip, parent_ip, flags, pc);
1630 }
1631
1632 #ifdef CONFIG_STACKTRACE
1633
1634 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1635 struct ftrace_stack {
1636         unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
1637 };
1638
1639 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1640 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1641
1642 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1643                                  unsigned long flags,
1644                                  int skip, int pc, struct pt_regs *regs)
1645 {
1646         struct ftrace_event_call *call = &event_kernel_stack;
1647         struct ring_buffer_event *event;
1648         struct stack_entry *entry;
1649         struct stack_trace trace;
1650         int use_stack;
1651         int size = FTRACE_STACK_ENTRIES;
1652
1653         trace.nr_entries        = 0;
1654         trace.skip              = skip;
1655
1656         /*
1657          * Since events can happen in NMIs there's no safe way to
1658          * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1659          * or NMI comes in, it will just have to use the default
1660          * FTRACE_STACK_SIZE.
1661          */
1662         preempt_disable_notrace();
1663
1664         use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1665         /*
1666          * We don't need any atomic variables, just a barrier.
1667          * If an interrupt comes in, we don't care, because it would
1668          * have exited and put the counter back to what we want.
1669          * We just need a barrier to keep gcc from moving things
1670          * around.
1671          */
1672         barrier();
1673         if (use_stack == 1) {
1674                 trace.entries           = &__get_cpu_var(ftrace_stack).calls[0];
1675                 trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
1676
1677                 if (regs)
1678                         save_stack_trace_regs(regs, &trace);
1679                 else
1680                         save_stack_trace(&trace);
1681
1682                 if (trace.nr_entries > size)
1683                         size = trace.nr_entries;
1684         } else
1685                 /* From now on, use_stack is a boolean */
1686                 use_stack = 0;
1687
1688         size *= sizeof(unsigned long);
1689
1690         event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1691                                           sizeof(*entry) + size, flags, pc);
1692         if (!event)
1693                 goto out;
1694         entry = ring_buffer_event_data(event);
1695
1696         memset(&entry->caller, 0, size);
1697
1698         if (use_stack)
1699                 memcpy(&entry->caller, trace.entries,
1700                        trace.nr_entries * sizeof(unsigned long));
1701         else {
1702                 trace.max_entries       = FTRACE_STACK_ENTRIES;
1703                 trace.entries           = entry->caller;
1704                 if (regs)
1705                         save_stack_trace_regs(regs, &trace);
1706                 else
1707                         save_stack_trace(&trace);
1708         }
1709
1710         entry->size = trace.nr_entries;
1711
1712         if (!filter_check_discard(call, entry, buffer, event))
1713                 __buffer_unlock_commit(buffer, event);
1714
1715  out:
1716         /* Again, don't let gcc optimize things here */
1717         barrier();
1718         __this_cpu_dec(ftrace_stack_reserve);
1719         preempt_enable_notrace();
1720
1721 }
1722
1723 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1724                              int skip, int pc, struct pt_regs *regs)
1725 {
1726         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1727                 return;
1728
1729         __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1730 }
1731
1732 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1733                         int skip, int pc)
1734 {
1735         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1736                 return;
1737
1738         __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1739 }
1740
1741 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1742                    int pc)
1743 {
1744         __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1745 }
1746
1747 /**
1748  * trace_dump_stack - record a stack back trace in the trace buffer
1749  * @skip: Number of functions to skip (helper handlers)
1750  */
1751 void trace_dump_stack(int skip)
1752 {
1753         unsigned long flags;
1754
1755         if (tracing_disabled || tracing_selftest_running)
1756                 return;
1757
1758         local_save_flags(flags);
1759
1760         /*
1761          * Skip 3 more, seems to get us at the caller of
1762          * this function.
1763          */
1764         skip += 3;
1765         __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1766                              flags, skip, preempt_count(), NULL);
1767 }
1768
1769 static DEFINE_PER_CPU(int, user_stack_count);
1770
1771 void
1772 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1773 {
1774         struct ftrace_event_call *call = &event_user_stack;
1775         struct ring_buffer_event *event;
1776         struct userstack_entry *entry;
1777         struct stack_trace trace;
1778
1779         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1780                 return;
1781
1782         /*
1783          * NMIs can not handle page faults, even with fix ups.
1784          * The save user stack can (and often does) fault.
1785          */
1786         if (unlikely(in_nmi()))
1787                 return;
1788
1789         /*
1790          * prevent recursion, since the user stack tracing may
1791          * trigger other kernel events.
1792          */
1793         preempt_disable();
1794         if (__this_cpu_read(user_stack_count))
1795                 goto out;
1796
1797         __this_cpu_inc(user_stack_count);
1798
1799         event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1800                                           sizeof(*entry), flags, pc);
1801         if (!event)
1802                 goto out_drop_count;
1803         entry   = ring_buffer_event_data(event);
1804
1805         entry->tgid             = current->tgid;
1806         memset(&entry->caller, 0, sizeof(entry->caller));
1807
1808         trace.nr_entries        = 0;
1809         trace.max_entries       = FTRACE_STACK_ENTRIES;
1810         trace.skip              = 0;
1811         trace.entries           = entry->caller;
1812
1813         save_stack_trace_user(&trace);
1814         if (!filter_check_discard(call, entry, buffer, event))
1815                 __buffer_unlock_commit(buffer, event);
1816
1817  out_drop_count:
1818         __this_cpu_dec(user_stack_count);
1819  out:
1820         preempt_enable();
1821 }
1822
1823 #ifdef UNUSED
1824 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1825 {
1826         ftrace_trace_userstack(tr, flags, preempt_count());
1827 }
1828 #endif /* UNUSED */
1829
1830 #endif /* CONFIG_STACKTRACE */
1831
1832 /* created for use with alloc_percpu */
1833 struct trace_buffer_struct {
1834         char buffer[TRACE_BUF_SIZE];
1835 };
1836
1837 static struct trace_buffer_struct *trace_percpu_buffer;
1838 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1839 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1840 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1841
1842 /*
1843  * The buffer used is dependent on the context. There is a per cpu
1844  * buffer for normal context, softirq contex, hard irq context and
1845  * for NMI context. Thise allows for lockless recording.
1846  *
1847  * Note, if the buffers failed to be allocated, then this returns NULL
1848  */
1849 static char *get_trace_buf(void)
1850 {
1851         struct trace_buffer_struct *percpu_buffer;
1852
1853         /*
1854          * If we have allocated per cpu buffers, then we do not
1855          * need to do any locking.
1856          */
1857         if (in_nmi())
1858                 percpu_buffer = trace_percpu_nmi_buffer;
1859         else if (in_irq())
1860                 percpu_buffer = trace_percpu_irq_buffer;
1861         else if (in_softirq())
1862                 percpu_buffer = trace_percpu_sirq_buffer;
1863         else
1864                 percpu_buffer = trace_percpu_buffer;
1865
1866         if (!percpu_buffer)
1867                 return NULL;
1868
1869         return this_cpu_ptr(&percpu_buffer->buffer[0]);
1870 }
1871
1872 static int alloc_percpu_trace_buffer(void)
1873 {
1874         struct trace_buffer_struct *buffers;
1875         struct trace_buffer_struct *sirq_buffers;
1876         struct trace_buffer_struct *irq_buffers;
1877         struct trace_buffer_struct *nmi_buffers;
1878
1879         buffers = alloc_percpu(struct trace_buffer_struct);
1880         if (!buffers)
1881                 goto err_warn;
1882
1883         sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1884         if (!sirq_buffers)
1885                 goto err_sirq;
1886
1887         irq_buffers = alloc_percpu(struct trace_buffer_struct);
1888         if (!irq_buffers)
1889                 goto err_irq;
1890
1891         nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1892         if (!nmi_buffers)
1893                 goto err_nmi;
1894
1895         trace_percpu_buffer = buffers;
1896         trace_percpu_sirq_buffer = sirq_buffers;
1897         trace_percpu_irq_buffer = irq_buffers;
1898         trace_percpu_nmi_buffer = nmi_buffers;
1899
1900         return 0;
1901
1902  err_nmi:
1903         free_percpu(irq_buffers);
1904  err_irq:
1905         free_percpu(sirq_buffers);
1906  err_sirq:
1907         free_percpu(buffers);
1908  err_warn:
1909         WARN(1, "Could not allocate percpu trace_printk buffer");
1910         return -ENOMEM;
1911 }
1912
1913 static int buffers_allocated;
1914
1915 void trace_printk_init_buffers(void)
1916 {
1917         if (buffers_allocated)
1918                 return;
1919
1920         if (alloc_percpu_trace_buffer())
1921                 return;
1922
1923         pr_info("ftrace: Allocated trace_printk buffers\n");
1924
1925         /* Expand the buffers to set size */
1926         tracing_update_buffers();
1927
1928         buffers_allocated = 1;
1929
1930         /*
1931          * trace_printk_init_buffers() can be called by modules.
1932          * If that happens, then we need to start cmdline recording
1933          * directly here. If the global_trace.buffer is already
1934          * allocated here, then this was called by module code.
1935          */
1936         if (global_trace.trace_buffer.buffer)
1937                 tracing_start_cmdline_record();
1938 }
1939
1940 void trace_printk_start_comm(void)
1941 {
1942         /* Start tracing comms if trace printk is set */
1943         if (!buffers_allocated)
1944                 return;
1945         tracing_start_cmdline_record();
1946 }
1947
1948 static void trace_printk_start_stop_comm(int enabled)
1949 {
1950         if (!buffers_allocated)
1951                 return;
1952
1953         if (enabled)
1954                 tracing_start_cmdline_record();
1955         else
1956                 tracing_stop_cmdline_record();
1957 }
1958
1959 /**
1960  * trace_vbprintk - write binary msg to tracing buffer
1961  *
1962  */
1963 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1964 {
1965         struct ftrace_event_call *call = &event_bprint;
1966         struct ring_buffer_event *event;
1967         struct ring_buffer *buffer;
1968         struct trace_array *tr = &global_trace;
1969         struct bprint_entry *entry;
1970         unsigned long flags;
1971         char *tbuffer;
1972         int len = 0, size, pc;
1973
1974         if (unlikely(tracing_selftest_running || tracing_disabled))
1975                 return 0;
1976
1977         /* Don't pollute graph traces with trace_vprintk internals */
1978         pause_graph_tracing();
1979
1980         pc = preempt_count();
1981         preempt_disable_notrace();
1982
1983         tbuffer = get_trace_buf();
1984         if (!tbuffer) {
1985                 len = 0;
1986                 goto out;
1987         }
1988
1989         len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1990
1991         if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1992                 goto out;
1993
1994         local_save_flags(flags);
1995         size = sizeof(*entry) + sizeof(u32) * len;
1996         buffer = tr->trace_buffer.buffer;
1997         event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1998                                           flags, pc);
1999         if (!event)
2000                 goto out;
2001         entry = ring_buffer_event_data(event);
2002         entry->ip                       = ip;
2003         entry->fmt                      = fmt;
2004
2005         memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2006         if (!filter_check_discard(call, entry, buffer, event)) {
2007                 __buffer_unlock_commit(buffer, event);
2008                 ftrace_trace_stack(buffer, flags, 6, pc);
2009         }
2010
2011 out:
2012         preempt_enable_notrace();
2013         unpause_graph_tracing();
2014
2015         return len;
2016 }
2017 EXPORT_SYMBOL_GPL(trace_vbprintk);
2018
2019 static int
2020 __trace_array_vprintk(struct ring_buffer *buffer,
2021                       unsigned long ip, const char *fmt, va_list args)
2022 {
2023         struct ftrace_event_call *call = &event_print;
2024         struct ring_buffer_event *event;
2025         int len = 0, size, pc;
2026         struct print_entry *entry;
2027         unsigned long flags;
2028         char *tbuffer;
2029
2030         if (tracing_disabled || tracing_selftest_running)
2031                 return 0;
2032
2033         /* Don't pollute graph traces with trace_vprintk internals */
2034         pause_graph_tracing();
2035
2036         pc = preempt_count();
2037         preempt_disable_notrace();
2038
2039
2040         tbuffer = get_trace_buf();
2041         if (!tbuffer) {
2042                 len = 0;
2043                 goto out;
2044         }
2045
2046         len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2047         if (len > TRACE_BUF_SIZE)
2048                 goto out;
2049
2050         local_save_flags(flags);
2051         size = sizeof(*entry) + len + 1;
2052         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2053                                           flags, pc);
2054         if (!event)
2055                 goto out;
2056         entry = ring_buffer_event_data(event);
2057         entry->ip = ip;
2058
2059         memcpy(&entry->buf, tbuffer, len);
2060         entry->buf[len] = '\0';
2061         if (!filter_check_discard(call, entry, buffer, event)) {
2062                 __buffer_unlock_commit(buffer, event);
2063                 ftrace_trace_stack(buffer, flags, 6, pc);
2064         }
2065  out:
2066         preempt_enable_notrace();
2067         unpause_graph_tracing();
2068
2069         return len;
2070 }
2071
2072 int trace_array_vprintk(struct trace_array *tr,
2073                         unsigned long ip, const char *fmt, va_list args)
2074 {
2075         return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2076 }
2077
2078 int trace_array_printk(struct trace_array *tr,
2079                        unsigned long ip, const char *fmt, ...)
2080 {
2081         int ret;
2082         va_list ap;
2083
2084         if (!(trace_flags & TRACE_ITER_PRINTK))
2085                 return 0;
2086
2087         va_start(ap, fmt);
2088         ret = trace_array_vprintk(tr, ip, fmt, ap);
2089         va_end(ap);
2090         return ret;
2091 }
2092
2093 int trace_array_printk_buf(struct ring_buffer *buffer,
2094                            unsigned long ip, const char *fmt, ...)
2095 {
2096         int ret;
2097         va_list ap;
2098
2099         if (!(trace_flags & TRACE_ITER_PRINTK))
2100                 return 0;
2101
2102         va_start(ap, fmt);
2103         ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2104         va_end(ap);
2105         return ret;
2106 }
2107
2108 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2109 {
2110         return trace_array_vprintk(&global_trace, ip, fmt, args);
2111 }
2112 EXPORT_SYMBOL_GPL(trace_vprintk);
2113
2114 static void trace_iterator_increment(struct trace_iterator *iter)
2115 {
2116         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2117
2118         iter->idx++;
2119         if (buf_iter)
2120                 ring_buffer_read(buf_iter, NULL);
2121 }
2122
2123 static struct trace_entry *
2124 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2125                 unsigned long *lost_events)
2126 {
2127         struct ring_buffer_event *event;
2128         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2129
2130         if (buf_iter)
2131                 event = ring_buffer_iter_peek(buf_iter, ts);
2132         else
2133                 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2134                                          lost_events);
2135
2136         if (event) {
2137                 iter->ent_size = ring_buffer_event_length(event);
2138                 return ring_buffer_event_data(event);
2139         }
2140         iter->ent_size = 0;
2141         return NULL;
2142 }
2143
2144 static struct trace_entry *
2145 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2146                   unsigned long *missing_events, u64 *ent_ts)
2147 {
2148         struct ring_buffer *buffer = iter->trace_buffer->buffer;
2149         struct trace_entry *ent, *next = NULL;
2150         unsigned long lost_events = 0, next_lost = 0;
2151         int cpu_file = iter->cpu_file;
2152         u64 next_ts = 0, ts;
2153         int next_cpu = -1;
2154         int next_size = 0;
2155         int cpu;
2156
2157         /*
2158          * If we are in a per_cpu trace file, don't bother by iterating over
2159          * all cpu and peek directly.
2160          */
2161         if (cpu_file > RING_BUFFER_ALL_CPUS) {
2162                 if (ring_buffer_empty_cpu(buffer, cpu_file))
2163                         return NULL;
2164                 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2165                 if (ent_cpu)
2166                         *ent_cpu = cpu_file;
2167
2168                 return ent;
2169         }
2170
2171         for_each_tracing_cpu(cpu) {
2172
2173                 if (ring_buffer_empty_cpu(buffer, cpu))
2174                         continue;
2175
2176                 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2177
2178                 /*
2179                  * Pick the entry with the smallest timestamp:
2180                  */
2181                 if (ent && (!next || ts < next_ts)) {
2182                         next = ent;
2183                         next_cpu = cpu;
2184                         next_ts = ts;
2185                         next_lost = lost_events;
2186                         next_size = iter->ent_size;
2187                 }
2188         }
2189
2190         iter->ent_size = next_size;
2191
2192         if (ent_cpu)
2193                 *ent_cpu = next_cpu;
2194
2195         if (ent_ts)
2196                 *ent_ts = next_ts;
2197
2198         if (missing_events)
2199                 *missing_events = next_lost;
2200
2201         return next;
2202 }
2203
2204 /* Find the next real entry, without updating the iterator itself */
2205 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2206                                           int *ent_cpu, u64 *ent_ts)
2207 {
2208         return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2209 }
2210
2211 /* Find the next real entry, and increment the iterator to the next entry */
2212 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2213 {
2214         iter->ent = __find_next_entry(iter, &iter->cpu,
2215                                       &iter->lost_events, &iter->ts);
2216
2217         if (iter->ent)
2218                 trace_iterator_increment(iter);
2219
2220         return iter->ent ? iter : NULL;
2221 }
2222
2223 static void trace_consume(struct trace_iterator *iter)
2224 {
2225         ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2226                             &iter->lost_events);
2227 }
2228
2229 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2230 {
2231         struct trace_iterator *iter = m->private;
2232         int i = (int)*pos;
2233         void *ent;
2234
2235         WARN_ON_ONCE(iter->leftover);
2236
2237         (*pos)++;
2238
2239         /* can't go backwards */
2240         if (iter->idx > i)
2241                 return NULL;
2242
2243         if (iter->idx < 0)
2244                 ent = trace_find_next_entry_inc(iter);
2245         else
2246                 ent = iter;
2247
2248         while (ent && iter->idx < i)
2249                 ent = trace_find_next_entry_inc(iter);
2250
2251         iter->pos = *pos;
2252
2253         return ent;
2254 }
2255
2256 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2257 {
2258         struct ring_buffer_event *event;
2259         struct ring_buffer_iter *buf_iter;
2260         unsigned long entries = 0;
2261         u64 ts;
2262
2263         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2264
2265         buf_iter = trace_buffer_iter(iter, cpu);
2266         if (!buf_iter)
2267                 return;
2268
2269         ring_buffer_iter_reset(buf_iter);
2270
2271         /*
2272          * We could have the case with the max latency tracers
2273          * that a reset never took place on a cpu. This is evident
2274          * by the timestamp being before the start of the buffer.
2275          */
2276         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2277                 if (ts >= iter->trace_buffer->time_start)
2278                         break;
2279                 entries++;
2280                 ring_buffer_read(buf_iter, NULL);
2281         }
2282
2283         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2284 }
2285
2286 /*
2287  * The current tracer is copied to avoid a global locking
2288  * all around.
2289  */
2290 static void *s_start(struct seq_file *m, loff_t *pos)
2291 {
2292         struct trace_iterator *iter = m->private;
2293         struct trace_array *tr = iter->tr;
2294         int cpu_file = iter->cpu_file;
2295         void *p = NULL;
2296         loff_t l = 0;
2297         int cpu;
2298
2299         /*
2300          * copy the tracer to avoid using a global lock all around.
2301          * iter->trace is a copy of current_trace, the pointer to the
2302          * name may be used instead of a strcmp(), as iter->trace->name
2303          * will point to the same string as current_trace->name.
2304          */
2305         mutex_lock(&trace_types_lock);
2306         if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2307                 *iter->trace = *tr->current_trace;
2308         mutex_unlock(&trace_types_lock);
2309
2310 #ifdef CONFIG_TRACER_MAX_TRACE
2311         if (iter->snapshot && iter->trace->use_max_tr)
2312                 return ERR_PTR(-EBUSY);
2313 #endif
2314
2315         if (!iter->snapshot)
2316                 atomic_inc(&trace_record_cmdline_disabled);
2317
2318         if (*pos != iter->pos) {
2319                 iter->ent = NULL;
2320                 iter->cpu = 0;
2321                 iter->idx = -1;
2322
2323                 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2324                         for_each_tracing_cpu(cpu)
2325                                 tracing_iter_reset(iter, cpu);
2326                 } else
2327                         tracing_iter_reset(iter, cpu_file);
2328
2329                 iter->leftover = 0;
2330                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2331                         ;
2332
2333         } else {
2334                 /*
2335                  * If we overflowed the seq_file before, then we want
2336                  * to just reuse the trace_seq buffer again.
2337                  */
2338                 if (iter->leftover)
2339                         p = iter;
2340                 else {
2341                         l = *pos - 1;
2342                         p = s_next(m, p, &l);
2343                 }
2344         }
2345
2346         trace_event_read_lock();
2347         trace_access_lock(cpu_file);
2348         return p;
2349 }
2350
2351 static void s_stop(struct seq_file *m, void *p)
2352 {
2353         struct trace_iterator *iter = m->private;
2354
2355 #ifdef CONFIG_TRACER_MAX_TRACE
2356         if (iter->snapshot && iter->trace->use_max_tr)
2357                 return;
2358 #endif
2359
2360         if (!iter->snapshot)
2361                 atomic_dec(&trace_record_cmdline_disabled);
2362
2363         trace_access_unlock(iter->cpu_file);
2364         trace_event_read_unlock();
2365 }
2366
2367 static void
2368 get_total_entries(struct trace_buffer *buf,
2369                   unsigned long *total, unsigned long *entries)
2370 {
2371         unsigned long count;
2372         int cpu;
2373
2374         *total = 0;
2375         *entries = 0;
2376
2377         for_each_tracing_cpu(cpu) {
2378                 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2379                 /*
2380                  * If this buffer has skipped entries, then we hold all
2381                  * entries for the trace and we need to ignore the
2382                  * ones before the time stamp.
2383                  */
2384                 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2385                         count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2386                         /* total is the same as the entries */
2387                         *total += count;
2388                 } else
2389                         *total += count +
2390                                 ring_buffer_overrun_cpu(buf->buffer, cpu);
2391                 *entries += count;
2392         }
2393 }
2394
2395 static void print_lat_help_header(struct seq_file *m)
2396 {
2397         seq_puts(m, "#                  _------=> CPU#            \n");
2398         seq_puts(m, "#                 / _-----=> irqs-off        \n");
2399         seq_puts(m, "#                | / _----=> need-resched    \n");
2400         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
2401         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
2402         seq_puts(m, "#                |||| /     delay             \n");
2403         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
2404         seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
2405 }
2406
2407 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2408 {
2409         unsigned long total;
2410         unsigned long entries;
2411
2412         get_total_entries(buf, &total, &entries);
2413         seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
2414                    entries, total, num_online_cpus());
2415         seq_puts(m, "#\n");
2416 }
2417
2418 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2419 {
2420         print_event_info(buf, m);
2421         seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2422         seq_puts(m, "#              | |       |          |         |\n");
2423 }
2424
2425 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2426 {
2427         print_event_info(buf, m);
2428         seq_puts(m, "#                              _-----=> irqs-off\n");
2429         seq_puts(m, "#                             / _----=> need-resched\n");
2430         seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
2431         seq_puts(m, "#                            || / _--=> preempt-depth\n");
2432         seq_puts(m, "#                            ||| /     delay\n");
2433         seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
2434         seq_puts(m, "#              | |       |   ||||       |         |\n");
2435 }
2436
2437 void
2438 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2439 {
2440         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2441         struct trace_buffer *buf = iter->trace_buffer;
2442         struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2443         struct tracer *type = iter->trace;
2444         unsigned long entries;
2445         unsigned long total;
2446         const char *name = "preemption";
2447
2448         name = type->name;
2449
2450         get_total_entries(buf, &total, &entries);
2451
2452         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2453                    name, UTS_RELEASE);
2454         seq_puts(m, "# -----------------------------------"
2455                  "---------------------------------\n");
2456         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2457                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2458                    nsecs_to_usecs(data->saved_latency),
2459                    entries,
2460                    total,
2461                    buf->cpu,
2462 #if defined(CONFIG_PREEMPT_NONE)
2463                    "server",
2464 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2465                    "desktop",
2466 #elif defined(CONFIG_PREEMPT)
2467                    "preempt",
2468 #else
2469                    "unknown",
2470 #endif
2471                    /* These are reserved for later use */
2472                    0, 0, 0, 0);
2473 #ifdef CONFIG_SMP
2474         seq_printf(m, " #P:%d)\n", num_online_cpus());
2475 #else
2476         seq_puts(m, ")\n");
2477 #endif
2478         seq_puts(m, "#    -----------------\n");
2479         seq_printf(m, "#    | task: %.16s-%d "
2480                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2481                    data->comm, data->pid,
2482                    from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2483                    data->policy, data->rt_priority);
2484         seq_puts(m, "#    -----------------\n");
2485
2486         if (data->critical_start) {
2487                 seq_puts(m, "#  => started at: ");
2488                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2489                 trace_print_seq(m, &iter->seq);
2490                 seq_puts(m, "\n#  => ended at:   ");
2491                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2492                 trace_print_seq(m, &iter->seq);
2493                 seq_puts(m, "\n#\n");
2494         }
2495
2496         seq_puts(m, "#\n");
2497 }
2498
2499 static void test_cpu_buff_start(struct trace_iterator *iter)
2500 {
2501         struct trace_seq *s = &iter->seq;
2502
2503         if (!(trace_flags & TRACE_ITER_ANNOTATE))
2504                 return;
2505
2506         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2507                 return;
2508
2509         if (cpumask_test_cpu(iter->cpu, iter->started))
2510                 return;
2511
2512         if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2513                 return;
2514
2515         cpumask_set_cpu(iter->cpu, iter->started);
2516
2517         /* Don't print started cpu buffer for the first entry of the trace */
2518         if (iter->idx > 1)
2519                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2520                                 iter->cpu);
2521 }
2522
2523 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2524 {
2525         struct trace_seq *s = &iter->seq;
2526         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2527         struct trace_entry *entry;
2528         struct trace_event *event;
2529
2530         entry = iter->ent;
2531
2532         test_cpu_buff_start(iter);
2533
2534         event = ftrace_find_event(entry->type);
2535
2536         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2537                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2538                         if (!trace_print_lat_context(iter))
2539                                 goto partial;
2540                 } else {
2541                         if (!trace_print_context(iter))
2542                                 goto partial;
2543                 }
2544         }
2545
2546         if (event)
2547                 return event->funcs->trace(iter, sym_flags, event);
2548
2549         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2550                 goto partial;
2551
2552         return TRACE_TYPE_HANDLED;
2553 partial:
2554         return TRACE_TYPE_PARTIAL_LINE;
2555 }
2556
2557 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2558 {
2559         struct trace_seq *s = &iter->seq;
2560         struct trace_entry *entry;
2561         struct trace_event *event;
2562
2563         entry = iter->ent;
2564
2565         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2566                 if (!trace_seq_printf(s, "%d %d %llu ",
2567                                       entry->pid, iter->cpu, iter->ts))
2568                         goto partial;
2569         }
2570
2571         event = ftrace_find_event(entry->type);
2572         if (event)
2573                 return event->funcs->raw(iter, 0, event);
2574
2575         if (!trace_seq_printf(s, "%d ?\n", entry->type))
2576                 goto partial;
2577
2578         return TRACE_TYPE_HANDLED;
2579 partial:
2580         return TRACE_TYPE_PARTIAL_LINE;
2581 }
2582
2583 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2584 {
2585         struct trace_seq *s = &iter->seq;
2586         unsigned char newline = '\n';
2587         struct trace_entry *entry;
2588         struct trace_event *event;
2589
2590         entry = iter->ent;
2591
2592         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2593                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2594                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2595                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2596         }
2597
2598         event = ftrace_find_event(entry->type);
2599         if (event) {
2600                 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2601                 if (ret != TRACE_TYPE_HANDLED)
2602                         return ret;
2603         }
2604
2605         SEQ_PUT_FIELD_RET(s, newline);
2606
2607         return TRACE_TYPE_HANDLED;
2608 }
2609
2610 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2611 {
2612         struct trace_seq *s = &iter->seq;
2613         struct trace_entry *entry;
2614         struct trace_event *event;
2615
2616         entry = iter->ent;
2617
2618         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2619                 SEQ_PUT_FIELD_RET(s, entry->pid);
2620                 SEQ_PUT_FIELD_RET(s, iter->cpu);
2621                 SEQ_PUT_FIELD_RET(s, iter->ts);
2622         }
2623
2624         event = ftrace_find_event(entry->type);
2625         return event ? event->funcs->binary(iter, 0, event) :
2626                 TRACE_TYPE_HANDLED;
2627 }
2628
2629 int trace_empty(struct trace_iterator *iter)
2630 {
2631         struct ring_buffer_iter *buf_iter;
2632         int cpu;
2633
2634         /* If we are looking at one CPU buffer, only check that one */
2635         if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2636                 cpu = iter->cpu_file;
2637                 buf_iter = trace_buffer_iter(iter, cpu);
2638                 if (buf_iter) {
2639                         if (!ring_buffer_iter_empty(buf_iter))
2640                                 return 0;
2641                 } else {
2642                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2643                                 return 0;
2644                 }
2645                 return 1;
2646         }
2647
2648         for_each_tracing_cpu(cpu) {
2649                 buf_iter = trace_buffer_iter(iter, cpu);
2650                 if (buf_iter) {
2651                         if (!ring_buffer_iter_empty(buf_iter))
2652                                 return 0;
2653                 } else {
2654                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2655                                 return 0;
2656                 }
2657         }
2658
2659         return 1;
2660 }
2661
2662 /*  Called with trace_event_read_lock() held. */
2663 enum print_line_t print_trace_line(struct trace_iterator *iter)
2664 {
2665         enum print_line_t ret;
2666
2667         if (iter->lost_events &&
2668             !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2669                                  iter->cpu, iter->lost_events))
2670                 return TRACE_TYPE_PARTIAL_LINE;
2671
2672         if (iter->trace && iter->trace->print_line) {
2673                 ret = iter->trace->print_line(iter);
2674                 if (ret != TRACE_TYPE_UNHANDLED)
2675                         return ret;
2676         }
2677
2678         if (iter->ent->type == TRACE_BPUTS &&
2679                         trace_flags & TRACE_ITER_PRINTK &&
2680                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2681                 return trace_print_bputs_msg_only(iter);
2682
2683         if (iter->ent->type == TRACE_BPRINT &&
2684                         trace_flags & TRACE_ITER_PRINTK &&
2685                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2686                 return trace_print_bprintk_msg_only(iter);
2687
2688         if (iter->ent->type == TRACE_PRINT &&
2689                         trace_flags & TRACE_ITER_PRINTK &&
2690                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2691                 return trace_print_printk_msg_only(iter);
2692
2693         if (trace_flags & TRACE_ITER_BIN)
2694                 return print_bin_fmt(iter);
2695
2696         if (trace_flags & TRACE_ITER_HEX)
2697                 return print_hex_fmt(iter);
2698
2699         if (trace_flags & TRACE_ITER_RAW)
2700                 return print_raw_fmt(iter);
2701
2702         return print_trace_fmt(iter);
2703 }
2704
2705 void trace_latency_header(struct seq_file *m)
2706 {
2707         struct trace_iterator *iter = m->private;
2708
2709         /* print nothing if the buffers are empty */
2710         if (trace_empty(iter))
2711                 return;
2712
2713         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2714                 print_trace_header(m, iter);
2715
2716         if (!(trace_flags & TRACE_ITER_VERBOSE))
2717                 print_lat_help_header(m);
2718 }
2719
2720 void trace_default_header(struct seq_file *m)
2721 {
2722         struct trace_iterator *iter = m->private;
2723
2724         if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2725                 return;
2726
2727         if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2728                 /* print nothing if the buffers are empty */
2729                 if (trace_empty(iter))
2730                         return;
2731                 print_trace_header(m, iter);
2732                 if (!(trace_flags & TRACE_ITER_VERBOSE))
2733                         print_lat_help_header(m);
2734         } else {
2735                 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2736                         if (trace_flags & TRACE_ITER_IRQ_INFO)
2737                                 print_func_help_header_irq(iter->trace_buffer, m);
2738                         else
2739                                 print_func_help_header(iter->trace_buffer, m);
2740                 }
2741         }
2742 }
2743
2744 static void test_ftrace_alive(struct seq_file *m)
2745 {
2746         if (!ftrace_is_dead())
2747                 return;
2748         seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2749         seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
2750 }
2751
2752 #ifdef CONFIG_TRACER_MAX_TRACE
2753 static void show_snapshot_main_help(struct seq_file *m)
2754 {
2755         seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2756         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2757         seq_printf(m, "#                      Takes a snapshot of the main buffer.\n");
2758         seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2759         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2760         seq_printf(m, "#                       is not a '0' or '1')\n");
2761 }
2762
2763 static void show_snapshot_percpu_help(struct seq_file *m)
2764 {
2765         seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2766 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2767         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2768         seq_printf(m, "#                      Takes a snapshot of the main buffer for this cpu.\n");
2769 #else
2770         seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2771         seq_printf(m, "#                     Must use main snapshot file to allocate.\n");
2772 #endif
2773         seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2774         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2775         seq_printf(m, "#                       is not a '0' or '1')\n");
2776 }
2777
2778 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2779 {
2780         if (iter->tr->allocated_snapshot)
2781                 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2782         else
2783                 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2784
2785         seq_printf(m, "# Snapshot commands:\n");
2786         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2787                 show_snapshot_main_help(m);
2788         else
2789                 show_snapshot_percpu_help(m);
2790 }
2791 #else
2792 /* Should never be called */
2793 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2794 #endif
2795
2796 static int s_show(struct seq_file *m, void *v)
2797 {
2798         struct trace_iterator *iter = v;
2799         int ret;
2800
2801         if (iter->ent == NULL) {
2802                 if (iter->tr) {
2803                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
2804                         seq_puts(m, "#\n");
2805                         test_ftrace_alive(m);
2806                 }
2807                 if (iter->snapshot && trace_empty(iter))
2808                         print_snapshot_help(m, iter);
2809                 else if (iter->trace && iter->trace->print_header)
2810                         iter->trace->print_header(m);
2811                 else
2812                         trace_default_header(m);
2813
2814         } else if (iter->leftover) {
2815                 /*
2816                  * If we filled the seq_file buffer earlier, we
2817                  * want to just show it now.
2818                  */
2819                 ret = trace_print_seq(m, &iter->seq);
2820
2821                 /* ret should this time be zero, but you never know */
2822                 iter->leftover = ret;
2823
2824         } else {
2825                 print_trace_line(iter);
2826                 ret = trace_print_seq(m, &iter->seq);
2827                 /*
2828                  * If we overflow the seq_file buffer, then it will
2829                  * ask us for this data again at start up.
2830                  * Use that instead.
2831                  *  ret is 0 if seq_file write succeeded.
2832                  *        -1 otherwise.
2833                  */
2834                 iter->leftover = ret;
2835         }
2836
2837         return 0;
2838 }
2839
2840 /*
2841  * Should be used after trace_array_get(), trace_types_lock
2842  * ensures that i_cdev was already initialized.
2843  */
2844 static inline int tracing_get_cpu(struct inode *inode)
2845 {
2846         if (inode->i_cdev) /* See trace_create_cpu_file() */
2847                 return (long)inode->i_cdev - 1;
2848         return RING_BUFFER_ALL_CPUS;
2849 }
2850
2851 static const struct seq_operations tracer_seq_ops = {
2852         .start          = s_start,
2853         .next           = s_next,
2854         .stop           = s_stop,
2855         .show           = s_show,
2856 };
2857
2858 static struct trace_iterator *
2859 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2860 {
2861         struct trace_array *tr = inode->i_private;
2862         struct trace_iterator *iter;
2863         int cpu;
2864
2865         if (tracing_disabled)
2866                 return ERR_PTR(-ENODEV);
2867
2868         iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2869         if (!iter)
2870                 return ERR_PTR(-ENOMEM);
2871
2872         iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2873                                     GFP_KERNEL);
2874         if (!iter->buffer_iter)
2875                 goto release;
2876
2877         /*
2878          * We make a copy of the current tracer to avoid concurrent
2879          * changes on it while we are reading.
2880          */
2881         mutex_lock(&trace_types_lock);
2882         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2883         if (!iter->trace)
2884                 goto fail;
2885
2886         *iter->trace = *tr->current_trace;
2887
2888         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2889                 goto fail;
2890
2891         iter->tr = tr;
2892
2893 #ifdef CONFIG_TRACER_MAX_TRACE
2894         /* Currently only the top directory has a snapshot */
2895         if (tr->current_trace->print_max || snapshot)
2896                 iter->trace_buffer = &tr->max_buffer;
2897         else
2898 #endif
2899                 iter->trace_buffer = &tr->trace_buffer;
2900         iter->snapshot = snapshot;
2901         iter->pos = -1;
2902         iter->cpu_file = tracing_get_cpu(inode);
2903         mutex_init(&iter->mutex);
2904
2905         /* Notify the tracer early; before we stop tracing. */
2906         if (iter->trace && iter->trace->open)
2907                 iter->trace->open(iter);
2908
2909         /* Annotate start of buffers if we had overruns */
2910         if (ring_buffer_overruns(iter->trace_buffer->buffer))
2911                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2912
2913         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2914         if (trace_clocks[tr->clock_id].in_ns)
2915                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2916
2917         /* stop the trace while dumping if we are not opening "snapshot" */
2918         if (!iter->snapshot)
2919                 tracing_stop_tr(tr);
2920
2921         if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2922                 for_each_tracing_cpu(cpu) {
2923                         iter->buffer_iter[cpu] =
2924                                 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2925                 }
2926                 ring_buffer_read_prepare_sync();
2927                 for_each_tracing_cpu(cpu) {
2928                         ring_buffer_read_start(iter->buffer_iter[cpu]);
2929                         tracing_iter_reset(iter, cpu);
2930                 }
2931         } else {
2932                 cpu = iter->cpu_file;
2933                 iter->buffer_iter[cpu] =
2934                         ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2935                 ring_buffer_read_prepare_sync();
2936                 ring_buffer_read_start(iter->buffer_iter[cpu]);
2937                 tracing_iter_reset(iter, cpu);
2938         }
2939
2940         mutex_unlock(&trace_types_lock);
2941
2942         return iter;
2943
2944  fail:
2945         mutex_unlock(&trace_types_lock);
2946         kfree(iter->trace);
2947         kfree(iter->buffer_iter);
2948 release:
2949         seq_release_private(inode, file);
2950         return ERR_PTR(-ENOMEM);
2951 }
2952
2953 int tracing_open_generic(struct inode *inode, struct file *filp)
2954 {
2955         if (tracing_disabled)
2956                 return -ENODEV;
2957
2958         filp->private_data = inode->i_private;
2959         return 0;
2960 }
2961
2962 /*
2963  * Open and update trace_array ref count.
2964  * Must have the current trace_array passed to it.
2965  */
2966 int tracing_open_generic_tr(struct inode *inode, struct file *filp)
2967 {
2968         struct trace_array *tr = inode->i_private;
2969
2970         if (tracing_disabled)
2971                 return -ENODEV;
2972
2973         if (trace_array_get(tr) < 0)
2974                 return -ENODEV;
2975
2976         filp->private_data = inode->i_private;
2977
2978         return 0;
2979 }
2980
2981 static int tracing_release(struct inode *inode, struct file *file)
2982 {
2983         struct trace_array *tr = inode->i_private;
2984         struct seq_file *m = file->private_data;
2985         struct trace_iterator *iter;
2986         int cpu;
2987
2988         if (!(file->f_mode & FMODE_READ)) {
2989                 trace_array_put(tr);
2990                 return 0;
2991         }
2992
2993         /* Writes do not use seq_file */
2994         iter = m->private;
2995         mutex_lock(&trace_types_lock);
2996
2997         for_each_tracing_cpu(cpu) {
2998                 if (iter->buffer_iter[cpu])
2999                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
3000         }
3001
3002         if (iter->trace && iter->trace->close)
3003                 iter->trace->close(iter);
3004
3005         if (!iter->snapshot)
3006                 /* reenable tracing if it was previously enabled */
3007                 tracing_start_tr(tr);
3008
3009         __trace_array_put(tr);
3010
3011         mutex_unlock(&trace_types_lock);
3012
3013         mutex_destroy(&iter->mutex);
3014         free_cpumask_var(iter->started);
3015         kfree(iter->trace);
3016         kfree(iter->buffer_iter);
3017         seq_release_private(inode, file);
3018
3019         return 0;
3020 }
3021
3022 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3023 {
3024         struct trace_array *tr = inode->i_private;
3025
3026         trace_array_put(tr);
3027         return 0;
3028 }
3029
3030 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3031 {
3032         struct trace_array *tr = inode->i_private;
3033
3034         trace_array_put(tr);
3035
3036         return single_release(inode, file);
3037 }
3038
3039 static int tracing_open(struct inode *inode, struct file *file)
3040 {
3041         struct trace_array *tr = inode->i_private;
3042         struct trace_iterator *iter;
3043         int ret = 0;
3044
3045         if (trace_array_get(tr) < 0)
3046                 return -ENODEV;
3047
3048         /* If this file was open for write, then erase contents */
3049         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3050                 int cpu = tracing_get_cpu(inode);
3051
3052                 if (cpu == RING_BUFFER_ALL_CPUS)
3053                         tracing_reset_online_cpus(&tr->trace_buffer);
3054                 else
3055                         tracing_reset(&tr->trace_buffer, cpu);
3056         }
3057
3058         if (file->f_mode & FMODE_READ) {
3059                 iter = __tracing_open(inode, file, false);
3060                 if (IS_ERR(iter))
3061                         ret = PTR_ERR(iter);
3062                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3063                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
3064         }
3065
3066         if (ret < 0)
3067                 trace_array_put(tr);
3068
3069         return ret;
3070 }
3071
3072 static void *
3073 t_next(struct seq_file *m, void *v, loff_t *pos)
3074 {
3075         struct tracer *t = v;
3076
3077         (*pos)++;
3078
3079         if (t)
3080                 t = t->next;
3081
3082         return t;
3083 }
3084
3085 static void *t_start(struct seq_file *m, loff_t *pos)
3086 {
3087         struct tracer *t;
3088         loff_t l = 0;
3089
3090         mutex_lock(&trace_types_lock);
3091         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
3092                 ;
3093
3094         return t;
3095 }
3096
3097 static void t_stop(struct seq_file *m, void *p)
3098 {
3099         mutex_unlock(&trace_types_lock);
3100 }
3101
3102 static int t_show(struct seq_file *m, void *v)
3103 {
3104         struct tracer *t = v;
3105
3106         if (!t)
3107                 return 0;
3108
3109         seq_printf(m, "%s", t->name);
3110         if (t->next)
3111                 seq_putc(m, ' ');
3112         else
3113                 seq_putc(m, '\n');
3114
3115         return 0;
3116 }
3117
3118 static const struct seq_operations show_traces_seq_ops = {
3119         .start          = t_start,
3120         .next           = t_next,
3121         .stop           = t_stop,
3122         .show           = t_show,
3123 };
3124
3125 static int show_traces_open(struct inode *inode, struct file *file)
3126 {
3127         if (tracing_disabled)
3128                 return -ENODEV;
3129
3130         return seq_open(file, &show_traces_seq_ops);
3131 }
3132
3133 static ssize_t
3134 tracing_write_stub(struct file *filp, const char __user *ubuf,
3135                    size_t count, loff_t *ppos)
3136 {
3137         return count;
3138 }
3139
3140 static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
3141 {
3142         if (file->f_mode & FMODE_READ)
3143                 return seq_lseek(file, offset, origin);
3144         else
3145                 return 0;
3146 }
3147
3148 static const struct file_operations tracing_fops = {
3149         .open           = tracing_open,
3150         .read           = seq_read,
3151         .write          = tracing_write_stub,
3152         .llseek         = tracing_seek,
3153         .release        = tracing_release,
3154 };
3155
3156 static const struct file_operations show_traces_fops = {
3157         .open           = show_traces_open,
3158         .read           = seq_read,
3159         .release        = seq_release,
3160         .llseek         = seq_lseek,
3161 };
3162
3163 /*
3164  * Only trace on a CPU if the bitmask is set:
3165  */
3166 static cpumask_var_t tracing_cpumask;
3167
3168 /*
3169  * The tracer itself will not take this lock, but still we want
3170  * to provide a consistent cpumask to user-space:
3171  */
3172 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3173
3174 /*
3175  * Temporary storage for the character representation of the
3176  * CPU bitmask (and one more byte for the newline):
3177  */
3178 static char mask_str[NR_CPUS + 1];
3179
3180 static ssize_t
3181 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3182                      size_t count, loff_t *ppos)
3183 {
3184         int len;
3185
3186         mutex_lock(&tracing_cpumask_update_lock);
3187
3188         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
3189         if (count - len < 2) {
3190                 count = -EINVAL;
3191                 goto out_err;
3192         }
3193         len += sprintf(mask_str + len, "\n");
3194         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3195
3196 out_err:
3197         mutex_unlock(&tracing_cpumask_update_lock);
3198
3199         return count;
3200 }
3201
3202 static ssize_t
3203 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3204                       size_t count, loff_t *ppos)
3205 {
3206         struct trace_array *tr = filp->private_data;
3207         cpumask_var_t tracing_cpumask_new;
3208         int err, cpu;
3209
3210         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3211                 return -ENOMEM;
3212
3213         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3214         if (err)
3215                 goto err_unlock;
3216
3217         mutex_lock(&tracing_cpumask_update_lock);
3218
3219         local_irq_disable();
3220         arch_spin_lock(&ftrace_max_lock);
3221         for_each_tracing_cpu(cpu) {
3222                 /*
3223                  * Increase/decrease the disabled counter if we are
3224                  * about to flip a bit in the cpumask:
3225                  */
3226                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
3227                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3228                         atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3229                         ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3230                 }
3231                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
3232                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3233                         atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3234                         ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3235                 }
3236         }
3237         arch_spin_unlock(&ftrace_max_lock);
3238         local_irq_enable();
3239
3240         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
3241
3242         mutex_unlock(&tracing_cpumask_update_lock);
3243         free_cpumask_var(tracing_cpumask_new);
3244
3245         return count;
3246
3247 err_unlock:
3248         free_cpumask_var(tracing_cpumask_new);
3249
3250         return err;
3251 }
3252
3253 static const struct file_operations tracing_cpumask_fops = {
3254         .open           = tracing_open_generic,
3255         .read           = tracing_cpumask_read,
3256         .write          = tracing_cpumask_write,
3257         .llseek         = generic_file_llseek,
3258 };
3259
3260 static int tracing_trace_options_show(struct seq_file *m, void *v)
3261 {
3262         struct tracer_opt *trace_opts;
3263         struct trace_array *tr = m->private;
3264         u32 tracer_flags;
3265         int i;
3266
3267         mutex_lock(&trace_types_lock);
3268         tracer_flags = tr->current_trace->flags->val;
3269         trace_opts = tr->current_trace->flags->opts;
3270
3271         for (i = 0; trace_options[i]; i++) {
3272                 if (trace_flags & (1 << i))
3273                         seq_printf(m, "%s\n", trace_options[i]);
3274                 else
3275                         seq_printf(m, "no%s\n", trace_options[i]);
3276         }
3277
3278         for (i = 0; trace_opts[i].name; i++) {
3279                 if (tracer_flags & trace_opts[i].bit)
3280                         seq_printf(m, "%s\n", trace_opts[i].name);
3281                 else
3282                         seq_printf(m, "no%s\n", trace_opts[i].name);
3283         }
3284         mutex_unlock(&trace_types_lock);
3285
3286         return 0;
3287 }
3288
3289 static int __set_tracer_option(struct tracer *trace,
3290                                struct tracer_flags *tracer_flags,
3291                                struct tracer_opt *opts, int neg)
3292 {
3293         int ret;
3294
3295         ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3296         if (ret)
3297                 return ret;
3298
3299         if (neg)
3300                 tracer_flags->val &= ~opts->bit;
3301         else
3302                 tracer_flags->val |= opts->bit;
3303         return 0;
3304 }
3305
3306 /* Try to assign a tracer specific option */
3307 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3308 {
3309         struct tracer_flags *tracer_flags = trace->flags;
3310         struct tracer_opt *opts = NULL;
3311         int i;
3312
3313         for (i = 0; tracer_flags->opts[i].name; i++) {
3314                 opts = &tracer_flags->opts[i];
3315
3316                 if (strcmp(cmp, opts->name) == 0)
3317                         return __set_tracer_option(trace, trace->flags,
3318                                                    opts, neg);
3319         }
3320
3321         return -EINVAL;
3322 }
3323
3324 /* Some tracers require overwrite to stay enabled */
3325 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3326 {
3327         if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3328                 return -1;
3329
3330         return 0;
3331 }
3332
3333 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3334 {
3335         /* do nothing if flag is already set */
3336         if (!!(trace_flags & mask) == !!enabled)
3337                 return 0;
3338
3339         /* Give the tracer a chance to approve the change */
3340         if (tr->current_trace->flag_changed)
3341                 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
3342                         return -EINVAL;
3343
3344         if (enabled)
3345                 trace_flags |= mask;
3346         else
3347                 trace_flags &= ~mask;
3348
3349         if (mask == TRACE_ITER_RECORD_CMD)
3350                 trace_event_enable_cmd_record(enabled);
3351
3352         if (mask == TRACE_ITER_OVERWRITE) {
3353                 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3354 #ifdef CONFIG_TRACER_MAX_TRACE
3355                 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3356 #endif
3357         }
3358
3359         if (mask == TRACE_ITER_PRINTK)
3360                 trace_printk_start_stop_comm(enabled);
3361
3362         return 0;
3363 }
3364
3365 static int trace_set_options(struct trace_array *tr, char *option)
3366 {
3367         char *cmp;
3368         int neg = 0;
3369         int ret = -ENODEV;
3370         int i;
3371
3372         cmp = strstrip(option);
3373
3374         if (strncmp(cmp, "no", 2) == 0) {
3375                 neg = 1;
3376                 cmp += 2;
3377         }
3378
3379         mutex_lock(&trace_types_lock);
3380
3381         for (i = 0; trace_options[i]; i++) {
3382                 if (strcmp(cmp, trace_options[i]) == 0) {
3383                         ret = set_tracer_flag(tr, 1 << i, !neg);
3384                         break;
3385                 }
3386         }
3387
3388         /* If no option could be set, test the specific tracer options */
3389         if (!trace_options[i])
3390                 ret = set_tracer_option(tr->current_trace, cmp, neg);
3391
3392         mutex_unlock(&trace_types_lock);
3393
3394         return ret;
3395 }
3396
3397 static ssize_t
3398 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3399                         size_t cnt, loff_t *ppos)
3400 {
3401         struct seq_file *m = filp->private_data;
3402         struct trace_array *tr = m->private;
3403         char buf[64];
3404         int ret;
3405
3406         if (cnt >= sizeof(buf))
3407                 return -EINVAL;
3408
3409         if (copy_from_user(&buf, ubuf, cnt))
3410                 return -EFAULT;
3411
3412         buf[cnt] = 0;
3413
3414         ret = trace_set_options(tr, buf);
3415         if (ret < 0)
3416                 return ret;
3417
3418         *ppos += cnt;
3419
3420         return cnt;
3421 }
3422
3423 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3424 {
3425         struct trace_array *tr = inode->i_private;
3426         int ret;
3427
3428         if (tracing_disabled)
3429                 return -ENODEV;
3430
3431         if (trace_array_get(tr) < 0)
3432                 return -ENODEV;
3433
3434         ret = single_open(file, tracing_trace_options_show, inode->i_private);
3435         if (ret < 0)
3436                 trace_array_put(tr);
3437
3438         return ret;
3439 }
3440
3441 static const struct file_operations tracing_iter_fops = {
3442         .open           = tracing_trace_options_open,
3443         .read           = seq_read,
3444         .llseek         = seq_lseek,
3445         .release        = tracing_single_release_tr,
3446         .write          = tracing_trace_options_write,
3447 };
3448
3449 static const char readme_msg[] =
3450         "tracing mini-HOWTO:\n\n"
3451         "# echo 0 > tracing_on : quick way to disable tracing\n"
3452         "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3453         " Important files:\n"
3454         "  trace\t\t\t- The static contents of the buffer\n"
3455         "\t\t\t  To clear the buffer write into this file: echo > trace\n"
3456         "  trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3457         "  current_tracer\t- function and latency tracers\n"
3458         "  available_tracers\t- list of configured tracers for current_tracer\n"
3459         "  buffer_size_kb\t- view and modify size of per cpu buffer\n"
3460         "  buffer_total_size_kb  - view total size of all cpu buffers\n\n"
3461         "  trace_clock\t\t-change the clock used to order events\n"
3462         "       local:   Per cpu clock but may not be synced across CPUs\n"
3463         "      global:   Synced across CPUs but slows tracing down.\n"
3464         "     counter:   Not a clock, but just an increment\n"
3465         "      uptime:   Jiffy counter from time of boot\n"
3466         "        perf:   Same clock that perf events use\n"
3467 #ifdef CONFIG_X86_64
3468         "     x86-tsc:   TSC cycle counter\n"
3469 #endif
3470         "\n  trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3471         "  tracing_cpumask\t- Limit which CPUs to trace\n"
3472         "  instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3473         "\t\t\t  Remove sub-buffer with rmdir\n"
3474         "  trace_options\t\t- Set format or modify how tracing happens\n"
3475         "\t\t\t  Disable an option by adding a suffix 'no' to the option name\n"
3476 #ifdef CONFIG_DYNAMIC_FTRACE
3477         "\n  available_filter_functions - list of functions that can be filtered on\n"
3478         "  set_ftrace_filter\t- echo function name in here to only trace these functions\n"
3479         "            accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3480         "            modules: Can select a group via module\n"
3481         "             Format: :mod:<module-name>\n"
3482         "             example: echo :mod:ext3 > set_ftrace_filter\n"
3483         "            triggers: a command to perform when function is hit\n"
3484         "              Format: <function>:<trigger>[:count]\n"
3485         "             trigger: traceon, traceoff\n"
3486         "                      enable_event:<system>:<event>\n"
3487         "                      disable_event:<system>:<event>\n"
3488 #ifdef CONFIG_STACKTRACE
3489         "                      stacktrace\n"
3490 #endif
3491 #ifdef CONFIG_TRACER_SNAPSHOT
3492         "                      snapshot\n"
3493 #endif
3494         "             example: echo do_fault:traceoff > set_ftrace_filter\n"
3495         "                      echo do_trap:traceoff:3 > set_ftrace_filter\n"
3496         "             The first one will disable tracing every time do_fault is hit\n"
3497         "             The second will disable tracing at most 3 times when do_trap is hit\n"
3498         "               The first time do trap is hit and it disables tracing, the counter\n"
3499         "               will decrement to 2. If tracing is already disabled, the counter\n"
3500         "               will not decrement. It only decrements when the trigger did work\n"
3501         "             To remove trigger without count:\n"
3502         "               echo '!<function>:<trigger> > set_ftrace_filter\n"
3503         "             To remove trigger with a count:\n"
3504         "               echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3505         "  set_ftrace_notrace\t- echo function name in here to never trace.\n"
3506         "            accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3507         "            modules: Can select a group via module command :mod:\n"
3508         "            Does not accept triggers\n"
3509 #endif /* CONFIG_DYNAMIC_FTRACE */
3510 #ifdef CONFIG_FUNCTION_TRACER
3511         "  set_ftrace_pid\t- Write pid(s) to only function trace those pids (function)\n"
3512 #endif
3513 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3514         "  set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3515         "  max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3516 #endif
3517 #ifdef CONFIG_TRACER_SNAPSHOT
3518         "\n  snapshot\t\t- Like 'trace' but shows the content of the static snapshot buffer\n"
3519         "\t\t\t  Read the contents for more information\n"
3520 #endif
3521 #ifdef CONFIG_STACKTRACE
3522         "  stack_trace\t\t- Shows the max stack trace when active\n"
3523         "  stack_max_size\t- Shows current max stack size that was traced\n"
3524         "\t\t\t  Write into this file to reset the max size (trigger a new trace)\n"
3525 #ifdef CONFIG_DYNAMIC_FTRACE
3526         "  stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace traces\n"
3527 #endif
3528 #endif /* CONFIG_STACKTRACE */
3529 ;
3530
3531 static ssize_t
3532 tracing_readme_read(struct file *filp, char __user *ubuf,
3533                        size_t cnt, loff_t *ppos)
3534 {
3535         return simple_read_from_buffer(ubuf, cnt, ppos,
3536                                         readme_msg, strlen(readme_msg));
3537 }
3538
3539 static const struct file_operations tracing_readme_fops = {
3540         .open           = tracing_open_generic,
3541         .read           = tracing_readme_read,
3542         .llseek         = generic_file_llseek,
3543 };
3544
3545 static ssize_t
3546 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3547                                 size_t cnt, loff_t *ppos)
3548 {
3549         char *buf_comm;
3550         char *file_buf;
3551         char *buf;
3552         int len = 0;
3553         int pid;
3554         int i;
3555
3556         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3557         if (!file_buf)
3558                 return -ENOMEM;
3559
3560         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3561         if (!buf_comm) {
3562                 kfree(file_buf);
3563                 return -ENOMEM;
3564         }
3565
3566         buf = file_buf;
3567
3568         for (i = 0; i < SAVED_CMDLINES; i++) {
3569                 int r;
3570
3571                 pid = map_cmdline_to_pid[i];
3572                 if (pid == -1 || pid == NO_CMDLINE_MAP)
3573                         continue;
3574
3575                 trace_find_cmdline(pid, buf_comm);
3576                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3577                 buf += r;
3578                 len += r;
3579         }
3580
3581         len = simple_read_from_buffer(ubuf, cnt, ppos,
3582                                       file_buf, len);
3583
3584         kfree(file_buf);
3585         kfree(buf_comm);
3586
3587         return len;
3588 }
3589
3590 static const struct file_operations tracing_saved_cmdlines_fops = {
3591     .open       = tracing_open_generic,
3592     .read       = tracing_saved_cmdlines_read,
3593     .llseek     = generic_file_llseek,
3594 };
3595
3596 static ssize_t
3597 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3598                        size_t cnt, loff_t *ppos)
3599 {
3600         struct trace_array *tr = filp->private_data;
3601         char buf[MAX_TRACER_SIZE+2];
3602         int r;
3603
3604         mutex_lock(&trace_types_lock);
3605         r = sprintf(buf, "%s\n", tr->current_trace->name);
3606         mutex_unlock(&trace_types_lock);
3607
3608         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3609 }
3610
3611 int tracer_init(struct tracer *t, struct trace_array *tr)
3612 {
3613         tracing_reset_online_cpus(&tr->trace_buffer);
3614         return t->init(tr);
3615 }
3616
3617 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3618 {
3619         int cpu;
3620
3621         for_each_tracing_cpu(cpu)
3622                 per_cpu_ptr(buf->data, cpu)->entries = val;
3623 }
3624
3625 #ifdef CONFIG_TRACER_MAX_TRACE
3626 /* resize @tr's buffer to the size of @size_tr's entries */
3627 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3628                                         struct trace_buffer *size_buf, int cpu_id)
3629 {
3630         int cpu, ret = 0;
3631
3632         if (cpu_id == RING_BUFFER_ALL_CPUS) {
3633                 for_each_tracing_cpu(cpu) {
3634                         ret = ring_buffer_resize(trace_buf->buffer,
3635                                  per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3636                         if (ret < 0)
3637                                 break;
3638                         per_cpu_ptr(trace_buf->data, cpu)->entries =
3639                                 per_cpu_ptr(size_buf->data, cpu)->entries;
3640                 }
3641         } else {
3642                 ret = ring_buffer_resize(trace_buf->buffer,
3643                                  per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3644                 if (ret == 0)
3645                         per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3646                                 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3647         }
3648
3649         return ret;
3650 }
3651 #endif /* CONFIG_TRACER_MAX_TRACE */
3652
3653 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3654                                         unsigned long size, int cpu)
3655 {
3656         int ret;
3657
3658         /*
3659          * If kernel or user changes the size of the ring buffer
3660          * we use the size that was given, and we can forget about
3661          * expanding it later.
3662          */
3663         ring_buffer_expanded = true;
3664
3665         /* May be called before buffers are initialized */
3666         if (!tr->trace_buffer.buffer)
3667                 return 0;
3668
3669         ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3670         if (ret < 0)
3671                 return ret;
3672
3673 #ifdef CONFIG_TRACER_MAX_TRACE
3674         if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3675             !tr->current_trace->use_max_tr)
3676                 goto out;
3677
3678         ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3679         if (ret < 0) {
3680                 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3681                                                      &tr->trace_buffer, cpu);
3682                 if (r < 0) {
3683                         /*
3684                          * AARGH! We are left with different
3685                          * size max buffer!!!!
3686                          * The max buffer is our "snapshot" buffer.
3687                          * When a tracer needs a snapshot (one of the
3688                          * latency tracers), it swaps the max buffer
3689                          * with the saved snap shot. We succeeded to
3690                          * update the size of the main buffer, but failed to
3691                          * update the size of the max buffer. But when we tried
3692                          * to reset the main buffer to the original size, we
3693                          * failed there too. This is very unlikely to
3694                          * happen, but if it does, warn and kill all
3695                          * tracing.
3696                          */
3697                         WARN_ON(1);
3698                         tracing_disabled = 1;
3699                 }
3700                 return ret;
3701         }
3702
3703         if (cpu == RING_BUFFER_ALL_CPUS)
3704                 set_buffer_entries(&tr->max_buffer, size);
3705         else
3706                 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3707
3708  out:
3709 #endif /* CONFIG_TRACER_MAX_TRACE */
3710
3711         if (cpu == RING_BUFFER_ALL_CPUS)
3712                 set_buffer_entries(&tr->trace_buffer, size);
3713         else
3714                 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
3715
3716         return ret;
3717 }
3718
3719 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3720                                           unsigned long size, int cpu_id)
3721 {
3722         int ret = size;
3723
3724         mutex_lock(&trace_types_lock);
3725
3726         if (cpu_id != RING_BUFFER_ALL_CPUS) {
3727                 /* make sure, this cpu is enabled in the mask */
3728                 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3729                         ret = -EINVAL;
3730                         goto out;
3731                 }
3732         }
3733
3734         ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3735         if (ret < 0)
3736                 ret = -ENOMEM;
3737
3738 out:
3739         mutex_unlock(&trace_types_lock);
3740
3741         return ret;
3742 }
3743
3744
3745 /**
3746  * tracing_update_buffers - used by tracing facility to expand ring buffers
3747  *
3748  * To save on memory when the tracing is never used on a system with it
3749  * configured in. The ring buffers are set to a minimum size. But once
3750  * a user starts to use the tracing facility, then they need to grow
3751  * to their default size.
3752  *
3753  * This function is to be called when a tracer is about to be used.
3754  */
3755 int tracing_update_buffers(void)
3756 {
3757         int ret = 0;
3758
3759         mutex_lock(&trace_types_lock);
3760         if (!ring_buffer_expanded)
3761                 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3762                                                 RING_BUFFER_ALL_CPUS);
3763         mutex_unlock(&trace_types_lock);
3764
3765         return ret;
3766 }
3767
3768 struct trace_option_dentry;
3769
3770 static struct trace_option_dentry *
3771 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
3772
3773 static void
3774 destroy_trace_option_files(struct trace_option_dentry *topts);
3775
3776 static int tracing_set_tracer(const char *buf)
3777 {
3778         static struct trace_option_dentry *topts;
3779         struct trace_array *tr = &global_trace;
3780         struct tracer *t;
3781 #ifdef CONFIG_TRACER_MAX_TRACE
3782         bool had_max_tr;
3783 #endif
3784         int ret = 0;
3785
3786         mutex_lock(&trace_types_lock);
3787
3788         if (!ring_buffer_expanded) {
3789                 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3790                                                 RING_BUFFER_ALL_CPUS);
3791                 if (ret < 0)
3792                         goto out;
3793                 ret = 0;
3794         }
3795
3796         for (t = trace_types; t; t = t->next) {
3797                 if (strcmp(t->name, buf) == 0)
3798                         break;
3799         }
3800         if (!t) {
3801                 ret = -EINVAL;
3802                 goto out;
3803         }
3804         if (t == tr->current_trace)
3805                 goto out;
3806
3807         trace_branch_disable();
3808
3809         tr->current_trace->enabled = false;
3810
3811         if (tr->current_trace->reset)
3812                 tr->current_trace->reset(tr);
3813
3814         /* Current trace needs to be nop_trace before synchronize_sched */
3815         tr->current_trace = &nop_trace;
3816
3817 #ifdef CONFIG_TRACER_MAX_TRACE
3818         had_max_tr = tr->allocated_snapshot;
3819
3820         if (had_max_tr && !t->use_max_tr) {
3821                 /*
3822                  * We need to make sure that the update_max_tr sees that
3823                  * current_trace changed to nop_trace to keep it from
3824                  * swapping the buffers after we resize it.
3825                  * The update_max_tr is called from interrupts disabled
3826                  * so a synchronized_sched() is sufficient.
3827                  */
3828                 synchronize_sched();
3829                 free_snapshot(tr);
3830         }
3831 #endif
3832         destroy_trace_option_files(topts);
3833
3834         topts = create_trace_option_files(tr, t);
3835
3836 #ifdef CONFIG_TRACER_MAX_TRACE
3837         if (t->use_max_tr && !had_max_tr) {
3838                 ret = alloc_snapshot(tr);
3839                 if (ret < 0)
3840                         goto out;
3841         }
3842 #endif
3843
3844         if (t->init) {
3845                 ret = tracer_init(t, tr);
3846                 if (ret)
3847                         goto out;
3848         }
3849
3850         tr->current_trace = t;
3851         tr->current_trace->enabled = true;
3852         trace_branch_enable(tr);
3853  out:
3854         mutex_unlock(&trace_types_lock);
3855
3856         return ret;
3857 }
3858
3859 static ssize_t
3860 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3861                         size_t cnt, loff_t *ppos)
3862 {
3863         char buf[MAX_TRACER_SIZE+1];
3864         int i;
3865         size_t ret;
3866         int err;
3867
3868         ret = cnt;
3869
3870         if (cnt > MAX_TRACER_SIZE)
3871                 cnt = MAX_TRACER_SIZE;
3872
3873         if (copy_from_user(&buf, ubuf, cnt))
3874                 return -EFAULT;
3875
3876         buf[cnt] = 0;
3877
3878         /* strip ending whitespace. */
3879         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3880                 buf[i] = 0;
3881
3882         err = tracing_set_tracer(buf);
3883         if (err)
3884                 return err;
3885
3886         *ppos += ret;
3887
3888         return ret;
3889 }
3890
3891 static ssize_t
3892 tracing_max_lat_read(struct file *filp, char __user *ubuf,
3893                      size_t cnt, loff_t *ppos)
3894 {
3895         unsigned long *ptr = filp->private_data;
3896         char buf[64];
3897         int r;
3898
3899         r = snprintf(buf, sizeof(buf), "%ld\n",
3900                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3901         if (r > sizeof(buf))
3902                 r = sizeof(buf);
3903         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3904 }
3905
3906 static ssize_t
3907 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3908                       size_t cnt, loff_t *ppos)
3909 {
3910         unsigned long *ptr = filp->private_data;
3911         unsigned long val;
3912         int ret;
3913
3914         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3915         if (ret)
3916                 return ret;
3917
3918         *ptr = val * 1000;
3919
3920         return cnt;
3921 }
3922
3923 static int tracing_open_pipe(struct inode *inode, struct file *filp)
3924 {
3925         struct trace_array *tr = inode->i_private;
3926         struct trace_iterator *iter;
3927         int ret = 0;
3928
3929         if (tracing_disabled)
3930                 return -ENODEV;
3931
3932         if (trace_array_get(tr) < 0)
3933                 return -ENODEV;
3934
3935         mutex_lock(&trace_types_lock);
3936
3937         /* create a buffer to store the information to pass to userspace */
3938         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3939         if (!iter) {
3940                 ret = -ENOMEM;
3941                 __trace_array_put(tr);
3942                 goto out;
3943         }
3944
3945         /*
3946          * We make a copy of the current tracer to avoid concurrent
3947          * changes on it while we are reading.
3948          */
3949         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3950         if (!iter->trace) {
3951                 ret = -ENOMEM;
3952                 goto fail;
3953         }
3954         *iter->trace = *tr->current_trace;
3955
3956         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3957                 ret = -ENOMEM;
3958                 goto fail;
3959         }
3960
3961         /* trace pipe does not show start of buffer */
3962         cpumask_setall(iter->started);
3963
3964         if (trace_flags & TRACE_ITER_LATENCY_FMT)
3965                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3966
3967         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3968         if (trace_clocks[tr->clock_id].in_ns)
3969                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3970
3971         iter->tr = tr;
3972         iter->trace_buffer = &tr->trace_buffer;
3973         iter->cpu_file = tracing_get_cpu(inode);
3974         mutex_init(&iter->mutex);
3975         filp->private_data = iter;
3976
3977         if (iter->trace->pipe_open)
3978                 iter->trace->pipe_open(iter);
3979
3980         nonseekable_open(inode, filp);
3981 out:
3982         mutex_unlock(&trace_types_lock);
3983         return ret;
3984
3985 fail:
3986         kfree(iter->trace);
3987         kfree(iter);
3988         __trace_array_put(tr);
3989         mutex_unlock(&trace_types_lock);
3990         return ret;
3991 }
3992
3993 static int tracing_release_pipe(struct inode *inode, struct file *file)
3994 {
3995         struct trace_iterator *iter = file->private_data;
3996         struct trace_array *tr = inode->i_private;
3997
3998         mutex_lock(&trace_types_lock);
3999
4000         if (iter->trace->pipe_close)
4001                 iter->trace->pipe_close(iter);
4002
4003         mutex_unlock(&trace_types_lock);
4004
4005         free_cpumask_var(iter->started);
4006         mutex_destroy(&iter->mutex);
4007         kfree(iter->trace);
4008         kfree(iter);
4009
4010         trace_array_put(tr);
4011
4012         return 0;
4013 }
4014
4015 static unsigned int
4016 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4017 {
4018         /* Iterators are static, they should be filled or empty */
4019         if (trace_buffer_iter(iter, iter->cpu_file))
4020                 return POLLIN | POLLRDNORM;
4021
4022         if (trace_flags & TRACE_ITER_BLOCK)
4023                 /*
4024                  * Always select as readable when in blocking mode
4025                  */
4026                 return POLLIN | POLLRDNORM;
4027         else
4028                 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4029                                              filp, poll_table);
4030 }
4031
4032 static unsigned int
4033 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4034 {
4035         struct trace_iterator *iter = filp->private_data;
4036
4037         return trace_poll(iter, filp, poll_table);
4038 }
4039
4040 /*
4041  * This is a make-shift waitqueue.
4042  * A tracer might use this callback on some rare cases:
4043  *
4044  *  1) the current tracer might hold the runqueue lock when it wakes up
4045  *     a reader, hence a deadlock (sched, function, and function graph tracers)
4046  *  2) the function tracers, trace all functions, we don't want
4047  *     the overhead of calling wake_up and friends
4048  *     (and tracing them too)
4049  *
4050  *     Anyway, this is really very primitive wakeup.
4051  */
4052 void poll_wait_pipe(struct trace_iterator *iter)
4053 {
4054         set_current_state(TASK_INTERRUPTIBLE);
4055         /* sleep for 100 msecs, and try again. */
4056         schedule_timeout(HZ / 10);
4057 }
4058
4059 /* Must be called with trace_types_lock mutex held. */
4060 static int tracing_wait_pipe(struct file *filp)
4061 {
4062         struct trace_iterator *iter = filp->private_data;
4063
4064         while (trace_empty(iter)) {
4065
4066                 if ((filp->f_flags & O_NONBLOCK)) {
4067                         return -EAGAIN;
4068                 }
4069
4070                 mutex_unlock(&iter->mutex);
4071
4072                 iter->trace->wait_pipe(iter);
4073
4074                 mutex_lock(&iter->mutex);
4075
4076                 if (signal_pending(current))
4077                         return -EINTR;
4078
4079                 /*
4080                  * We block until we read something and tracing is disabled.
4081                  * We still block if tracing is disabled, but we have never
4082                  * read anything. This allows a user to cat this file, and
4083                  * then enable tracing. But after we have read something,
4084                  * we give an EOF when tracing is again disabled.
4085                  *
4086                  * iter->pos will be 0 if we haven't read anything.
4087                  */
4088                 if (!tracing_is_on() && iter->pos)
4089                         break;
4090         }
4091
4092         return 1;
4093 }
4094
4095 /*
4096  * Consumer reader.
4097  */
4098 static ssize_t
4099 tracing_read_pipe(struct file *filp, char __user *ubuf,
4100                   size_t cnt, loff_t *ppos)
4101 {
4102         struct trace_iterator *iter = filp->private_data;
4103         struct trace_array *tr = iter->tr;
4104         ssize_t sret;
4105
4106         /* return any leftover data */
4107         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4108         if (sret != -EBUSY)
4109                 return sret;
4110
4111         trace_seq_init(&iter->seq);
4112
4113         /* copy the tracer to avoid using a global lock all around */
4114         mutex_lock(&trace_types_lock);
4115         if (unlikely(iter->trace->name != tr->current_trace->name))
4116                 *iter->trace = *tr->current_trace;
4117         mutex_unlock(&trace_types_lock);
4118
4119         /*
4120          * Avoid more than one consumer on a single file descriptor
4121          * This is just a matter of traces coherency, the ring buffer itself
4122          * is protected.
4123          */
4124         mutex_lock(&iter->mutex);
4125         if (iter->trace->read) {
4126                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4127                 if (sret)
4128                         goto out;
4129         }
4130
4131 waitagain:
4132         sret = tracing_wait_pipe(filp);
4133         if (sret <= 0)
4134                 goto out;
4135
4136         /* stop when tracing is finished */
4137         if (trace_empty(iter)) {
4138                 sret = 0;
4139                 goto out;
4140         }
4141
4142         if (cnt >= PAGE_SIZE)
4143                 cnt = PAGE_SIZE - 1;
4144
4145         /* reset all but tr, trace, and overruns */
4146         memset(&iter->seq, 0,
4147                sizeof(struct trace_iterator) -
4148                offsetof(struct trace_iterator, seq));
4149         cpumask_clear(iter->started);
4150         iter->pos = -1;
4151
4152         trace_event_read_lock();
4153         trace_access_lock(iter->cpu_file);
4154         while (trace_find_next_entry_inc(iter) != NULL) {
4155                 enum print_line_t ret;
4156                 int len = iter->seq.len;
4157
4158                 ret = print_trace_line(iter);
4159                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4160                         /* don't print partial lines */
4161                         iter->seq.len = len;
4162                         break;
4163                 }
4164                 if (ret != TRACE_TYPE_NO_CONSUME)
4165                         trace_consume(iter);
4166
4167                 if (iter->seq.len >= cnt)
4168                         break;
4169
4170                 /*
4171                  * Setting the full flag means we reached the trace_seq buffer
4172                  * size and we should leave by partial output condition above.
4173                  * One of the trace_seq_* functions is not used properly.
4174                  */
4175                 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4176                           iter->ent->type);
4177         }
4178         trace_access_unlock(iter->cpu_file);
4179         trace_event_read_unlock();
4180
4181         /* Now copy what we have to the user */
4182         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4183         if (iter->seq.readpos >= iter->seq.len)
4184                 trace_seq_init(&iter->seq);
4185
4186         /*
4187          * If there was nothing to send to user, in spite of consuming trace
4188          * entries, go back to wait for more entries.
4189          */
4190         if (sret == -EBUSY)
4191                 goto waitagain;
4192
4193 out:
4194         mutex_unlock(&iter->mutex);
4195
4196         return sret;
4197 }
4198
4199 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
4200                                      struct pipe_buffer *buf)
4201 {
4202         __free_page(buf->page);
4203 }
4204
4205 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4206                                      unsigned int idx)
4207 {
4208         __free_page(spd->pages[idx]);
4209 }
4210
4211 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4212         .can_merge              = 0,
4213         .map                    = generic_pipe_buf_map,
4214         .unmap                  = generic_pipe_buf_unmap,
4215         .confirm                = generic_pipe_buf_confirm,
4216         .release                = tracing_pipe_buf_release,
4217         .steal                  = generic_pipe_buf_steal,
4218         .get                    = generic_pipe_buf_get,
4219 };
4220
4221 static size_t
4222 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4223 {
4224         size_t count;
4225         int ret;
4226
4227         /* Seq buffer is page-sized, exactly what we need. */
4228         for (;;) {
4229                 count = iter->seq.len;
4230                 ret = print_trace_line(iter);
4231                 count = iter->seq.len - count;
4232                 if (rem < count) {
4233                         rem = 0;
4234                         iter->seq.len -= count;
4235                         break;
4236                 }
4237                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4238                         iter->seq.len -= count;
4239                         break;
4240                 }
4241
4242                 if (ret != TRACE_TYPE_NO_CONSUME)
4243                         trace_consume(iter);
4244                 rem -= count;
4245                 if (!trace_find_next_entry_inc(iter))   {
4246                         rem = 0;
4247                         iter->ent = NULL;
4248                         break;
4249                 }
4250         }
4251
4252         return rem;
4253 }
4254
4255 static ssize_t tracing_splice_read_pipe(struct file *filp,
4256                                         loff_t *ppos,
4257                                         struct pipe_inode_info *pipe,
4258                                         size_t len,
4259                                         unsigned int flags)
4260 {
4261         struct page *pages_def[PIPE_DEF_BUFFERS];
4262         struct partial_page partial_def[PIPE_DEF_BUFFERS];
4263         struct trace_iterator *iter = filp->private_data;
4264         struct splice_pipe_desc spd = {
4265                 .pages          = pages_def,
4266                 .partial        = partial_def,
4267                 .nr_pages       = 0, /* This gets updated below. */
4268                 .nr_pages_max   = PIPE_DEF_BUFFERS,
4269                 .flags          = flags,
4270                 .ops            = &tracing_pipe_buf_ops,
4271                 .spd_release    = tracing_spd_release_pipe,
4272         };
4273         struct trace_array *tr = iter->tr;
4274         ssize_t ret;
4275         size_t rem;
4276         unsigned int i;
4277
4278         if (splice_grow_spd(pipe, &spd))
4279                 return -ENOMEM;
4280
4281         /* copy the tracer to avoid using a global lock all around */
4282         mutex_lock(&trace_types_lock);
4283         if (unlikely(iter->trace->name != tr->current_trace->name))
4284                 *iter->trace = *tr->current_trace;
4285         mutex_unlock(&trace_types_lock);
4286
4287         mutex_lock(&iter->mutex);
4288
4289         if (iter->trace->splice_read) {
4290                 ret = iter->trace->splice_read(iter, filp,
4291                                                ppos, pipe, len, flags);
4292                 if (ret)
4293                         goto out_err;
4294         }
4295
4296         ret = tracing_wait_pipe(filp);
4297         if (ret <= 0)
4298                 goto out_err;
4299
4300         if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4301                 ret = -EFAULT;
4302                 goto out_err;
4303         }
4304
4305         trace_event_read_lock();
4306         trace_access_lock(iter->cpu_file);
4307
4308         /* Fill as many pages as possible. */
4309         for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4310                 spd.pages[i] = alloc_page(GFP_KERNEL);
4311                 if (!spd.pages[i])
4312                         break;
4313
4314                 rem = tracing_fill_pipe_page(rem, iter);
4315
4316                 /* Copy the data into the page, so we can start over. */
4317                 ret = trace_seq_to_buffer(&iter->seq,
4318                                           page_address(spd.pages[i]),
4319                                           iter->seq.len);
4320                 if (ret < 0) {
4321                         __free_page(spd.pages[i]);
4322                         break;
4323                 }
4324                 spd.partial[i].offset = 0;
4325                 spd.partial[i].len = iter->seq.len;
4326
4327                 trace_seq_init(&iter->seq);
4328         }
4329
4330         trace_access_unlock(iter->cpu_file);
4331         trace_event_read_unlock();
4332         mutex_unlock(&iter->mutex);
4333
4334         spd.nr_pages = i;
4335
4336         ret = splice_to_pipe(pipe, &spd);
4337 out:
4338         splice_shrink_spd(&spd);
4339         return ret;
4340
4341 out_err:
4342         mutex_unlock(&iter->mutex);
4343         goto out;
4344 }
4345
4346 static ssize_t
4347 tracing_entries_read(struct file *filp, char __user *ubuf,
4348                      size_t cnt, loff_t *ppos)
4349 {
4350         struct inode *inode = file_inode(filp);
4351         struct trace_array *tr = inode->i_private;
4352         int cpu = tracing_get_cpu(inode);
4353         char buf[64];
4354         int r = 0;
4355         ssize_t ret;
4356
4357         mutex_lock(&trace_types_lock);
4358
4359         if (cpu == RING_BUFFER_ALL_CPUS) {
4360                 int cpu, buf_size_same;
4361                 unsigned long size;
4362
4363                 size = 0;
4364                 buf_size_same = 1;
4365                 /* check if all cpu sizes are same */
4366                 for_each_tracing_cpu(cpu) {
4367                         /* fill in the size from first enabled cpu */
4368                         if (size == 0)
4369                                 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4370                         if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4371                                 buf_size_same = 0;
4372                                 break;
4373                         }
4374                 }
4375
4376                 if (buf_size_same) {
4377                         if (!ring_buffer_expanded)
4378                                 r = sprintf(buf, "%lu (expanded: %lu)\n",
4379                                             size >> 10,
4380                                             trace_buf_size >> 10);
4381                         else
4382                                 r = sprintf(buf, "%lu\n", size >> 10);
4383                 } else
4384                         r = sprintf(buf, "X\n");
4385         } else
4386                 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4387
4388         mutex_unlock(&trace_types_lock);
4389
4390         ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4391         return ret;
4392 }
4393
4394 static ssize_t
4395 tracing_entries_write(struct file *filp, const char __user *ubuf,
4396                       size_t cnt, loff_t *ppos)
4397 {
4398         struct inode *inode = file_inode(filp);
4399         struct trace_array *tr = inode->i_private;
4400         unsigned long val;
4401         int ret;
4402
4403         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4404         if (ret)
4405                 return ret;
4406
4407         /* must have at least 1 entry */
4408         if (!val)
4409                 return -EINVAL;
4410
4411         /* value is in KB */
4412         val <<= 10;
4413         ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4414         if (ret < 0)
4415                 return ret;
4416
4417         *ppos += cnt;
4418
4419         return cnt;
4420 }
4421
4422 static ssize_t
4423 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4424                                 size_t cnt, loff_t *ppos)
4425 {
4426         struct trace_array *tr = filp->private_data;
4427         char buf[64];
4428         int r, cpu;
4429         unsigned long size = 0, expanded_size = 0;
4430
4431         mutex_lock(&trace_types_lock);
4432         for_each_tracing_cpu(cpu) {
4433                 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4434                 if (!ring_buffer_expanded)
4435                         expanded_size += trace_buf_size >> 10;
4436         }
4437         if (ring_buffer_expanded)
4438                 r = sprintf(buf, "%lu\n", size);
4439         else
4440                 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4441         mutex_unlock(&trace_types_lock);
4442
4443         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4444 }
4445
4446 static ssize_t
4447 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4448                           size_t cnt, loff_t *ppos)
4449 {
4450         /*
4451          * There is no need to read what the user has written, this function
4452          * is just to make sure that there is no error when "echo" is used
4453          */
4454
4455         *ppos += cnt;
4456
4457         return cnt;
4458 }
4459
4460 static int
4461 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4462 {
4463         struct trace_array *tr = inode->i_private;
4464
4465         /* disable tracing ? */
4466         if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4467                 tracer_tracing_off(tr);
4468         /* resize the ring buffer to 0 */
4469         tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4470
4471         trace_array_put(tr);
4472
4473         return 0;
4474 }
4475
4476 static ssize_t
4477 tracing_mark_write(struct file *filp, const char __user *ubuf,
4478                                         size_t cnt, loff_t *fpos)
4479 {
4480         unsigned long addr = (unsigned long)ubuf;
4481         struct trace_array *tr = filp->private_data;
4482         struct ring_buffer_event *event;
4483         struct ring_buffer *buffer;
4484         struct print_entry *entry;
4485         unsigned long irq_flags;
4486         struct page *pages[2];
4487         void *map_page[2];
4488         int nr_pages = 1;
4489         ssize_t written;
4490         int offset;
4491         int size;
4492         int len;
4493         int ret;
4494         int i;
4495
4496         if (tracing_disabled)
4497                 return -EINVAL;
4498
4499         if (!(trace_flags & TRACE_ITER_MARKERS))
4500                 return -EINVAL;
4501
4502         if (cnt > TRACE_BUF_SIZE)
4503                 cnt = TRACE_BUF_SIZE;
4504
4505         /*
4506          * Userspace is injecting traces into the kernel trace buffer.
4507          * We want to be as non intrusive as possible.
4508          * To do so, we do not want to allocate any special buffers
4509          * or take any locks, but instead write the userspace data
4510          * straight into the ring buffer.
4511          *
4512          * First we need to pin the userspace buffer into memory,
4513          * which, most likely it is, because it just referenced it.
4514          * But there's no guarantee that it is. By using get_user_pages_fast()
4515          * and kmap_atomic/kunmap_atomic() we can get access to the
4516          * pages directly. We then write the data directly into the
4517          * ring buffer.
4518          */
4519         BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4520
4521         /* check if we cross pages */
4522         if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4523                 nr_pages = 2;
4524
4525         offset = addr & (PAGE_SIZE - 1);
4526         addr &= PAGE_MASK;
4527
4528         ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4529         if (ret < nr_pages) {
4530                 while (--ret >= 0)
4531                         put_page(pages[ret]);
4532                 written = -EFAULT;
4533                 goto out;
4534         }
4535
4536         for (i = 0; i < nr_pages; i++)
4537                 map_page[i] = kmap_atomic(pages[i]);
4538
4539         local_save_flags(irq_flags);
4540         size = sizeof(*entry) + cnt + 2; /* possible \n added */
4541         buffer = tr->trace_buffer.buffer;
4542         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4543                                           irq_flags, preempt_count());
4544         if (!event) {
4545                 /* Ring buffer disabled, return as if not open for write */
4546                 written = -EBADF;
4547                 goto out_unlock;
4548         }
4549
4550         entry = ring_buffer_event_data(event);
4551         entry->ip = _THIS_IP_;
4552
4553         if (nr_pages == 2) {
4554                 len = PAGE_SIZE - offset;
4555                 memcpy(&entry->buf, map_page[0] + offset, len);
4556                 memcpy(&entry->buf[len], map_page[1], cnt - len);
4557         } else
4558                 memcpy(&entry->buf, map_page[0] + offset, cnt);
4559
4560         if (entry->buf[cnt - 1] != '\n') {
4561                 entry->buf[cnt] = '\n';
4562                 entry->buf[cnt + 1] = '\0';
4563         } else
4564                 entry->buf[cnt] = '\0';
4565
4566         __buffer_unlock_commit(buffer, event);
4567
4568         written = cnt;
4569
4570         *fpos += written;
4571
4572  out_unlock:
4573         for (i = 0; i < nr_pages; i++){
4574                 kunmap_atomic(map_page[i]);
4575                 put_page(pages[i]);
4576         }
4577  out:
4578         return written;
4579 }
4580
4581 static int tracing_clock_show(struct seq_file *m, void *v)
4582 {
4583         struct trace_array *tr = m->private;
4584         int i;
4585
4586         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4587                 seq_printf(m,
4588                         "%s%s%s%s", i ? " " : "",
4589                         i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4590                         i == tr->clock_id ? "]" : "");
4591         seq_putc(m, '\n');
4592
4593         return 0;
4594 }
4595
4596 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4597                                    size_t cnt, loff_t *fpos)
4598 {
4599         struct seq_file *m = filp->private_data;
4600         struct trace_array *tr = m->private;
4601         char buf[64];
4602         const char *clockstr;
4603         int i;
4604
4605         if (cnt >= sizeof(buf))
4606                 return -EINVAL;
4607
4608         if (copy_from_user(&buf, ubuf, cnt))
4609                 return -EFAULT;
4610
4611         buf[cnt] = 0;
4612
4613         clockstr = strstrip(buf);
4614
4615         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4616                 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4617                         break;
4618         }
4619         if (i == ARRAY_SIZE(trace_clocks))
4620                 return -EINVAL;
4621
4622         mutex_lock(&trace_types_lock);
4623
4624         tr->clock_id = i;
4625
4626         ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4627
4628         /*
4629          * New clock may not be consistent with the previous clock.
4630          * Reset the buffer so that it doesn't have incomparable timestamps.
4631          */
4632         tracing_reset_online_cpus(&tr->trace_buffer);
4633
4634 #ifdef CONFIG_TRACER_MAX_TRACE
4635         if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4636                 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4637         tracing_reset_online_cpus(&tr->max_buffer);
4638 #endif
4639
4640         mutex_unlock(&trace_types_lock);
4641
4642         *fpos += cnt;
4643
4644         return cnt;
4645 }
4646
4647 static int tracing_clock_open(struct inode *inode, struct file *file)
4648 {
4649         struct trace_array *tr = inode->i_private;
4650         int ret;
4651
4652         if (tracing_disabled)
4653                 return -ENODEV;
4654
4655         if (trace_array_get(tr))
4656                 return -ENODEV;
4657
4658         ret = single_open(file, tracing_clock_show, inode->i_private);
4659         if (ret < 0)
4660                 trace_array_put(tr);
4661
4662         return ret;
4663 }
4664
4665 struct ftrace_buffer_info {
4666         struct trace_iterator   iter;
4667         void                    *spare;
4668         unsigned int            read;
4669 };
4670
4671 #ifdef CONFIG_TRACER_SNAPSHOT
4672 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4673 {
4674         struct trace_array *tr = inode->i_private;
4675         struct trace_iterator *iter;
4676         struct seq_file *m;
4677         int ret = 0;
4678
4679         if (trace_array_get(tr) < 0)
4680                 return -ENODEV;
4681
4682         if (file->f_mode & FMODE_READ) {
4683                 iter = __tracing_open(inode, file, true);
4684                 if (IS_ERR(iter))
4685                         ret = PTR_ERR(iter);
4686         } else {
4687                 /* Writes still need the seq_file to hold the private data */
4688                 ret = -ENOMEM;
4689                 m = kzalloc(sizeof(*m), GFP_KERNEL);
4690                 if (!m)
4691                         goto out;
4692                 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4693                 if (!iter) {
4694                         kfree(m);
4695                         goto out;
4696                 }
4697                 ret = 0;
4698
4699                 iter->tr = tr;
4700                 iter->trace_buffer = &tr->max_buffer;
4701                 iter->cpu_file = tracing_get_cpu(inode);
4702                 m->private = iter;
4703                 file->private_data = m;
4704         }
4705 out:
4706         if (ret < 0)
4707                 trace_array_put(tr);
4708
4709         return ret;
4710 }
4711
4712 static ssize_t
4713 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4714                        loff_t *ppos)
4715 {
4716         struct seq_file *m = filp->private_data;
4717         struct trace_iterator *iter = m->private;
4718         struct trace_array *tr = iter->tr;
4719         unsigned long val;
4720         int ret;
4721
4722         ret = tracing_update_buffers();
4723         if (ret < 0)
4724                 return ret;
4725
4726         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4727         if (ret)
4728                 return ret;
4729
4730         mutex_lock(&trace_types_lock);
4731
4732         if (tr->current_trace->use_max_tr) {
4733                 ret = -EBUSY;
4734                 goto out;
4735         }
4736
4737         switch (val) {
4738         case 0:
4739                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4740                         ret = -EINVAL;
4741                         break;
4742                 }
4743                 if (tr->allocated_snapshot)
4744                         free_snapshot(tr);
4745                 break;
4746         case 1:
4747 /* Only allow per-cpu swap if the ring buffer supports it */
4748 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4749                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4750                         ret = -EINVAL;
4751                         break;
4752                 }
4753 #endif
4754                 if (!tr->allocated_snapshot) {
4755                         ret = alloc_snapshot(tr);
4756                         if (ret < 0)
4757                                 break;
4758                 }
4759                 local_irq_disable();
4760                 /* Now, we're going to swap */
4761                 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4762                         update_max_tr(tr, current, smp_processor_id());
4763                 else
4764                         update_max_tr_single(tr, current, iter->cpu_file);
4765                 local_irq_enable();
4766                 break;
4767         default:
4768                 if (tr->allocated_snapshot) {
4769                         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4770                                 tracing_reset_online_cpus(&tr->max_buffer);
4771                         else
4772                                 tracing_reset(&tr->max_buffer, iter->cpu_file);
4773                 }
4774                 break;
4775         }
4776
4777         if (ret >= 0) {
4778                 *ppos += cnt;
4779                 ret = cnt;
4780         }
4781 out:
4782         mutex_unlock(&trace_types_lock);
4783         return ret;
4784 }
4785
4786 static int tracing_snapshot_release(struct inode *inode, struct file *file)
4787 {
4788         struct seq_file *m = file->private_data;
4789         int ret;
4790
4791         ret = tracing_release(inode, file);
4792
4793         if (file->f_mode & FMODE_READ)
4794                 return ret;
4795
4796         /* If write only, the seq_file is just a stub */
4797         if (m)
4798                 kfree(m->private);
4799         kfree(m);
4800
4801         return 0;
4802 }
4803
4804 static int tracing_buffers_open(struct inode *inode, struct file *filp);
4805 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4806                                     size_t count, loff_t *ppos);
4807 static int tracing_buffers_release(struct inode *inode, struct file *file);
4808 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4809                    struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4810
4811 static int snapshot_raw_open(struct inode *inode, struct file *filp)
4812 {
4813         struct ftrace_buffer_info *info;
4814         int ret;
4815
4816         ret = tracing_buffers_open(inode, filp);
4817         if (ret < 0)
4818                 return ret;
4819
4820         info = filp->private_data;
4821
4822         if (info->iter.trace->use_max_tr) {
4823                 tracing_buffers_release(inode, filp);
4824                 return -EBUSY;
4825         }
4826
4827         info->iter.snapshot = true;
4828         info->iter.trace_buffer = &info->iter.tr->max_buffer;
4829
4830         return ret;
4831 }
4832
4833 #endif /* CONFIG_TRACER_SNAPSHOT */
4834
4835
4836 static const struct file_operations tracing_max_lat_fops = {
4837         .open           = tracing_open_generic,
4838         .read           = tracing_max_lat_read,
4839         .write          = tracing_max_lat_write,
4840         .llseek         = generic_file_llseek,
4841 };
4842
4843 static const struct file_operations set_tracer_fops = {
4844         .open           = tracing_open_generic,
4845         .read           = tracing_set_trace_read,
4846         .write          = tracing_set_trace_write,
4847         .llseek         = generic_file_llseek,
4848 };
4849
4850 static const struct file_operations tracing_pipe_fops = {
4851         .open           = tracing_open_pipe,
4852         .poll           = tracing_poll_pipe,
4853         .read           = tracing_read_pipe,
4854         .splice_read    = tracing_splice_read_pipe,
4855         .release        = tracing_release_pipe,
4856         .llseek         = no_llseek,
4857 };
4858
4859 static const struct file_operations tracing_entries_fops = {
4860         .open           = tracing_open_generic_tr,
4861         .read           = tracing_entries_read,
4862         .write          = tracing_entries_write,
4863         .llseek         = generic_file_llseek,
4864         .release        = tracing_release_generic_tr,
4865 };
4866
4867 static const struct file_operations tracing_total_entries_fops = {
4868         .open           = tracing_open_generic_tr,
4869         .read           = tracing_total_entries_read,
4870         .llseek         = generic_file_llseek,
4871         .release        = tracing_release_generic_tr,
4872 };
4873
4874 static const struct file_operations tracing_free_buffer_fops = {
4875         .open           = tracing_open_generic_tr,
4876         .write          = tracing_free_buffer_write,
4877         .release        = tracing_free_buffer_release,
4878 };
4879
4880 static const struct file_operations tracing_mark_fops = {
4881         .open           = tracing_open_generic_tr,
4882         .write          = tracing_mark_write,
4883         .llseek         = generic_file_llseek,
4884         .release        = tracing_release_generic_tr,
4885 };
4886
4887 static const struct file_operations trace_clock_fops = {
4888         .open           = tracing_clock_open,
4889         .read           = seq_read,
4890         .llseek         = seq_lseek,
4891         .release        = tracing_single_release_tr,
4892         .write          = tracing_clock_write,
4893 };
4894
4895 #ifdef CONFIG_TRACER_SNAPSHOT
4896 static const struct file_operations snapshot_fops = {
4897         .open           = tracing_snapshot_open,
4898         .read           = seq_read,
4899         .write          = tracing_snapshot_write,
4900         .llseek         = tracing_seek,
4901         .release        = tracing_snapshot_release,
4902 };
4903
4904 static const struct file_operations snapshot_raw_fops = {
4905         .open           = snapshot_raw_open,
4906         .read           = tracing_buffers_read,
4907         .release        = tracing_buffers_release,
4908         .splice_read    = tracing_buffers_splice_read,
4909         .llseek         = no_llseek,
4910 };
4911
4912 #endif /* CONFIG_TRACER_SNAPSHOT */
4913
4914 static int tracing_buffers_open(struct inode *inode, struct file *filp)
4915 {
4916         struct trace_array *tr = inode->i_private;
4917         struct ftrace_buffer_info *info;
4918         int ret;
4919
4920         if (tracing_disabled)
4921                 return -ENODEV;
4922
4923         if (trace_array_get(tr) < 0)
4924                 return -ENODEV;
4925
4926         info = kzalloc(sizeof(*info), GFP_KERNEL);
4927         if (!info) {
4928                 trace_array_put(tr);
4929                 return -ENOMEM;
4930         }
4931
4932         mutex_lock(&trace_types_lock);
4933
4934         info->iter.tr           = tr;
4935         info->iter.cpu_file     = tracing_get_cpu(inode);
4936         info->iter.trace        = tr->current_trace;
4937         info->iter.trace_buffer = &tr->trace_buffer;
4938         info->spare             = NULL;
4939         /* Force reading ring buffer for first read */
4940         info->read              = (unsigned int)-1;
4941
4942         filp->private_data = info;
4943
4944         mutex_unlock(&trace_types_lock);
4945
4946         ret = nonseekable_open(inode, filp);
4947         if (ret < 0)
4948                 trace_array_put(tr);
4949
4950         return ret;
4951 }
4952
4953 static unsigned int
4954 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4955 {
4956         struct ftrace_buffer_info *info = filp->private_data;
4957         struct trace_iterator *iter = &info->iter;
4958
4959         return trace_poll(iter, filp, poll_table);
4960 }
4961
4962 static ssize_t
4963 tracing_buffers_read(struct file *filp, char __user *ubuf,
4964                      size_t count, loff_t *ppos)
4965 {
4966         struct ftrace_buffer_info *info = filp->private_data;
4967         struct trace_iterator *iter = &info->iter;
4968         ssize_t ret;
4969         ssize_t size;
4970
4971         if (!count)
4972                 return 0;
4973
4974         mutex_lock(&trace_types_lock);
4975
4976 #ifdef CONFIG_TRACER_MAX_TRACE
4977         if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4978                 size = -EBUSY;
4979                 goto out_unlock;
4980         }
4981 #endif
4982
4983         if (!info->spare)
4984                 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4985                                                           iter->cpu_file);
4986         size = -ENOMEM;
4987         if (!info->spare)
4988                 goto out_unlock;
4989
4990         /* Do we have previous read data to read? */
4991         if (info->read < PAGE_SIZE)
4992                 goto read;
4993
4994  again:
4995         trace_access_lock(iter->cpu_file);
4996         ret = ring_buffer_read_page(iter->trace_buffer->buffer,
4997                                     &info->spare,
4998                                     count,
4999                                     iter->cpu_file, 0);
5000         trace_access_unlock(iter->cpu_file);
5001
5002         if (ret < 0) {
5003                 if (trace_empty(iter)) {
5004                         if ((filp->f_flags & O_NONBLOCK)) {
5005                                 size = -EAGAIN;
5006                                 goto out_unlock;
5007                         }
5008                         mutex_unlock(&trace_types_lock);
5009                         iter->trace->wait_pipe(iter);
5010                         mutex_lock(&trace_types_lock);
5011                         if (signal_pending(current)) {
5012                                 size = -EINTR;
5013                                 goto out_unlock;
5014                         }
5015                         goto again;
5016                 }
5017                 size = 0;
5018                 goto out_unlock;
5019         }
5020
5021         info->read = 0;
5022  read:
5023         size = PAGE_SIZE - info->read;
5024         if (size > count)
5025                 size = count;
5026
5027         ret = copy_to_user(ubuf, info->spare + info->read, size);
5028         if (ret == size) {
5029                 size = -EFAULT;
5030                 goto out_unlock;
5031         }
5032         size -= ret;
5033
5034         *ppos += size;
5035         info->read += size;
5036
5037  out_unlock:
5038         mutex_unlock(&trace_types_lock);
5039
5040         return size;
5041 }
5042
5043 static int tracing_buffers_release(struct inode *inode, struct file *file)
5044 {
5045         struct ftrace_buffer_info *info = file->private_data;
5046         struct trace_iterator *iter = &info->iter;
5047
5048         mutex_lock(&trace_types_lock);
5049
5050         __trace_array_put(iter->tr);
5051
5052         if (info->spare)
5053                 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5054         kfree(info);
5055
5056         mutex_unlock(&trace_types_lock);
5057
5058         return 0;
5059 }
5060
5061 struct buffer_ref {
5062         struct ring_buffer      *buffer;
5063         void                    *page;
5064         int                     ref;
5065 };
5066
5067 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5068                                     struct pipe_buffer *buf)
5069 {
5070         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5071
5072         if (--ref->ref)
5073                 return;
5074
5075         ring_buffer_free_read_page(ref->buffer, ref->page);
5076         kfree(ref);
5077         buf->private = 0;
5078 }
5079
5080 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5081                                 struct pipe_buffer *buf)
5082 {
5083         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5084
5085         ref->ref++;
5086 }
5087
5088 /* Pipe buffer operations for a buffer. */
5089 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5090         .can_merge              = 0,
5091         .map                    = generic_pipe_buf_map,
5092         .unmap                  = generic_pipe_buf_unmap,
5093         .confirm                = generic_pipe_buf_confirm,
5094         .release                = buffer_pipe_buf_release,
5095         .steal                  = generic_pipe_buf_steal,
5096         .get                    = buffer_pipe_buf_get,
5097 };
5098
5099 /*
5100  * Callback from splice_to_pipe(), if we need to release some pages
5101  * at the end of the spd in case we error'ed out in filling the pipe.
5102  */
5103 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5104 {
5105         struct buffer_ref *ref =
5106                 (struct buffer_ref *)spd->partial[i].private;
5107
5108         if (--ref->ref)
5109                 return;
5110
5111         ring_buffer_free_read_page(ref->buffer, ref->page);
5112         kfree(ref);
5113         spd->partial[i].private = 0;
5114 }
5115
5116 static ssize_t
5117 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5118                             struct pipe_inode_info *pipe, size_t len,
5119                             unsigned int flags)
5120 {
5121         struct ftrace_buffer_info *info = file->private_data;
5122         struct trace_iterator *iter = &info->iter;
5123         struct partial_page partial_def[PIPE_DEF_BUFFERS];
5124         struct page *pages_def[PIPE_DEF_BUFFERS];
5125         struct splice_pipe_desc spd = {
5126                 .pages          = pages_def,
5127                 .partial        = partial_def,
5128                 .nr_pages_max   = PIPE_DEF_BUFFERS,
5129                 .flags          = flags,
5130                 .ops            = &buffer_pipe_buf_ops,
5131                 .spd_release    = buffer_spd_release,
5132         };
5133         struct buffer_ref *ref;
5134         int entries, size, i;
5135         ssize_t ret;
5136
5137         mutex_lock(&trace_types_lock);
5138
5139 #ifdef CONFIG_TRACER_MAX_TRACE
5140         if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5141                 ret = -EBUSY;
5142                 goto out;
5143         }
5144 #endif
5145
5146         if (splice_grow_spd(pipe, &spd)) {
5147                 ret = -ENOMEM;
5148                 goto out;
5149         }
5150
5151         if (*ppos & (PAGE_SIZE - 1)) {
5152                 ret = -EINVAL;
5153                 goto out;
5154         }
5155
5156         if (len & (PAGE_SIZE - 1)) {
5157                 if (len < PAGE_SIZE) {
5158                         ret = -EINVAL;
5159                         goto out;
5160                 }
5161                 len &= PAGE_MASK;
5162         }
5163
5164  again:
5165         trace_access_lock(iter->cpu_file);
5166         entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5167
5168         for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
5169                 struct page *page;
5170                 int r;
5171
5172                 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5173                 if (!ref)
5174                         break;
5175
5176                 ref->ref = 1;
5177                 ref->buffer = iter->trace_buffer->buffer;
5178                 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5179                 if (!ref->page) {
5180                         kfree(ref);
5181                         break;
5182                 }
5183
5184                 r = ring_buffer_read_page(ref->buffer, &ref->page,
5185                                           len, iter->cpu_file, 1);
5186                 if (r < 0) {
5187                         ring_buffer_free_read_page(ref->buffer, ref->page);
5188                         kfree(ref);
5189                         break;
5190                 }
5191
5192                 /*
5193                  * zero out any left over data, this is going to
5194                  * user land.
5195                  */
5196                 size = ring_buffer_page_len(ref->page);
5197                 if (size < PAGE_SIZE)
5198                         memset(ref->page + size, 0, PAGE_SIZE - size);
5199
5200                 page = virt_to_page(ref->page);
5201
5202                 spd.pages[i] = page;
5203                 spd.partial[i].len = PAGE_SIZE;
5204                 spd.partial[i].offset = 0;
5205                 spd.partial[i].private = (unsigned long)ref;
5206                 spd.nr_pages++;
5207                 *ppos += PAGE_SIZE;
5208
5209                 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5210         }
5211
5212         trace_access_unlock(iter->cpu_file);
5213         spd.nr_pages = i;
5214
5215         /* did we read anything? */
5216         if (!spd.nr_pages) {
5217                 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5218                         ret = -EAGAIN;
5219                         goto out;
5220                 }
5221                 mutex_unlock(&trace_types_lock);
5222                 iter->trace->wait_pipe(iter);
5223                 mutex_lock(&trace_types_lock);
5224                 if (signal_pending(current)) {
5225                         ret = -EINTR;
5226                         goto out;
5227                 }
5228                 goto again;
5229         }
5230
5231         ret = splice_to_pipe(pipe, &spd);
5232         splice_shrink_spd(&spd);
5233 out:
5234         mutex_unlock(&trace_types_lock);
5235
5236         return ret;
5237 }
5238
5239 static const struct file_operations tracing_buffers_fops = {
5240         .open           = tracing_buffers_open,
5241         .read           = tracing_buffers_read,
5242         .poll           = tracing_buffers_poll,
5243         .release        = tracing_buffers_release,
5244         .splice_read    = tracing_buffers_splice_read,
5245         .llseek         = no_llseek,
5246 };
5247
5248 static ssize_t
5249 tracing_stats_read(struct file *filp, char __user *ubuf,
5250                    size_t count, loff_t *ppos)
5251 {
5252         struct inode *inode = file_inode(filp);
5253         struct trace_array *tr = inode->i_private;
5254         struct trace_buffer *trace_buf = &tr->trace_buffer;
5255         int cpu = tracing_get_cpu(inode);
5256         struct trace_seq *s;
5257         unsigned long cnt;
5258         unsigned long long t;
5259         unsigned long usec_rem;
5260
5261         s = kmalloc(sizeof(*s), GFP_KERNEL);
5262         if (!s)
5263                 return -ENOMEM;
5264
5265         trace_seq_init(s);
5266
5267         cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5268         trace_seq_printf(s, "entries: %ld\n", cnt);
5269
5270         cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5271         trace_seq_printf(s, "overrun: %ld\n", cnt);
5272
5273         cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5274         trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5275
5276         cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5277         trace_seq_printf(s, "bytes: %ld\n", cnt);
5278
5279         if (trace_clocks[tr->clock_id].in_ns) {
5280                 /* local or global for trace_clock */
5281                 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5282                 usec_rem = do_div(t, USEC_PER_SEC);
5283                 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5284                                                                 t, usec_rem);
5285
5286                 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5287                 usec_rem = do_div(t, USEC_PER_SEC);
5288                 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5289         } else {
5290                 /* counter or tsc mode for trace_clock */
5291                 trace_seq_printf(s, "oldest event ts: %llu\n",
5292                                 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5293
5294                 trace_seq_printf(s, "now ts: %llu\n",
5295                                 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5296         }
5297
5298         cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5299         trace_seq_printf(s, "dropped events: %ld\n", cnt);
5300
5301         cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5302         trace_seq_printf(s, "read events: %ld\n", cnt);
5303
5304         count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5305
5306         kfree(s);
5307
5308         return count;
5309 }
5310
5311 static const struct file_operations tracing_stats_fops = {
5312         .open           = tracing_open_generic_tr,
5313         .read           = tracing_stats_read,
5314         .llseek         = generic_file_llseek,
5315         .release        = tracing_release_generic_tr,
5316 };
5317
5318 #ifdef CONFIG_DYNAMIC_FTRACE
5319
5320 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5321 {
5322         return 0;
5323 }
5324
5325 static ssize_t
5326 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5327                   size_t cnt, loff_t *ppos)
5328 {
5329         static char ftrace_dyn_info_buffer[1024];
5330         static DEFINE_MUTEX(dyn_info_mutex);
5331         unsigned long *p = filp->private_data;
5332         char *buf = ftrace_dyn_info_buffer;
5333         int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5334         int r;
5335
5336         mutex_lock(&dyn_info_mutex);
5337         r = sprintf(buf, "%ld ", *p);
5338
5339         r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5340         buf[r++] = '\n';
5341
5342         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5343
5344         mutex_unlock(&dyn_info_mutex);
5345
5346         return r;
5347 }
5348
5349 static const struct file_operations tracing_dyn_info_fops = {
5350         .open           = tracing_open_generic,
5351         .read           = tracing_read_dyn_info,
5352         .llseek         = generic_file_llseek,
5353 };
5354 #endif /* CONFIG_DYNAMIC_FTRACE */
5355
5356 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5357 static void
5358 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5359 {
5360         tracing_snapshot();
5361 }
5362
5363 static void
5364 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5365 {
5366         unsigned long *count = (long *)data;
5367
5368         if (!*count)
5369                 return;
5370
5371         if (*count != -1)
5372                 (*count)--;
5373
5374         tracing_snapshot();
5375 }
5376
5377 static int
5378 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5379                       struct ftrace_probe_ops *ops, void *data)
5380 {
5381         long count = (long)data;
5382
5383         seq_printf(m, "%ps:", (void *)ip);
5384
5385         seq_printf(m, "snapshot");
5386
5387         if (count == -1)
5388                 seq_printf(m, ":unlimited\n");
5389         else
5390                 seq_printf(m, ":count=%ld\n", count);
5391
5392         return 0;
5393 }
5394
5395 static struct ftrace_probe_ops snapshot_probe_ops = {
5396         .func                   = ftrace_snapshot,
5397         .print                  = ftrace_snapshot_print,
5398 };
5399
5400 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5401         .func                   = ftrace_count_snapshot,
5402         .print                  = ftrace_snapshot_print,
5403 };
5404
5405 static int
5406 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5407                                char *glob, char *cmd, char *param, int enable)
5408 {
5409         struct ftrace_probe_ops *ops;
5410         void *count = (void *)-1;
5411         char *number;
5412         int ret;
5413
5414         /* hash funcs only work with set_ftrace_filter */
5415         if (!enable)
5416                 return -EINVAL;
5417
5418         ops = param ? &snapshot_count_probe_ops :  &snapshot_probe_ops;
5419
5420         if (glob[0] == '!') {
5421                 unregister_ftrace_function_probe_func(glob+1, ops);
5422                 return 0;
5423         }
5424
5425         if (!param)
5426                 goto out_reg;
5427
5428         number = strsep(&param, ":");
5429
5430         if (!strlen(number))
5431                 goto out_reg;
5432
5433         /*
5434          * We use the callback data field (which is a pointer)
5435          * as our counter.
5436          */
5437         ret = kstrtoul(number, 0, (unsigned long *)&count);
5438         if (ret)
5439                 return ret;
5440
5441  out_reg:
5442         ret = register_ftrace_function_probe(glob, ops, count);
5443
5444         if (ret >= 0)
5445                 alloc_snapshot(&global_trace);
5446
5447         return ret < 0 ? ret : 0;
5448 }
5449
5450 static struct ftrace_func_command ftrace_snapshot_cmd = {
5451         .name                   = "snapshot",
5452         .func                   = ftrace_trace_snapshot_callback,
5453 };
5454
5455 static int register_snapshot_cmd(void)
5456 {
5457         return register_ftrace_command(&ftrace_snapshot_cmd);
5458 }
5459 #else
5460 static inline int register_snapshot_cmd(void) { return 0; }
5461 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5462
5463 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5464 {
5465         if (tr->dir)
5466                 return tr->dir;
5467
5468         if (!debugfs_initialized())
5469                 return NULL;
5470
5471         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5472                 tr->dir = debugfs_create_dir("tracing", NULL);
5473
5474         if (!tr->dir)
5475                 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5476
5477         return tr->dir;
5478 }
5479
5480 struct dentry *tracing_init_dentry(void)
5481 {
5482         return tracing_init_dentry_tr(&global_trace);
5483 }
5484
5485 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5486 {
5487         struct dentry *d_tracer;
5488
5489         if (tr->percpu_dir)
5490                 return tr->percpu_dir;
5491
5492         d_tracer = tracing_init_dentry_tr(tr);
5493         if (!d_tracer)
5494                 return NULL;
5495
5496         tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5497
5498         WARN_ONCE(!tr->percpu_dir,
5499                   "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5500
5501         return tr->percpu_dir;
5502 }
5503
5504 static struct dentry *
5505 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5506                       void *data, long cpu, const struct file_operations *fops)
5507 {
5508         struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5509
5510         if (ret) /* See tracing_get_cpu() */
5511                 ret->d_inode->i_cdev = (void *)(cpu + 1);
5512         return ret;
5513 }
5514
5515 static void
5516 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5517 {
5518         struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5519         struct dentry *d_cpu;
5520         char cpu_dir[30]; /* 30 characters should be more than enough */
5521
5522         if (!d_percpu)
5523                 return;
5524
5525         snprintf(cpu_dir, 30, "cpu%ld", cpu);
5526         d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5527         if (!d_cpu) {
5528                 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5529                 return;
5530         }
5531
5532         /* per cpu trace_pipe */
5533         trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5534                                 tr, cpu, &tracing_pipe_fops);
5535
5536         /* per cpu trace */
5537         trace_create_cpu_file("trace", 0644, d_cpu,
5538                                 tr, cpu, &tracing_fops);
5539
5540         trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5541                                 tr, cpu, &tracing_buffers_fops);
5542
5543         trace_create_cpu_file("stats", 0444, d_cpu,
5544                                 tr, cpu, &tracing_stats_fops);
5545
5546         trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5547                                 tr, cpu, &tracing_entries_fops);
5548
5549 #ifdef CONFIG_TRACER_SNAPSHOT
5550         trace_create_cpu_file("snapshot", 0644, d_cpu,
5551                                 tr, cpu, &snapshot_fops);
5552
5553         trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5554                                 tr, cpu, &snapshot_raw_fops);
5555 #endif
5556 }
5557
5558 #ifdef CONFIG_FTRACE_SELFTEST
5559 /* Let selftest have access to static functions in this file */
5560 #include "trace_selftest.c"
5561 #endif
5562
5563 struct trace_option_dentry {
5564         struct tracer_opt               *opt;
5565         struct tracer_flags             *flags;
5566         struct trace_array              *tr;
5567         struct dentry                   *entry;
5568 };
5569
5570 static ssize_t
5571 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5572                         loff_t *ppos)
5573 {
5574         struct trace_option_dentry *topt = filp->private_data;
5575         char *buf;
5576
5577         if (topt->flags->val & topt->opt->bit)
5578                 buf = "1\n";
5579         else
5580                 buf = "0\n";
5581
5582         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5583 }
5584
5585 static ssize_t
5586 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5587                          loff_t *ppos)
5588 {
5589         struct trace_option_dentry *topt = filp->private_data;
5590         unsigned long val;
5591         int ret;
5592
5593         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5594         if (ret)
5595                 return ret;
5596
5597         if (val != 0 && val != 1)
5598                 return -EINVAL;
5599
5600         if (!!(topt->flags->val & topt->opt->bit) != val) {
5601                 mutex_lock(&trace_types_lock);
5602                 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
5603                                           topt->opt, !val);
5604                 mutex_unlock(&trace_types_lock);
5605                 if (ret)
5606                         return ret;
5607         }
5608
5609         *ppos += cnt;
5610
5611         return cnt;
5612 }
5613
5614
5615 static const struct file_operations trace_options_fops = {
5616         .open = tracing_open_generic,
5617         .read = trace_options_read,
5618         .write = trace_options_write,
5619         .llseek = generic_file_llseek,
5620 };
5621
5622 static ssize_t
5623 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5624                         loff_t *ppos)
5625 {
5626         long index = (long)filp->private_data;
5627         char *buf;
5628
5629         if (trace_flags & (1 << index))
5630                 buf = "1\n";
5631         else
5632                 buf = "0\n";
5633
5634         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5635 }
5636
5637 static ssize_t
5638 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5639                          loff_t *ppos)
5640 {
5641         struct trace_array *tr = &global_trace;
5642         long index = (long)filp->private_data;
5643         unsigned long val;
5644         int ret;
5645
5646         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5647         if (ret)
5648                 return ret;
5649
5650         if (val != 0 && val != 1)
5651                 return -EINVAL;
5652
5653         mutex_lock(&trace_types_lock);
5654         ret = set_tracer_flag(tr, 1 << index, val);
5655         mutex_unlock(&trace_types_lock);
5656
5657         if (ret < 0)
5658                 return ret;
5659
5660         *ppos += cnt;
5661
5662         return cnt;
5663 }
5664
5665 static const struct file_operations trace_options_core_fops = {
5666         .open = tracing_open_generic,
5667         .read = trace_options_core_read,
5668         .write = trace_options_core_write,
5669         .llseek = generic_file_llseek,
5670 };
5671
5672 struct dentry *trace_create_file(const char *name,
5673                                  umode_t mode,
5674                                  struct dentry *parent,
5675                                  void *data,
5676                                  const struct file_operations *fops)
5677 {
5678         struct dentry *ret;
5679
5680         ret = debugfs_create_file(name, mode, parent, data, fops);
5681         if (!ret)
5682                 pr_warning("Could not create debugfs '%s' entry\n", name);
5683
5684         return ret;
5685 }
5686
5687
5688 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
5689 {
5690         struct dentry *d_tracer;
5691
5692         if (tr->options)
5693                 return tr->options;
5694
5695         d_tracer = tracing_init_dentry_tr(tr);
5696         if (!d_tracer)
5697                 return NULL;
5698
5699         tr->options = debugfs_create_dir("options", d_tracer);
5700         if (!tr->options) {
5701                 pr_warning("Could not create debugfs directory 'options'\n");
5702                 return NULL;
5703         }
5704
5705         return tr->options;
5706 }
5707
5708 static void
5709 create_trace_option_file(struct trace_array *tr,
5710                          struct trace_option_dentry *topt,
5711                          struct tracer_flags *flags,
5712                          struct tracer_opt *opt)
5713 {
5714         struct dentry *t_options;
5715
5716         t_options = trace_options_init_dentry(tr);
5717         if (!t_options)
5718                 return;
5719
5720         topt->flags = flags;
5721         topt->opt = opt;
5722         topt->tr = tr;
5723
5724         topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
5725                                     &trace_options_fops);
5726
5727 }
5728
5729 static struct trace_option_dentry *
5730 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
5731 {
5732         struct trace_option_dentry *topts;
5733         struct tracer_flags *flags;
5734         struct tracer_opt *opts;
5735         int cnt;
5736
5737         if (!tracer)
5738                 return NULL;
5739
5740         flags = tracer->flags;
5741
5742         if (!flags || !flags->opts)
5743                 return NULL;
5744
5745         opts = flags->opts;
5746
5747         for (cnt = 0; opts[cnt].name; cnt++)
5748                 ;
5749
5750         topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
5751         if (!topts)
5752                 return NULL;
5753
5754         for (cnt = 0; opts[cnt].name; cnt++)
5755                 create_trace_option_file(tr, &topts[cnt], flags,
5756                                          &opts[cnt]);
5757
5758         return topts;
5759 }
5760
5761 static void
5762 destroy_trace_option_files(struct trace_option_dentry *topts)
5763 {
5764         int cnt;
5765
5766         if (!topts)
5767                 return;
5768
5769         for (cnt = 0; topts[cnt].opt; cnt++) {
5770                 if (topts[cnt].entry)
5771                         debugfs_remove(topts[cnt].entry);
5772         }
5773
5774         kfree(topts);
5775 }
5776
5777 static struct dentry *
5778 create_trace_option_core_file(struct trace_array *tr,
5779                               const char *option, long index)
5780 {
5781         struct dentry *t_options;
5782
5783         t_options = trace_options_init_dentry(tr);
5784         if (!t_options)
5785                 return NULL;
5786
5787         return trace_create_file(option, 0644, t_options, (void *)index,
5788                                     &trace_options_core_fops);
5789 }
5790
5791 static __init void create_trace_options_dir(struct trace_array *tr)
5792 {
5793         struct dentry *t_options;
5794         int i;
5795
5796         t_options = trace_options_init_dentry(tr);
5797         if (!t_options)
5798                 return;
5799
5800         for (i = 0; trace_options[i]; i++)
5801                 create_trace_option_core_file(tr, trace_options[i], i);
5802 }
5803
5804 static ssize_t
5805 rb_simple_read(struct file *filp, char __user *ubuf,
5806                size_t cnt, loff_t *ppos)
5807 {
5808         struct trace_array *tr = filp->private_data;
5809         char buf[64];
5810         int r;
5811
5812         r = tracer_tracing_is_on(tr);
5813         r = sprintf(buf, "%d\n", r);
5814
5815         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5816 }
5817
5818 static ssize_t
5819 rb_simple_write(struct file *filp, const char __user *ubuf,
5820                 size_t cnt, loff_t *ppos)
5821 {
5822         struct trace_array *tr = filp->private_data;
5823         struct ring_buffer *buffer = tr->trace_buffer.buffer;
5824         unsigned long val;
5825         int ret;
5826
5827         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5828         if (ret)
5829                 return ret;
5830
5831         if (buffer) {
5832                 mutex_lock(&trace_types_lock);
5833                 if (val) {
5834                         tracer_tracing_on(tr);
5835                         if (tr->current_trace->start)
5836                                 tr->current_trace->start(tr);
5837                 } else {
5838                         tracer_tracing_off(tr);
5839                         if (tr->current_trace->stop)
5840                                 tr->current_trace->stop(tr);
5841                 }
5842                 mutex_unlock(&trace_types_lock);
5843         }
5844
5845         (*ppos)++;
5846
5847         return cnt;
5848 }
5849
5850 static const struct file_operations rb_simple_fops = {
5851         .open           = tracing_open_generic_tr,
5852         .read           = rb_simple_read,
5853         .write          = rb_simple_write,
5854         .release        = tracing_release_generic_tr,
5855         .llseek         = default_llseek,
5856 };
5857
5858 struct dentry *trace_instance_dir;
5859
5860 static void
5861 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5862
5863 static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
5864 {
5865         int cpu;
5866
5867         for_each_tracing_cpu(cpu) {
5868                 memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
5869                 per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
5870                 per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
5871         }
5872 }
5873
5874 static int
5875 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
5876 {
5877         enum ring_buffer_flags rb_flags;
5878
5879         rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5880
5881         buf->buffer = ring_buffer_alloc(size, rb_flags);
5882         if (!buf->buffer)
5883                 return -ENOMEM;
5884
5885         buf->data = alloc_percpu(struct trace_array_cpu);
5886         if (!buf->data) {
5887                 ring_buffer_free(buf->buffer);
5888                 return -ENOMEM;
5889         }
5890
5891         init_trace_buffers(tr, buf);
5892
5893         /* Allocate the first page for all buffers */
5894         set_buffer_entries(&tr->trace_buffer,
5895                            ring_buffer_size(tr->trace_buffer.buffer, 0));
5896
5897         return 0;
5898 }
5899
5900 static int allocate_trace_buffers(struct trace_array *tr, int size)
5901 {
5902         int ret;
5903
5904         ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5905         if (ret)
5906                 return ret;
5907
5908 #ifdef CONFIG_TRACER_MAX_TRACE
5909         ret = allocate_trace_buffer(tr, &tr->max_buffer,
5910                                     allocate_snapshot ? size : 1);
5911         if (WARN_ON(ret)) {
5912                 ring_buffer_free(tr->trace_buffer.buffer);
5913                 free_percpu(tr->trace_buffer.data);
5914                 return -ENOMEM;
5915         }
5916         tr->allocated_snapshot = allocate_snapshot;
5917
5918         /*
5919          * Only the top level trace array gets its snapshot allocated
5920          * from the kernel command line.
5921          */
5922         allocate_snapshot = false;
5923 #endif
5924         return 0;
5925 }
5926
5927 static int new_instance_create(const char *name)
5928 {
5929         struct trace_array *tr;
5930         int ret;
5931
5932         mutex_lock(&trace_types_lock);
5933
5934         ret = -EEXIST;
5935         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5936                 if (tr->name && strcmp(tr->name, name) == 0)
5937                         goto out_unlock;
5938         }
5939
5940         ret = -ENOMEM;
5941         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5942         if (!tr)
5943                 goto out_unlock;
5944
5945         tr->name = kstrdup(name, GFP_KERNEL);
5946         if (!tr->name)
5947                 goto out_free_tr;
5948
5949         raw_spin_lock_init(&tr->start_lock);
5950
5951         tr->current_trace = &nop_trace;
5952
5953         INIT_LIST_HEAD(&tr->systems);
5954         INIT_LIST_HEAD(&tr->events);
5955
5956         if (allocate_trace_buffers(tr, trace_buf_size) < 0)
5957                 goto out_free_tr;
5958
5959         /* Holder for file callbacks */
5960         tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5961         tr->trace_cpu.tr = tr;
5962
5963         tr->dir = debugfs_create_dir(name, trace_instance_dir);
5964         if (!tr->dir)
5965                 goto out_free_tr;
5966
5967         ret = event_trace_add_tracer(tr->dir, tr);
5968         if (ret) {
5969                 debugfs_remove_recursive(tr->dir);
5970                 goto out_free_tr;
5971         }
5972
5973         init_tracer_debugfs(tr, tr->dir);
5974
5975         list_add(&tr->list, &ftrace_trace_arrays);
5976
5977         mutex_unlock(&trace_types_lock);
5978
5979         return 0;
5980
5981  out_free_tr:
5982         if (tr->trace_buffer.buffer)
5983                 ring_buffer_free(tr->trace_buffer.buffer);
5984         kfree(tr->name);
5985         kfree(tr);
5986
5987  out_unlock:
5988         mutex_unlock(&trace_types_lock);
5989
5990         return ret;
5991
5992 }
5993
5994 static int instance_delete(const char *name)
5995 {
5996         struct trace_array *tr;
5997         int found = 0;
5998         int ret;
5999
6000         mutex_lock(&trace_types_lock);
6001
6002         ret = -ENODEV;
6003         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6004                 if (tr->name && strcmp(tr->name, name) == 0) {
6005                         found = 1;
6006                         break;
6007                 }
6008         }
6009         if (!found)
6010                 goto out_unlock;
6011
6012         ret = -EBUSY;
6013         if (tr->ref)
6014                 goto out_unlock;
6015
6016         list_del(&tr->list);
6017
6018         event_trace_del_tracer(tr);
6019         debugfs_remove_recursive(tr->dir);
6020         free_percpu(tr->trace_buffer.data);
6021         ring_buffer_free(tr->trace_buffer.buffer);
6022
6023         kfree(tr->name);
6024         kfree(tr);
6025
6026         ret = 0;
6027
6028  out_unlock:
6029         mutex_unlock(&trace_types_lock);
6030
6031         return ret;
6032 }
6033
6034 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6035 {
6036         struct dentry *parent;
6037         int ret;
6038
6039         /* Paranoid: Make sure the parent is the "instances" directory */
6040         parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6041         if (WARN_ON_ONCE(parent != trace_instance_dir))
6042                 return -ENOENT;
6043
6044         /*
6045          * The inode mutex is locked, but debugfs_create_dir() will also
6046          * take the mutex. As the instances directory can not be destroyed
6047          * or changed in any other way, it is safe to unlock it, and
6048          * let the dentry try. If two users try to make the same dir at
6049          * the same time, then the new_instance_create() will determine the
6050          * winner.
6051          */
6052         mutex_unlock(&inode->i_mutex);
6053
6054         ret = new_instance_create(dentry->d_iname);
6055
6056         mutex_lock(&inode->i_mutex);
6057
6058         return ret;
6059 }
6060
6061 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6062 {
6063         struct dentry *parent;
6064         int ret;
6065
6066         /* Paranoid: Make sure the parent is the "instances" directory */
6067         parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6068         if (WARN_ON_ONCE(parent != trace_instance_dir))
6069                 return -ENOENT;
6070
6071         /* The caller did a dget() on dentry */
6072         mutex_unlock(&dentry->d_inode->i_mutex);
6073
6074         /*
6075          * The inode mutex is locked, but debugfs_create_dir() will also
6076          * take the mutex. As the instances directory can not be destroyed
6077          * or changed in any other way, it is safe to unlock it, and
6078          * let the dentry try. If two users try to make the same dir at
6079          * the same time, then the instance_delete() will determine the
6080          * winner.
6081          */
6082         mutex_unlock(&inode->i_mutex);
6083
6084         ret = instance_delete(dentry->d_iname);
6085
6086         mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6087         mutex_lock(&dentry->d_inode->i_mutex);
6088
6089         return ret;
6090 }
6091
6092 static const struct inode_operations instance_dir_inode_operations = {
6093         .lookup         = simple_lookup,
6094         .mkdir          = instance_mkdir,
6095         .rmdir          = instance_rmdir,
6096 };
6097
6098 static __init void create_trace_instances(struct dentry *d_tracer)
6099 {
6100         trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6101         if (WARN_ON(!trace_instance_dir))
6102                 return;
6103
6104         /* Hijack the dir inode operations, to allow mkdir */
6105         trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6106 }
6107
6108 static void
6109 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6110 {
6111         int cpu;
6112
6113         trace_create_file("trace_options", 0644, d_tracer,
6114                           tr, &tracing_iter_fops);
6115
6116         trace_create_file("trace", 0644, d_tracer,
6117                           tr, &tracing_fops);
6118
6119         trace_create_file("trace_pipe", 0444, d_tracer,
6120                           tr, &tracing_pipe_fops);
6121
6122         trace_create_file("buffer_size_kb", 0644, d_tracer,
6123                           tr, &tracing_entries_fops);
6124
6125         trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6126                           tr, &tracing_total_entries_fops);
6127
6128         trace_create_file("free_buffer", 0644, d_tracer,
6129                           tr, &tracing_free_buffer_fops);
6130
6131         trace_create_file("trace_marker", 0220, d_tracer,
6132                           tr, &tracing_mark_fops);
6133
6134         trace_create_file("trace_clock", 0644, d_tracer, tr,
6135                           &trace_clock_fops);
6136
6137         trace_create_file("tracing_on", 0644, d_tracer,
6138                           tr, &rb_simple_fops);
6139
6140 #ifdef CONFIG_TRACER_SNAPSHOT
6141         trace_create_file("snapshot", 0644, d_tracer,
6142                           tr, &snapshot_fops);
6143 #endif
6144
6145         for_each_tracing_cpu(cpu)
6146                 tracing_init_debugfs_percpu(tr, cpu);
6147
6148 }
6149
6150 static __init int tracer_init_debugfs(void)
6151 {
6152         struct dentry *d_tracer;
6153
6154         trace_access_lock_init();
6155
6156         d_tracer = tracing_init_dentry();
6157         if (!d_tracer)
6158                 return 0;
6159
6160         init_tracer_debugfs(&global_trace, d_tracer);
6161
6162         trace_create_file("tracing_cpumask", 0644, d_tracer,
6163                         &global_trace, &tracing_cpumask_fops);
6164
6165         trace_create_file("available_tracers", 0444, d_tracer,
6166                         &global_trace, &show_traces_fops);
6167
6168         trace_create_file("current_tracer", 0644, d_tracer,
6169                         &global_trace, &set_tracer_fops);
6170
6171 #ifdef CONFIG_TRACER_MAX_TRACE
6172         trace_create_file("tracing_max_latency", 0644, d_tracer,
6173                         &tracing_max_latency, &tracing_max_lat_fops);
6174 #endif
6175
6176         trace_create_file("tracing_thresh", 0644, d_tracer,
6177                         &tracing_thresh, &tracing_max_lat_fops);
6178
6179         trace_create_file("README", 0444, d_tracer,
6180                         NULL, &tracing_readme_fops);
6181
6182         trace_create_file("saved_cmdlines", 0444, d_tracer,
6183                         NULL, &tracing_saved_cmdlines_fops);
6184
6185 #ifdef CONFIG_DYNAMIC_FTRACE
6186         trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6187                         &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6188 #endif
6189
6190         create_trace_instances(d_tracer);
6191
6192         create_trace_options_dir(&global_trace);
6193
6194         return 0;
6195 }
6196
6197 static int trace_panic_handler(struct notifier_block *this,
6198                                unsigned long event, void *unused)
6199 {
6200         if (ftrace_dump_on_oops)
6201                 ftrace_dump(ftrace_dump_on_oops);
6202         return NOTIFY_OK;
6203 }
6204
6205 static struct notifier_block trace_panic_notifier = {
6206         .notifier_call  = trace_panic_handler,
6207         .next           = NULL,
6208         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
6209 };
6210
6211 static int trace_die_handler(struct notifier_block *self,
6212                              unsigned long val,
6213                              void *data)
6214 {
6215         switch (val) {
6216         case DIE_OOPS:
6217                 if (ftrace_dump_on_oops)
6218                         ftrace_dump(ftrace_dump_on_oops);
6219                 break;
6220         default:
6221                 break;
6222         }
6223         return NOTIFY_OK;
6224 }
6225
6226 static struct notifier_block trace_die_notifier = {
6227         .notifier_call = trace_die_handler,
6228         .priority = 200
6229 };
6230
6231 /*
6232  * printk is set to max of 1024, we really don't need it that big.
6233  * Nothing should be printing 1000 characters anyway.
6234  */
6235 #define TRACE_MAX_PRINT         1000
6236
6237 /*
6238  * Define here KERN_TRACE so that we have one place to modify
6239  * it if we decide to change what log level the ftrace dump
6240  * should be at.
6241  */
6242 #define KERN_TRACE              KERN_EMERG
6243
6244 void
6245 trace_printk_seq(struct trace_seq *s)
6246 {
6247         /* Probably should print a warning here. */
6248         if (s->len >= TRACE_MAX_PRINT)
6249                 s->len = TRACE_MAX_PRINT;
6250
6251         /* should be zero ended, but we are paranoid. */
6252         s->buffer[s->len] = 0;
6253
6254         printk(KERN_TRACE "%s", s->buffer);
6255
6256         trace_seq_init(s);
6257 }
6258
6259 void trace_init_global_iter(struct trace_iterator *iter)
6260 {
6261         iter->tr = &global_trace;
6262         iter->trace = iter->tr->current_trace;
6263         iter->cpu_file = RING_BUFFER_ALL_CPUS;
6264         iter->trace_buffer = &global_trace.trace_buffer;
6265 }
6266
6267 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6268 {
6269         /* use static because iter can be a bit big for the stack */
6270         static struct trace_iterator iter;
6271         static atomic_t dump_running;
6272         unsigned int old_userobj;
6273         unsigned long flags;
6274         int cnt = 0, cpu;
6275
6276         /* Only allow one dump user at a time. */
6277         if (atomic_inc_return(&dump_running) != 1) {
6278                 atomic_dec(&dump_running);
6279                 return;
6280         }
6281
6282         /*
6283          * Always turn off tracing when we dump.
6284          * We don't need to show trace output of what happens
6285          * between multiple crashes.
6286          *
6287          * If the user does a sysrq-z, then they can re-enable
6288          * tracing with echo 1 > tracing_on.
6289          */
6290         tracing_off();
6291
6292         local_irq_save(flags);
6293
6294         /* Simulate the iterator */
6295         trace_init_global_iter(&iter);
6296
6297         for_each_tracing_cpu(cpu) {
6298                 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6299         }
6300
6301         old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6302
6303         /* don't look at user memory in panic mode */
6304         trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6305
6306         switch (oops_dump_mode) {
6307         case DUMP_ALL:
6308                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6309                 break;
6310         case DUMP_ORIG:
6311                 iter.cpu_file = raw_smp_processor_id();
6312                 break;
6313         case DUMP_NONE:
6314                 goto out_enable;
6315         default:
6316                 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6317                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6318         }
6319
6320         printk(KERN_TRACE "Dumping ftrace buffer:\n");
6321
6322         /* Did function tracer already get disabled? */
6323         if (ftrace_is_dead()) {
6324                 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6325                 printk("#          MAY BE MISSING FUNCTION EVENTS\n");
6326         }
6327
6328         /*
6329          * We need to stop all tracing on all CPUS to read the
6330          * the next buffer. This is a bit expensive, but is
6331          * not done often. We fill all what we can read,
6332          * and then release the locks again.
6333          */
6334
6335         while (!trace_empty(&iter)) {
6336
6337                 if (!cnt)
6338                         printk(KERN_TRACE "---------------------------------\n");
6339
6340                 cnt++;
6341
6342                 /* reset all but tr, trace, and overruns */
6343                 memset(&iter.seq, 0,
6344                        sizeof(struct trace_iterator) -
6345                        offsetof(struct trace_iterator, seq));
6346                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6347                 iter.pos = -1;
6348
6349                 if (trace_find_next_entry_inc(&iter) != NULL) {
6350                         int ret;
6351
6352                         ret = print_trace_line(&iter);
6353                         if (ret != TRACE_TYPE_NO_CONSUME)
6354                                 trace_consume(&iter);
6355                 }
6356                 touch_nmi_watchdog();
6357
6358                 trace_printk_seq(&iter.seq);
6359         }
6360
6361         if (!cnt)
6362                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
6363         else
6364                 printk(KERN_TRACE "---------------------------------\n");
6365
6366  out_enable:
6367         trace_flags |= old_userobj;
6368
6369         for_each_tracing_cpu(cpu) {
6370                 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6371         }
6372         atomic_dec(&dump_running);
6373         local_irq_restore(flags);
6374 }
6375 EXPORT_SYMBOL_GPL(ftrace_dump);
6376
6377 __init static int tracer_alloc_buffers(void)
6378 {
6379         int ring_buf_size;
6380         int ret = -ENOMEM;
6381
6382
6383         if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6384                 goto out;
6385
6386         if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
6387                 goto out_free_buffer_mask;
6388
6389         /* Only allocate trace_printk buffers if a trace_printk exists */
6390         if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6391                 /* Must be called before global_trace.buffer is allocated */
6392                 trace_printk_init_buffers();
6393
6394         /* To save memory, keep the ring buffer size to its minimum */
6395         if (ring_buffer_expanded)
6396                 ring_buf_size = trace_buf_size;
6397         else
6398                 ring_buf_size = 1;
6399
6400         cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6401         cpumask_copy(tracing_cpumask, cpu_all_mask);
6402
6403         raw_spin_lock_init(&global_trace.start_lock);
6404
6405         /* TODO: make the number of buffers hot pluggable with CPUS */
6406         if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6407                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6408                 WARN_ON(1);
6409                 goto out_free_cpumask;
6410         }
6411
6412         if (global_trace.buffer_disabled)
6413                 tracing_off();
6414
6415         trace_init_cmdlines();
6416
6417         /*
6418          * register_tracer() might reference current_trace, so it
6419          * needs to be set before we register anything. This is
6420          * just a bootstrap of current_trace anyway.
6421          */
6422         global_trace.current_trace = &nop_trace;
6423
6424         register_tracer(&nop_trace);
6425
6426         /* All seems OK, enable tracing */
6427         tracing_disabled = 0;
6428
6429         atomic_notifier_chain_register(&panic_notifier_list,
6430                                        &trace_panic_notifier);
6431
6432         register_die_notifier(&trace_die_notifier);
6433
6434         global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6435
6436         /* Holder for file callbacks */
6437         global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6438         global_trace.trace_cpu.tr = &global_trace;
6439
6440         INIT_LIST_HEAD(&global_trace.systems);
6441         INIT_LIST_HEAD(&global_trace.events);
6442         list_add(&global_trace.list, &ftrace_trace_arrays);
6443
6444         while (trace_boot_options) {
6445                 char *option;
6446
6447                 option = strsep(&trace_boot_options, ",");
6448                 trace_set_options(&global_trace, option);
6449         }
6450
6451         register_snapshot_cmd();
6452
6453         return 0;
6454
6455 out_free_cpumask:
6456         free_percpu(global_trace.trace_buffer.data);
6457 #ifdef CONFIG_TRACER_MAX_TRACE
6458         free_percpu(global_trace.max_buffer.data);
6459 #endif
6460         free_cpumask_var(tracing_cpumask);
6461 out_free_buffer_mask:
6462         free_cpumask_var(tracing_buffer_mask);
6463 out:
6464         return ret;
6465 }
6466
6467 __init static int clear_boot_tracer(void)
6468 {
6469         /*
6470          * The default tracer at boot buffer is an init section.
6471          * This function is called in lateinit. If we did not
6472          * find the boot tracer, then clear it out, to prevent
6473          * later registration from accessing the buffer that is
6474          * about to be freed.
6475          */
6476         if (!default_bootup_tracer)
6477                 return 0;
6478
6479         printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6480                default_bootup_tracer);
6481         default_bootup_tracer = NULL;
6482
6483         return 0;
6484 }
6485
6486 early_initcall(tracer_alloc_buffers);
6487 fs_initcall(tracer_init_debugfs);
6488 late_initcall(clear_boot_tracer);