5c53e4092269d40e73e0e2bcca20f02297406b31
[linux-3.10.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 Nadia Yvette Chambers
13  */
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
40 #include <linux/fs.h>
41 #include <linux/sched/rt.h>
42
43 #include "trace.h"
44 #include "trace_output.h"
45
46 /*
47  * On boot up, the ring buffer is set to the minimum size, so that
48  * we do not waste memory on systems that are not using tracing.
49  */
50 bool ring_buffer_expanded;
51
52 /*
53  * We need to change this state when a selftest is running.
54  * A selftest will lurk into the ring-buffer to count the
55  * entries inserted during the selftest although some concurrent
56  * insertions into the ring-buffer such as trace_printk could occurred
57  * at the same time, giving false positive or negative results.
58  */
59 static bool __read_mostly tracing_selftest_running;
60
61 /*
62  * If a tracer is running, we do not want to run SELFTEST.
63  */
64 bool __read_mostly tracing_selftest_disabled;
65
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
68         { }
69 };
70
71 static struct tracer_flags dummy_tracer_flags = {
72         .val = 0,
73         .opts = dummy_tracer_opt
74 };
75
76 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77 {
78         return 0;
79 }
80
81 /*
82  * To prevent the comm cache from being overwritten when no
83  * tracing is active, only save the comm when a trace event
84  * occurred.
85  */
86 static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88 /*
89  * Kill all tracing for good (never come back).
90  * It is initialized to 1 but will turn to zero if the initialization
91  * of the tracer is successful. But that is the only place that sets
92  * this back to zero.
93  */
94 static int tracing_disabled = 1;
95
96 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
97
98 cpumask_var_t __read_mostly     tracing_buffer_mask;
99
100 /*
101  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102  *
103  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104  * is set, then ftrace_dump is called. This will output the contents
105  * of the ftrace buffers to the console.  This is very useful for
106  * capturing traces that lead to crashes and outputing it to a
107  * serial console.
108  *
109  * It is default off, but you can enable it with either specifying
110  * "ftrace_dump_on_oops" in the kernel command line, or setting
111  * /proc/sys/kernel/ftrace_dump_on_oops
112  * Set 1 if you want to dump buffers of all CPUs
113  * Set 2 if you want to dump the buffer of the CPU that triggered oops
114  */
115
116 enum ftrace_dump_mode ftrace_dump_on_oops;
117
118 static int tracing_set_tracer(const char *buf);
119
120 #define MAX_TRACER_SIZE         100
121 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
122 static char *default_bootup_tracer;
123
124 static bool allocate_snapshot;
125
126 static int __init set_cmdline_ftrace(char *str)
127 {
128         strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
129         default_bootup_tracer = bootup_tracer_buf;
130         /* We are using ftrace early, expand it */
131         ring_buffer_expanded = true;
132         return 1;
133 }
134 __setup("ftrace=", set_cmdline_ftrace);
135
136 static int __init set_ftrace_dump_on_oops(char *str)
137 {
138         if (*str++ != '=' || !*str) {
139                 ftrace_dump_on_oops = DUMP_ALL;
140                 return 1;
141         }
142
143         if (!strcmp("orig_cpu", str)) {
144                 ftrace_dump_on_oops = DUMP_ORIG;
145                 return 1;
146         }
147
148         return 0;
149 }
150 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
151
152 static int __init alloc_snapshot(char *str)
153 {
154         allocate_snapshot = true;
155         /* We also need the main ring buffer expanded */
156         ring_buffer_expanded = true;
157         return 1;
158 }
159 __setup("alloc_snapshot", alloc_snapshot);
160
161
162 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
163 static char *trace_boot_options __initdata;
164
165 static int __init set_trace_boot_options(char *str)
166 {
167         strncpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
168         trace_boot_options = trace_boot_options_buf;
169         return 0;
170 }
171 __setup("trace_options=", set_trace_boot_options);
172
173 unsigned long long ns2usecs(cycle_t nsec)
174 {
175         nsec += 500;
176         do_div(nsec, 1000);
177         return nsec;
178 }
179
180 /*
181  * The global_trace is the descriptor that holds the tracing
182  * buffers for the live tracing. For each CPU, it contains
183  * a link list of pages that will store trace entries. The
184  * page descriptor of the pages in the memory is used to hold
185  * the link list by linking the lru item in the page descriptor
186  * to each of the pages in the buffer per CPU.
187  *
188  * For each active CPU there is a data field that holds the
189  * pages for the buffer for that CPU. Each CPU has the same number
190  * of pages allocated for its buffer.
191  */
192 static struct trace_array       global_trace;
193
194 LIST_HEAD(ftrace_trace_arrays);
195
196 int filter_current_check_discard(struct ring_buffer *buffer,
197                                  struct ftrace_event_call *call, void *rec,
198                                  struct ring_buffer_event *event)
199 {
200         return filter_check_discard(call, rec, buffer, event);
201 }
202 EXPORT_SYMBOL_GPL(filter_current_check_discard);
203
204 cycle_t ftrace_now(int cpu)
205 {
206         u64 ts;
207
208         /* Early boot up does not have a buffer yet */
209         if (!global_trace.trace_buffer.buffer)
210                 return trace_clock_local();
211
212         ts = ring_buffer_time_stamp(global_trace.trace_buffer.buffer, cpu);
213         ring_buffer_normalize_time_stamp(global_trace.trace_buffer.buffer, cpu, &ts);
214
215         return ts;
216 }
217
218 int tracing_is_enabled(void)
219 {
220         return tracing_is_on();
221 }
222
223 /*
224  * trace_buf_size is the size in bytes that is allocated
225  * for a buffer. Note, the number of bytes is always rounded
226  * to page size.
227  *
228  * This number is purposely set to a low number of 16384.
229  * If the dump on oops happens, it will be much appreciated
230  * to not have to wait for all that output. Anyway this can be
231  * boot time and run time configurable.
232  */
233 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
234
235 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
236
237 /* trace_types holds a link list of available tracers. */
238 static struct tracer            *trace_types __read_mostly;
239
240 /*
241  * trace_types_lock is used to protect the trace_types list.
242  */
243 static DEFINE_MUTEX(trace_types_lock);
244
245 /*
246  * serialize the access of the ring buffer
247  *
248  * ring buffer serializes readers, but it is low level protection.
249  * The validity of the events (which returns by ring_buffer_peek() ..etc)
250  * are not protected by ring buffer.
251  *
252  * The content of events may become garbage if we allow other process consumes
253  * these events concurrently:
254  *   A) the page of the consumed events may become a normal page
255  *      (not reader page) in ring buffer, and this page will be rewrited
256  *      by events producer.
257  *   B) The page of the consumed events may become a page for splice_read,
258  *      and this page will be returned to system.
259  *
260  * These primitives allow multi process access to different cpu ring buffer
261  * concurrently.
262  *
263  * These primitives don't distinguish read-only and read-consume access.
264  * Multi read-only access are also serialized.
265  */
266
267 #ifdef CONFIG_SMP
268 static DECLARE_RWSEM(all_cpu_access_lock);
269 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
270
271 static inline void trace_access_lock(int cpu)
272 {
273         if (cpu == RING_BUFFER_ALL_CPUS) {
274                 /* gain it for accessing the whole ring buffer. */
275                 down_write(&all_cpu_access_lock);
276         } else {
277                 /* gain it for accessing a cpu ring buffer. */
278
279                 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
280                 down_read(&all_cpu_access_lock);
281
282                 /* Secondly block other access to this @cpu ring buffer. */
283                 mutex_lock(&per_cpu(cpu_access_lock, cpu));
284         }
285 }
286
287 static inline void trace_access_unlock(int cpu)
288 {
289         if (cpu == RING_BUFFER_ALL_CPUS) {
290                 up_write(&all_cpu_access_lock);
291         } else {
292                 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
293                 up_read(&all_cpu_access_lock);
294         }
295 }
296
297 static inline void trace_access_lock_init(void)
298 {
299         int cpu;
300
301         for_each_possible_cpu(cpu)
302                 mutex_init(&per_cpu(cpu_access_lock, cpu));
303 }
304
305 #else
306
307 static DEFINE_MUTEX(access_lock);
308
309 static inline void trace_access_lock(int cpu)
310 {
311         (void)cpu;
312         mutex_lock(&access_lock);
313 }
314
315 static inline void trace_access_unlock(int cpu)
316 {
317         (void)cpu;
318         mutex_unlock(&access_lock);
319 }
320
321 static inline void trace_access_lock_init(void)
322 {
323 }
324
325 #endif
326
327 /* trace_flags holds trace_options default values */
328 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
329         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
330         TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
331         TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS;
332
333 /**
334  * tracing_on - enable tracing buffers
335  *
336  * This function enables tracing buffers that may have been
337  * disabled with tracing_off.
338  */
339 void tracing_on(void)
340 {
341         if (global_trace.trace_buffer.buffer)
342                 ring_buffer_record_on(global_trace.trace_buffer.buffer);
343         /*
344          * This flag is only looked at when buffers haven't been
345          * allocated yet. We don't really care about the race
346          * between setting this flag and actually turning
347          * on the buffer.
348          */
349         global_trace.buffer_disabled = 0;
350 }
351 EXPORT_SYMBOL_GPL(tracing_on);
352
353 /**
354  * __trace_puts - write a constant string into the trace buffer.
355  * @ip:    The address of the caller
356  * @str:   The constant string to write
357  * @size:  The size of the string.
358  */
359 int __trace_puts(unsigned long ip, const char *str, int size)
360 {
361         struct ring_buffer_event *event;
362         struct ring_buffer *buffer;
363         struct print_entry *entry;
364         unsigned long irq_flags;
365         int alloc;
366
367         alloc = sizeof(*entry) + size + 2; /* possible \n added */
368
369         local_save_flags(irq_flags);
370         buffer = global_trace.trace_buffer.buffer;
371         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
372                                           irq_flags, preempt_count());
373         if (!event)
374                 return 0;
375
376         entry = ring_buffer_event_data(event);
377         entry->ip = ip;
378
379         memcpy(&entry->buf, str, size);
380
381         /* Add a newline if necessary */
382         if (entry->buf[size - 1] != '\n') {
383                 entry->buf[size] = '\n';
384                 entry->buf[size + 1] = '\0';
385         } else
386                 entry->buf[size] = '\0';
387
388         __buffer_unlock_commit(buffer, event);
389
390         return size;
391 }
392 EXPORT_SYMBOL_GPL(__trace_puts);
393
394 /**
395  * __trace_bputs - write the pointer to a constant string into trace buffer
396  * @ip:    The address of the caller
397  * @str:   The constant string to write to the buffer to
398  */
399 int __trace_bputs(unsigned long ip, const char *str)
400 {
401         struct ring_buffer_event *event;
402         struct ring_buffer *buffer;
403         struct bputs_entry *entry;
404         unsigned long irq_flags;
405         int size = sizeof(struct bputs_entry);
406
407         local_save_flags(irq_flags);
408         buffer = global_trace.trace_buffer.buffer;
409         event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
410                                           irq_flags, preempt_count());
411         if (!event)
412                 return 0;
413
414         entry = ring_buffer_event_data(event);
415         entry->ip                       = ip;
416         entry->str                      = str;
417
418         __buffer_unlock_commit(buffer, event);
419
420         return 1;
421 }
422 EXPORT_SYMBOL_GPL(__trace_bputs);
423
424 #ifdef CONFIG_TRACER_SNAPSHOT
425 /**
426  * trace_snapshot - take a snapshot of the current buffer.
427  *
428  * This causes a swap between the snapshot buffer and the current live
429  * tracing buffer. You can use this to take snapshots of the live
430  * trace when some condition is triggered, but continue to trace.
431  *
432  * Note, make sure to allocate the snapshot with either
433  * a tracing_snapshot_alloc(), or by doing it manually
434  * with: echo 1 > /sys/kernel/debug/tracing/snapshot
435  *
436  * If the snapshot buffer is not allocated, it will stop tracing.
437  * Basically making a permanent snapshot.
438  */
439 void tracing_snapshot(void)
440 {
441         struct trace_array *tr = &global_trace;
442         struct tracer *tracer = tr->current_trace;
443         unsigned long flags;
444
445         if (in_nmi()) {
446                 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
447                 internal_trace_puts("*** snapshot is being ignored        ***\n");
448                 return;
449         }
450
451         if (!tr->allocated_snapshot) {
452                 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
453                 internal_trace_puts("*** stopping trace here!   ***\n");
454                 tracing_off();
455                 return;
456         }
457
458         /* Note, snapshot can not be used when the tracer uses it */
459         if (tracer->use_max_tr) {
460                 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
461                 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
462                 return;
463         }
464
465         local_irq_save(flags);
466         update_max_tr(tr, current, smp_processor_id());
467         local_irq_restore(flags);
468 }
469 EXPORT_SYMBOL_GPL(tracing_snapshot);
470
471 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
472                                         struct trace_buffer *size_buf, int cpu_id);
473
474 /**
475  * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
476  *
477  * This is similar to trace_snapshot(), but it will allocate the
478  * snapshot buffer if it isn't already allocated. Use this only
479  * where it is safe to sleep, as the allocation may sleep.
480  *
481  * This causes a swap between the snapshot buffer and the current live
482  * tracing buffer. You can use this to take snapshots of the live
483  * trace when some condition is triggered, but continue to trace.
484  */
485 void tracing_snapshot_alloc(void)
486 {
487         struct trace_array *tr = &global_trace;
488         int ret;
489
490         if (!tr->allocated_snapshot) {
491
492                 /* allocate spare buffer */
493                 ret = resize_buffer_duplicate_size(&tr->max_buffer,
494                                    &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
495                 if (WARN_ON(ret < 0))
496                         return;
497
498                 tr->allocated_snapshot = true;
499         }
500
501         tracing_snapshot();
502 }
503 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
504 #else
505 void tracing_snapshot(void)
506 {
507         WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
508 }
509 EXPORT_SYMBOL_GPL(tracing_snapshot);
510 void tracing_snapshot_alloc(void)
511 {
512         /* Give warning */
513         tracing_snapshot();
514 }
515 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
516 #endif /* CONFIG_TRACER_SNAPSHOT */
517
518 /**
519  * tracing_off - turn off tracing buffers
520  *
521  * This function stops the tracing buffers from recording data.
522  * It does not disable any overhead the tracers themselves may
523  * be causing. This function simply causes all recording to
524  * the ring buffers to fail.
525  */
526 void tracing_off(void)
527 {
528         if (global_trace.trace_buffer.buffer)
529                 ring_buffer_record_off(global_trace.trace_buffer.buffer);
530         /*
531          * This flag is only looked at when buffers haven't been
532          * allocated yet. We don't really care about the race
533          * between setting this flag and actually turning
534          * on the buffer.
535          */
536         global_trace.buffer_disabled = 1;
537 }
538 EXPORT_SYMBOL_GPL(tracing_off);
539
540 /**
541  * tracing_is_on - show state of ring buffers enabled
542  */
543 int tracing_is_on(void)
544 {
545         if (global_trace.trace_buffer.buffer)
546                 return ring_buffer_record_is_on(global_trace.trace_buffer.buffer);
547         return !global_trace.buffer_disabled;
548 }
549 EXPORT_SYMBOL_GPL(tracing_is_on);
550
551 static int __init set_buf_size(char *str)
552 {
553         unsigned long buf_size;
554
555         if (!str)
556                 return 0;
557         buf_size = memparse(str, &str);
558         /* nr_entries can not be zero */
559         if (buf_size == 0)
560                 return 0;
561         trace_buf_size = buf_size;
562         return 1;
563 }
564 __setup("trace_buf_size=", set_buf_size);
565
566 static int __init set_tracing_thresh(char *str)
567 {
568         unsigned long threshold;
569         int ret;
570
571         if (!str)
572                 return 0;
573         ret = kstrtoul(str, 0, &threshold);
574         if (ret < 0)
575                 return 0;
576         tracing_thresh = threshold * 1000;
577         return 1;
578 }
579 __setup("tracing_thresh=", set_tracing_thresh);
580
581 unsigned long nsecs_to_usecs(unsigned long nsecs)
582 {
583         return nsecs / 1000;
584 }
585
586 /* These must match the bit postions in trace_iterator_flags */
587 static const char *trace_options[] = {
588         "print-parent",
589         "sym-offset",
590         "sym-addr",
591         "verbose",
592         "raw",
593         "hex",
594         "bin",
595         "block",
596         "stacktrace",
597         "trace_printk",
598         "ftrace_preempt",
599         "branch",
600         "annotate",
601         "userstacktrace",
602         "sym-userobj",
603         "printk-msg-only",
604         "context-info",
605         "latency-format",
606         "sleep-time",
607         "graph-time",
608         "record-cmd",
609         "overwrite",
610         "disable_on_free",
611         "irq-info",
612         "markers",
613         NULL
614 };
615
616 static struct {
617         u64 (*func)(void);
618         const char *name;
619         int in_ns;              /* is this clock in nanoseconds? */
620 } trace_clocks[] = {
621         { trace_clock_local,    "local",        1 },
622         { trace_clock_global,   "global",       1 },
623         { trace_clock_counter,  "counter",      0 },
624         ARCH_TRACE_CLOCKS
625 };
626
627 int trace_clock_id;
628
629 /*
630  * trace_parser_get_init - gets the buffer for trace parser
631  */
632 int trace_parser_get_init(struct trace_parser *parser, int size)
633 {
634         memset(parser, 0, sizeof(*parser));
635
636         parser->buffer = kmalloc(size, GFP_KERNEL);
637         if (!parser->buffer)
638                 return 1;
639
640         parser->size = size;
641         return 0;
642 }
643
644 /*
645  * trace_parser_put - frees the buffer for trace parser
646  */
647 void trace_parser_put(struct trace_parser *parser)
648 {
649         kfree(parser->buffer);
650 }
651
652 /*
653  * trace_get_user - reads the user input string separated by  space
654  * (matched by isspace(ch))
655  *
656  * For each string found the 'struct trace_parser' is updated,
657  * and the function returns.
658  *
659  * Returns number of bytes read.
660  *
661  * See kernel/trace/trace.h for 'struct trace_parser' details.
662  */
663 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
664         size_t cnt, loff_t *ppos)
665 {
666         char ch;
667         size_t read = 0;
668         ssize_t ret;
669
670         if (!*ppos)
671                 trace_parser_clear(parser);
672
673         ret = get_user(ch, ubuf++);
674         if (ret)
675                 goto out;
676
677         read++;
678         cnt--;
679
680         /*
681          * The parser is not finished with the last write,
682          * continue reading the user input without skipping spaces.
683          */
684         if (!parser->cont) {
685                 /* skip white space */
686                 while (cnt && isspace(ch)) {
687                         ret = get_user(ch, ubuf++);
688                         if (ret)
689                                 goto out;
690                         read++;
691                         cnt--;
692                 }
693
694                 /* only spaces were written */
695                 if (isspace(ch)) {
696                         *ppos += read;
697                         ret = read;
698                         goto out;
699                 }
700
701                 parser->idx = 0;
702         }
703
704         /* read the non-space input */
705         while (cnt && !isspace(ch)) {
706                 if (parser->idx < parser->size - 1)
707                         parser->buffer[parser->idx++] = ch;
708                 else {
709                         ret = -EINVAL;
710                         goto out;
711                 }
712                 ret = get_user(ch, ubuf++);
713                 if (ret)
714                         goto out;
715                 read++;
716                 cnt--;
717         }
718
719         /* We either got finished input or we have to wait for another call. */
720         if (isspace(ch)) {
721                 parser->buffer[parser->idx] = 0;
722                 parser->cont = false;
723         } else {
724                 parser->cont = true;
725                 parser->buffer[parser->idx++] = ch;
726         }
727
728         *ppos += read;
729         ret = read;
730
731 out:
732         return ret;
733 }
734
735 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
736 {
737         int len;
738         int ret;
739
740         if (!cnt)
741                 return 0;
742
743         if (s->len <= s->readpos)
744                 return -EBUSY;
745
746         len = s->len - s->readpos;
747         if (cnt > len)
748                 cnt = len;
749         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
750         if (ret == cnt)
751                 return -EFAULT;
752
753         cnt -= ret;
754
755         s->readpos += cnt;
756         return cnt;
757 }
758
759 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
760 {
761         int len;
762
763         if (s->len <= s->readpos)
764                 return -EBUSY;
765
766         len = s->len - s->readpos;
767         if (cnt > len)
768                 cnt = len;
769         memcpy(buf, s->buffer + s->readpos, cnt);
770
771         s->readpos += cnt;
772         return cnt;
773 }
774
775 /*
776  * ftrace_max_lock is used to protect the swapping of buffers
777  * when taking a max snapshot. The buffers themselves are
778  * protected by per_cpu spinlocks. But the action of the swap
779  * needs its own lock.
780  *
781  * This is defined as a arch_spinlock_t in order to help
782  * with performance when lockdep debugging is enabled.
783  *
784  * It is also used in other places outside the update_max_tr
785  * so it needs to be defined outside of the
786  * CONFIG_TRACER_MAX_TRACE.
787  */
788 static arch_spinlock_t ftrace_max_lock =
789         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
790
791 unsigned long __read_mostly     tracing_thresh;
792
793 #ifdef CONFIG_TRACER_MAX_TRACE
794 unsigned long __read_mostly     tracing_max_latency;
795
796 /*
797  * Copy the new maximum trace into the separate maximum-trace
798  * structure. (this way the maximum trace is permanently saved,
799  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
800  */
801 static void
802 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
803 {
804         struct trace_buffer *trace_buf = &tr->trace_buffer;
805         struct trace_buffer *max_buf = &tr->max_buffer;
806         struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
807         struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
808
809         max_buf->cpu = cpu;
810         max_buf->time_start = data->preempt_timestamp;
811
812         max_data->saved_latency = tracing_max_latency;
813         max_data->critical_start = data->critical_start;
814         max_data->critical_end = data->critical_end;
815
816         memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
817         max_data->pid = tsk->pid;
818         max_data->uid = task_uid(tsk);
819         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
820         max_data->policy = tsk->policy;
821         max_data->rt_priority = tsk->rt_priority;
822
823         /* record this tasks comm */
824         tracing_record_cmdline(tsk);
825 }
826
827 /**
828  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
829  * @tr: tracer
830  * @tsk: the task with the latency
831  * @cpu: The cpu that initiated the trace.
832  *
833  * Flip the buffers between the @tr and the max_tr and record information
834  * about which task was the cause of this latency.
835  */
836 void
837 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
838 {
839         struct ring_buffer *buf;
840
841         if (tr->stop_count)
842                 return;
843
844         WARN_ON_ONCE(!irqs_disabled());
845
846         if (!tr->allocated_snapshot) {
847                 /* Only the nop tracer should hit this when disabling */
848                 WARN_ON_ONCE(tr->current_trace != &nop_trace);
849                 return;
850         }
851
852         arch_spin_lock(&ftrace_max_lock);
853
854         buf = tr->trace_buffer.buffer;
855         tr->trace_buffer.buffer = tr->max_buffer.buffer;
856         tr->max_buffer.buffer = buf;
857
858         __update_max_tr(tr, tsk, cpu);
859         arch_spin_unlock(&ftrace_max_lock);
860 }
861
862 /**
863  * update_max_tr_single - only copy one trace over, and reset the rest
864  * @tr - tracer
865  * @tsk - task with the latency
866  * @cpu - the cpu of the buffer to copy.
867  *
868  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
869  */
870 void
871 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
872 {
873         int ret;
874
875         if (tr->stop_count)
876                 return;
877
878         WARN_ON_ONCE(!irqs_disabled());
879         if (WARN_ON_ONCE(!tr->allocated_snapshot))
880                 return;
881
882         arch_spin_lock(&ftrace_max_lock);
883
884         ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
885
886         if (ret == -EBUSY) {
887                 /*
888                  * We failed to swap the buffer due to a commit taking
889                  * place on this CPU. We fail to record, but we reset
890                  * the max trace buffer (no one writes directly to it)
891                  * and flag that it failed.
892                  */
893                 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
894                         "Failed to swap buffers due to commit in progress\n");
895         }
896
897         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
898
899         __update_max_tr(tr, tsk, cpu);
900         arch_spin_unlock(&ftrace_max_lock);
901 }
902 #endif /* CONFIG_TRACER_MAX_TRACE */
903
904 static void default_wait_pipe(struct trace_iterator *iter)
905 {
906         /* Iterators are static, they should be filled or empty */
907         if (trace_buffer_iter(iter, iter->cpu_file))
908                 return;
909
910         ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
911 }
912
913 #ifdef CONFIG_FTRACE_STARTUP_TEST
914 static int run_tracer_selftest(struct tracer *type)
915 {
916         struct trace_array *tr = &global_trace;
917         struct tracer *saved_tracer = tr->current_trace;
918         int ret;
919
920         if (!type->selftest || tracing_selftest_disabled)
921                 return 0;
922
923         /*
924          * Run a selftest on this tracer.
925          * Here we reset the trace buffer, and set the current
926          * tracer to be this tracer. The tracer can then run some
927          * internal tracing to verify that everything is in order.
928          * If we fail, we do not register this tracer.
929          */
930         tracing_reset_online_cpus(&tr->trace_buffer);
931
932         tr->current_trace = type;
933
934 #ifdef CONFIG_TRACER_MAX_TRACE
935         if (type->use_max_tr) {
936                 /* If we expanded the buffers, make sure the max is expanded too */
937                 if (ring_buffer_expanded)
938                         ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
939                                            RING_BUFFER_ALL_CPUS);
940                 tr->allocated_snapshot = true;
941         }
942 #endif
943
944         /* the test is responsible for initializing and enabling */
945         pr_info("Testing tracer %s: ", type->name);
946         ret = type->selftest(type, tr);
947         /* the test is responsible for resetting too */
948         tr->current_trace = saved_tracer;
949         if (ret) {
950                 printk(KERN_CONT "FAILED!\n");
951                 /* Add the warning after printing 'FAILED' */
952                 WARN_ON(1);
953                 return -1;
954         }
955         /* Only reset on passing, to avoid touching corrupted buffers */
956         tracing_reset_online_cpus(&tr->trace_buffer);
957
958 #ifdef CONFIG_TRACER_MAX_TRACE
959         if (type->use_max_tr) {
960                 tr->allocated_snapshot = false;
961
962                 /* Shrink the max buffer again */
963                 if (ring_buffer_expanded)
964                         ring_buffer_resize(tr->max_buffer.buffer, 1,
965                                            RING_BUFFER_ALL_CPUS);
966         }
967 #endif
968
969         printk(KERN_CONT "PASSED\n");
970         return 0;
971 }
972 #else
973 static inline int run_tracer_selftest(struct tracer *type)
974 {
975         return 0;
976 }
977 #endif /* CONFIG_FTRACE_STARTUP_TEST */
978
979 /**
980  * register_tracer - register a tracer with the ftrace system.
981  * @type - the plugin for the tracer
982  *
983  * Register a new plugin tracer.
984  */
985 int register_tracer(struct tracer *type)
986 {
987         struct tracer *t;
988         int ret = 0;
989
990         if (!type->name) {
991                 pr_info("Tracer must have a name\n");
992                 return -1;
993         }
994
995         if (strlen(type->name) >= MAX_TRACER_SIZE) {
996                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
997                 return -1;
998         }
999
1000         mutex_lock(&trace_types_lock);
1001
1002         tracing_selftest_running = true;
1003
1004         for (t = trace_types; t; t = t->next) {
1005                 if (strcmp(type->name, t->name) == 0) {
1006                         /* already found */
1007                         pr_info("Tracer %s already registered\n",
1008                                 type->name);
1009                         ret = -1;
1010                         goto out;
1011                 }
1012         }
1013
1014         if (!type->set_flag)
1015                 type->set_flag = &dummy_set_flag;
1016         if (!type->flags)
1017                 type->flags = &dummy_tracer_flags;
1018         else
1019                 if (!type->flags->opts)
1020                         type->flags->opts = dummy_tracer_opt;
1021         if (!type->wait_pipe)
1022                 type->wait_pipe = default_wait_pipe;
1023
1024         ret = run_tracer_selftest(type);
1025         if (ret < 0)
1026                 goto out;
1027
1028         type->next = trace_types;
1029         trace_types = type;
1030
1031  out:
1032         tracing_selftest_running = false;
1033         mutex_unlock(&trace_types_lock);
1034
1035         if (ret || !default_bootup_tracer)
1036                 goto out_unlock;
1037
1038         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1039                 goto out_unlock;
1040
1041         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1042         /* Do we want this tracer to start on bootup? */
1043         tracing_set_tracer(type->name);
1044         default_bootup_tracer = NULL;
1045         /* disable other selftests, since this will break it. */
1046         tracing_selftest_disabled = true;
1047 #ifdef CONFIG_FTRACE_STARTUP_TEST
1048         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1049                type->name);
1050 #endif
1051
1052  out_unlock:
1053         return ret;
1054 }
1055
1056 void tracing_reset(struct trace_buffer *buf, int cpu)
1057 {
1058         struct ring_buffer *buffer = buf->buffer;
1059
1060         if (!buffer)
1061                 return;
1062
1063         ring_buffer_record_disable(buffer);
1064
1065         /* Make sure all commits have finished */
1066         synchronize_sched();
1067         ring_buffer_reset_cpu(buffer, cpu);
1068
1069         ring_buffer_record_enable(buffer);
1070 }
1071
1072 void tracing_reset_online_cpus(struct trace_buffer *buf)
1073 {
1074         struct ring_buffer *buffer = buf->buffer;
1075         int cpu;
1076
1077         if (!buffer)
1078                 return;
1079
1080         ring_buffer_record_disable(buffer);
1081
1082         /* Make sure all commits have finished */
1083         synchronize_sched();
1084
1085         buf->time_start = ftrace_now(buf->cpu);
1086
1087         for_each_online_cpu(cpu)
1088                 ring_buffer_reset_cpu(buffer, cpu);
1089
1090         ring_buffer_record_enable(buffer);
1091 }
1092
1093 void tracing_reset_current(int cpu)
1094 {
1095         tracing_reset(&global_trace.trace_buffer, cpu);
1096 }
1097
1098 void tracing_reset_all_online_cpus(void)
1099 {
1100         struct trace_array *tr;
1101
1102         mutex_lock(&trace_types_lock);
1103         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1104                 tracing_reset_online_cpus(&tr->trace_buffer);
1105 #ifdef CONFIG_TRACER_MAX_TRACE
1106                 tracing_reset_online_cpus(&tr->max_buffer);
1107 #endif
1108         }
1109         mutex_unlock(&trace_types_lock);
1110 }
1111
1112 #define SAVED_CMDLINES 128
1113 #define NO_CMDLINE_MAP UINT_MAX
1114 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1115 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1116 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1117 static int cmdline_idx;
1118 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1119
1120 /* temporary disable recording */
1121 static atomic_t trace_record_cmdline_disabled __read_mostly;
1122
1123 static void trace_init_cmdlines(void)
1124 {
1125         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1126         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
1127         cmdline_idx = 0;
1128 }
1129
1130 int is_tracing_stopped(void)
1131 {
1132         return global_trace.stop_count;
1133 }
1134
1135 /**
1136  * ftrace_off_permanent - disable all ftrace code permanently
1137  *
1138  * This should only be called when a serious anomally has
1139  * been detected.  This will turn off the function tracing,
1140  * ring buffers, and other tracing utilites. It takes no
1141  * locks and can be called from any context.
1142  */
1143 void ftrace_off_permanent(void)
1144 {
1145         tracing_disabled = 1;
1146         ftrace_stop();
1147         tracing_off_permanent();
1148 }
1149
1150 /**
1151  * tracing_start - quick start of the tracer
1152  *
1153  * If tracing is enabled but was stopped by tracing_stop,
1154  * this will start the tracer back up.
1155  */
1156 void tracing_start(void)
1157 {
1158         struct ring_buffer *buffer;
1159         unsigned long flags;
1160
1161         if (tracing_disabled)
1162                 return;
1163
1164         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1165         if (--global_trace.stop_count) {
1166                 if (global_trace.stop_count < 0) {
1167                         /* Someone screwed up their debugging */
1168                         WARN_ON_ONCE(1);
1169                         global_trace.stop_count = 0;
1170                 }
1171                 goto out;
1172         }
1173
1174         /* Prevent the buffers from switching */
1175         arch_spin_lock(&ftrace_max_lock);
1176
1177         buffer = global_trace.trace_buffer.buffer;
1178         if (buffer)
1179                 ring_buffer_record_enable(buffer);
1180
1181 #ifdef CONFIG_TRACER_MAX_TRACE
1182         buffer = global_trace.max_buffer.buffer;
1183         if (buffer)
1184                 ring_buffer_record_enable(buffer);
1185 #endif
1186
1187         arch_spin_unlock(&ftrace_max_lock);
1188
1189         ftrace_start();
1190  out:
1191         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1192 }
1193
1194 static void tracing_start_tr(struct trace_array *tr)
1195 {
1196         struct ring_buffer *buffer;
1197         unsigned long flags;
1198
1199         if (tracing_disabled)
1200                 return;
1201
1202         /* If global, we need to also start the max tracer */
1203         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1204                 return tracing_start();
1205
1206         raw_spin_lock_irqsave(&tr->start_lock, flags);
1207
1208         if (--tr->stop_count) {
1209                 if (tr->stop_count < 0) {
1210                         /* Someone screwed up their debugging */
1211                         WARN_ON_ONCE(1);
1212                         tr->stop_count = 0;
1213                 }
1214                 goto out;
1215         }
1216
1217         buffer = tr->trace_buffer.buffer;
1218         if (buffer)
1219                 ring_buffer_record_enable(buffer);
1220
1221  out:
1222         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1223 }
1224
1225 /**
1226  * tracing_stop - quick stop of the tracer
1227  *
1228  * Light weight way to stop tracing. Use in conjunction with
1229  * tracing_start.
1230  */
1231 void tracing_stop(void)
1232 {
1233         struct ring_buffer *buffer;
1234         unsigned long flags;
1235
1236         ftrace_stop();
1237         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1238         if (global_trace.stop_count++)
1239                 goto out;
1240
1241         /* Prevent the buffers from switching */
1242         arch_spin_lock(&ftrace_max_lock);
1243
1244         buffer = global_trace.trace_buffer.buffer;
1245         if (buffer)
1246                 ring_buffer_record_disable(buffer);
1247
1248 #ifdef CONFIG_TRACER_MAX_TRACE
1249         buffer = global_trace.max_buffer.buffer;
1250         if (buffer)
1251                 ring_buffer_record_disable(buffer);
1252 #endif
1253
1254         arch_spin_unlock(&ftrace_max_lock);
1255
1256  out:
1257         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1258 }
1259
1260 static void tracing_stop_tr(struct trace_array *tr)
1261 {
1262         struct ring_buffer *buffer;
1263         unsigned long flags;
1264
1265         /* If global, we need to also stop the max tracer */
1266         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1267                 return tracing_stop();
1268
1269         raw_spin_lock_irqsave(&tr->start_lock, flags);
1270         if (tr->stop_count++)
1271                 goto out;
1272
1273         buffer = tr->trace_buffer.buffer;
1274         if (buffer)
1275                 ring_buffer_record_disable(buffer);
1276
1277  out:
1278         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1279 }
1280
1281 void trace_stop_cmdline_recording(void);
1282
1283 static void trace_save_cmdline(struct task_struct *tsk)
1284 {
1285         unsigned pid, idx;
1286
1287         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1288                 return;
1289
1290         /*
1291          * It's not the end of the world if we don't get
1292          * the lock, but we also don't want to spin
1293          * nor do we want to disable interrupts,
1294          * so if we miss here, then better luck next time.
1295          */
1296         if (!arch_spin_trylock(&trace_cmdline_lock))
1297                 return;
1298
1299         idx = map_pid_to_cmdline[tsk->pid];
1300         if (idx == NO_CMDLINE_MAP) {
1301                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1302
1303                 /*
1304                  * Check whether the cmdline buffer at idx has a pid
1305                  * mapped. We are going to overwrite that entry so we
1306                  * need to clear the map_pid_to_cmdline. Otherwise we
1307                  * would read the new comm for the old pid.
1308                  */
1309                 pid = map_cmdline_to_pid[idx];
1310                 if (pid != NO_CMDLINE_MAP)
1311                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1312
1313                 map_cmdline_to_pid[idx] = tsk->pid;
1314                 map_pid_to_cmdline[tsk->pid] = idx;
1315
1316                 cmdline_idx = idx;
1317         }
1318
1319         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1320
1321         arch_spin_unlock(&trace_cmdline_lock);
1322 }
1323
1324 void trace_find_cmdline(int pid, char comm[])
1325 {
1326         unsigned map;
1327
1328         if (!pid) {
1329                 strcpy(comm, "<idle>");
1330                 return;
1331         }
1332
1333         if (WARN_ON_ONCE(pid < 0)) {
1334                 strcpy(comm, "<XXX>");
1335                 return;
1336         }
1337
1338         if (pid > PID_MAX_DEFAULT) {
1339                 strcpy(comm, "<...>");
1340                 return;
1341         }
1342
1343         preempt_disable();
1344         arch_spin_lock(&trace_cmdline_lock);
1345         map = map_pid_to_cmdline[pid];
1346         if (map != NO_CMDLINE_MAP)
1347                 strcpy(comm, saved_cmdlines[map]);
1348         else
1349                 strcpy(comm, "<...>");
1350
1351         arch_spin_unlock(&trace_cmdline_lock);
1352         preempt_enable();
1353 }
1354
1355 void tracing_record_cmdline(struct task_struct *tsk)
1356 {
1357         if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1358                 return;
1359
1360         if (!__this_cpu_read(trace_cmdline_save))
1361                 return;
1362
1363         __this_cpu_write(trace_cmdline_save, false);
1364
1365         trace_save_cmdline(tsk);
1366 }
1367
1368 void
1369 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1370                              int pc)
1371 {
1372         struct task_struct *tsk = current;
1373
1374         entry->preempt_count            = pc & 0xff;
1375         entry->pid                      = (tsk) ? tsk->pid : 0;
1376         entry->flags =
1377 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1378                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1379 #else
1380                 TRACE_FLAG_IRQS_NOSUPPORT |
1381 #endif
1382                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1383                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1384                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1385 }
1386 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1387
1388 struct ring_buffer_event *
1389 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1390                           int type,
1391                           unsigned long len,
1392                           unsigned long flags, int pc)
1393 {
1394         struct ring_buffer_event *event;
1395
1396         event = ring_buffer_lock_reserve(buffer, len);
1397         if (event != NULL) {
1398                 struct trace_entry *ent = ring_buffer_event_data(event);
1399
1400                 tracing_generic_entry_update(ent, flags, pc);
1401                 ent->type = type;
1402         }
1403
1404         return event;
1405 }
1406
1407 void
1408 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1409 {
1410         __this_cpu_write(trace_cmdline_save, true);
1411         ring_buffer_unlock_commit(buffer, event);
1412 }
1413
1414 static inline void
1415 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1416                              struct ring_buffer_event *event,
1417                              unsigned long flags, int pc)
1418 {
1419         __buffer_unlock_commit(buffer, event);
1420
1421         ftrace_trace_stack(buffer, flags, 6, pc);
1422         ftrace_trace_userstack(buffer, flags, pc);
1423 }
1424
1425 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1426                                 struct ring_buffer_event *event,
1427                                 unsigned long flags, int pc)
1428 {
1429         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1430 }
1431 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1432
1433 struct ring_buffer_event *
1434 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1435                           struct ftrace_event_file *ftrace_file,
1436                           int type, unsigned long len,
1437                           unsigned long flags, int pc)
1438 {
1439         *current_rb = ftrace_file->tr->trace_buffer.buffer;
1440         return trace_buffer_lock_reserve(*current_rb,
1441                                          type, len, flags, pc);
1442 }
1443 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1444
1445 struct ring_buffer_event *
1446 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1447                                   int type, unsigned long len,
1448                                   unsigned long flags, int pc)
1449 {
1450         *current_rb = global_trace.trace_buffer.buffer;
1451         return trace_buffer_lock_reserve(*current_rb,
1452                                          type, len, flags, pc);
1453 }
1454 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1455
1456 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1457                                         struct ring_buffer_event *event,
1458                                         unsigned long flags, int pc)
1459 {
1460         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1461 }
1462 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1463
1464 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1465                                      struct ring_buffer_event *event,
1466                                      unsigned long flags, int pc,
1467                                      struct pt_regs *regs)
1468 {
1469         __buffer_unlock_commit(buffer, event);
1470
1471         ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1472         ftrace_trace_userstack(buffer, flags, pc);
1473 }
1474 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1475
1476 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1477                                          struct ring_buffer_event *event)
1478 {
1479         ring_buffer_discard_commit(buffer, event);
1480 }
1481 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1482
1483 void
1484 trace_function(struct trace_array *tr,
1485                unsigned long ip, unsigned long parent_ip, unsigned long flags,
1486                int pc)
1487 {
1488         struct ftrace_event_call *call = &event_function;
1489         struct ring_buffer *buffer = tr->trace_buffer.buffer;
1490         struct ring_buffer_event *event;
1491         struct ftrace_entry *entry;
1492
1493         /* If we are reading the ring buffer, don't trace */
1494         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1495                 return;
1496
1497         event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1498                                           flags, pc);
1499         if (!event)
1500                 return;
1501         entry   = ring_buffer_event_data(event);
1502         entry->ip                       = ip;
1503         entry->parent_ip                = parent_ip;
1504
1505         if (!filter_check_discard(call, entry, buffer, event))
1506                 __buffer_unlock_commit(buffer, event);
1507 }
1508
1509 void
1510 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1511        unsigned long ip, unsigned long parent_ip, unsigned long flags,
1512        int pc)
1513 {
1514         if (likely(!atomic_read(&data->disabled)))
1515                 trace_function(tr, ip, parent_ip, flags, pc);
1516 }
1517
1518 #ifdef CONFIG_STACKTRACE
1519
1520 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1521 struct ftrace_stack {
1522         unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
1523 };
1524
1525 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1526 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1527
1528 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1529                                  unsigned long flags,
1530                                  int skip, int pc, struct pt_regs *regs)
1531 {
1532         struct ftrace_event_call *call = &event_kernel_stack;
1533         struct ring_buffer_event *event;
1534         struct stack_entry *entry;
1535         struct stack_trace trace;
1536         int use_stack;
1537         int size = FTRACE_STACK_ENTRIES;
1538
1539         trace.nr_entries        = 0;
1540         trace.skip              = skip;
1541
1542         /*
1543          * Since events can happen in NMIs there's no safe way to
1544          * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1545          * or NMI comes in, it will just have to use the default
1546          * FTRACE_STACK_SIZE.
1547          */
1548         preempt_disable_notrace();
1549
1550         use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1551         /*
1552          * We don't need any atomic variables, just a barrier.
1553          * If an interrupt comes in, we don't care, because it would
1554          * have exited and put the counter back to what we want.
1555          * We just need a barrier to keep gcc from moving things
1556          * around.
1557          */
1558         barrier();
1559         if (use_stack == 1) {
1560                 trace.entries           = &__get_cpu_var(ftrace_stack).calls[0];
1561                 trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
1562
1563                 if (regs)
1564                         save_stack_trace_regs(regs, &trace);
1565                 else
1566                         save_stack_trace(&trace);
1567
1568                 if (trace.nr_entries > size)
1569                         size = trace.nr_entries;
1570         } else
1571                 /* From now on, use_stack is a boolean */
1572                 use_stack = 0;
1573
1574         size *= sizeof(unsigned long);
1575
1576         event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1577                                           sizeof(*entry) + size, flags, pc);
1578         if (!event)
1579                 goto out;
1580         entry = ring_buffer_event_data(event);
1581
1582         memset(&entry->caller, 0, size);
1583
1584         if (use_stack)
1585                 memcpy(&entry->caller, trace.entries,
1586                        trace.nr_entries * sizeof(unsigned long));
1587         else {
1588                 trace.max_entries       = FTRACE_STACK_ENTRIES;
1589                 trace.entries           = entry->caller;
1590                 if (regs)
1591                         save_stack_trace_regs(regs, &trace);
1592                 else
1593                         save_stack_trace(&trace);
1594         }
1595
1596         entry->size = trace.nr_entries;
1597
1598         if (!filter_check_discard(call, entry, buffer, event))
1599                 __buffer_unlock_commit(buffer, event);
1600
1601  out:
1602         /* Again, don't let gcc optimize things here */
1603         barrier();
1604         __this_cpu_dec(ftrace_stack_reserve);
1605         preempt_enable_notrace();
1606
1607 }
1608
1609 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1610                              int skip, int pc, struct pt_regs *regs)
1611 {
1612         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1613                 return;
1614
1615         __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1616 }
1617
1618 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1619                         int skip, int pc)
1620 {
1621         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1622                 return;
1623
1624         __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1625 }
1626
1627 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1628                    int pc)
1629 {
1630         __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1631 }
1632
1633 /**
1634  * trace_dump_stack - record a stack back trace in the trace buffer
1635  */
1636 void trace_dump_stack(void)
1637 {
1638         unsigned long flags;
1639
1640         if (tracing_disabled || tracing_selftest_running)
1641                 return;
1642
1643         local_save_flags(flags);
1644
1645         /* skipping 3 traces, seems to get us at the caller of this function */
1646         __ftrace_trace_stack(global_trace.trace_buffer.buffer, flags, 3,
1647                              preempt_count(), NULL);
1648 }
1649
1650 static DEFINE_PER_CPU(int, user_stack_count);
1651
1652 void
1653 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1654 {
1655         struct ftrace_event_call *call = &event_user_stack;
1656         struct ring_buffer_event *event;
1657         struct userstack_entry *entry;
1658         struct stack_trace trace;
1659
1660         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1661                 return;
1662
1663         /*
1664          * NMIs can not handle page faults, even with fix ups.
1665          * The save user stack can (and often does) fault.
1666          */
1667         if (unlikely(in_nmi()))
1668                 return;
1669
1670         /*
1671          * prevent recursion, since the user stack tracing may
1672          * trigger other kernel events.
1673          */
1674         preempt_disable();
1675         if (__this_cpu_read(user_stack_count))
1676                 goto out;
1677
1678         __this_cpu_inc(user_stack_count);
1679
1680         event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1681                                           sizeof(*entry), flags, pc);
1682         if (!event)
1683                 goto out_drop_count;
1684         entry   = ring_buffer_event_data(event);
1685
1686         entry->tgid             = current->tgid;
1687         memset(&entry->caller, 0, sizeof(entry->caller));
1688
1689         trace.nr_entries        = 0;
1690         trace.max_entries       = FTRACE_STACK_ENTRIES;
1691         trace.skip              = 0;
1692         trace.entries           = entry->caller;
1693
1694         save_stack_trace_user(&trace);
1695         if (!filter_check_discard(call, entry, buffer, event))
1696                 __buffer_unlock_commit(buffer, event);
1697
1698  out_drop_count:
1699         __this_cpu_dec(user_stack_count);
1700  out:
1701         preempt_enable();
1702 }
1703
1704 #ifdef UNUSED
1705 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1706 {
1707         ftrace_trace_userstack(tr, flags, preempt_count());
1708 }
1709 #endif /* UNUSED */
1710
1711 #endif /* CONFIG_STACKTRACE */
1712
1713 /* created for use with alloc_percpu */
1714 struct trace_buffer_struct {
1715         char buffer[TRACE_BUF_SIZE];
1716 };
1717
1718 static struct trace_buffer_struct *trace_percpu_buffer;
1719 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1720 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1721 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1722
1723 /*
1724  * The buffer used is dependent on the context. There is a per cpu
1725  * buffer for normal context, softirq contex, hard irq context and
1726  * for NMI context. Thise allows for lockless recording.
1727  *
1728  * Note, if the buffers failed to be allocated, then this returns NULL
1729  */
1730 static char *get_trace_buf(void)
1731 {
1732         struct trace_buffer_struct *percpu_buffer;
1733
1734         /*
1735          * If we have allocated per cpu buffers, then we do not
1736          * need to do any locking.
1737          */
1738         if (in_nmi())
1739                 percpu_buffer = trace_percpu_nmi_buffer;
1740         else if (in_irq())
1741                 percpu_buffer = trace_percpu_irq_buffer;
1742         else if (in_softirq())
1743                 percpu_buffer = trace_percpu_sirq_buffer;
1744         else
1745                 percpu_buffer = trace_percpu_buffer;
1746
1747         if (!percpu_buffer)
1748                 return NULL;
1749
1750         return this_cpu_ptr(&percpu_buffer->buffer[0]);
1751 }
1752
1753 static int alloc_percpu_trace_buffer(void)
1754 {
1755         struct trace_buffer_struct *buffers;
1756         struct trace_buffer_struct *sirq_buffers;
1757         struct trace_buffer_struct *irq_buffers;
1758         struct trace_buffer_struct *nmi_buffers;
1759
1760         buffers = alloc_percpu(struct trace_buffer_struct);
1761         if (!buffers)
1762                 goto err_warn;
1763
1764         sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1765         if (!sirq_buffers)
1766                 goto err_sirq;
1767
1768         irq_buffers = alloc_percpu(struct trace_buffer_struct);
1769         if (!irq_buffers)
1770                 goto err_irq;
1771
1772         nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1773         if (!nmi_buffers)
1774                 goto err_nmi;
1775
1776         trace_percpu_buffer = buffers;
1777         trace_percpu_sirq_buffer = sirq_buffers;
1778         trace_percpu_irq_buffer = irq_buffers;
1779         trace_percpu_nmi_buffer = nmi_buffers;
1780
1781         return 0;
1782
1783  err_nmi:
1784         free_percpu(irq_buffers);
1785  err_irq:
1786         free_percpu(sirq_buffers);
1787  err_sirq:
1788         free_percpu(buffers);
1789  err_warn:
1790         WARN(1, "Could not allocate percpu trace_printk buffer");
1791         return -ENOMEM;
1792 }
1793
1794 static int buffers_allocated;
1795
1796 void trace_printk_init_buffers(void)
1797 {
1798         if (buffers_allocated)
1799                 return;
1800
1801         if (alloc_percpu_trace_buffer())
1802                 return;
1803
1804         pr_info("ftrace: Allocated trace_printk buffers\n");
1805
1806         /* Expand the buffers to set size */
1807         tracing_update_buffers();
1808
1809         buffers_allocated = 1;
1810
1811         /*
1812          * trace_printk_init_buffers() can be called by modules.
1813          * If that happens, then we need to start cmdline recording
1814          * directly here. If the global_trace.buffer is already
1815          * allocated here, then this was called by module code.
1816          */
1817         if (global_trace.trace_buffer.buffer)
1818                 tracing_start_cmdline_record();
1819 }
1820
1821 void trace_printk_start_comm(void)
1822 {
1823         /* Start tracing comms if trace printk is set */
1824         if (!buffers_allocated)
1825                 return;
1826         tracing_start_cmdline_record();
1827 }
1828
1829 static void trace_printk_start_stop_comm(int enabled)
1830 {
1831         if (!buffers_allocated)
1832                 return;
1833
1834         if (enabled)
1835                 tracing_start_cmdline_record();
1836         else
1837                 tracing_stop_cmdline_record();
1838 }
1839
1840 /**
1841  * trace_vbprintk - write binary msg to tracing buffer
1842  *
1843  */
1844 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1845 {
1846         struct ftrace_event_call *call = &event_bprint;
1847         struct ring_buffer_event *event;
1848         struct ring_buffer *buffer;
1849         struct trace_array *tr = &global_trace;
1850         struct bprint_entry *entry;
1851         unsigned long flags;
1852         char *tbuffer;
1853         int len = 0, size, pc;
1854
1855         if (unlikely(tracing_selftest_running || tracing_disabled))
1856                 return 0;
1857
1858         /* Don't pollute graph traces with trace_vprintk internals */
1859         pause_graph_tracing();
1860
1861         pc = preempt_count();
1862         preempt_disable_notrace();
1863
1864         tbuffer = get_trace_buf();
1865         if (!tbuffer) {
1866                 len = 0;
1867                 goto out;
1868         }
1869
1870         len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1871
1872         if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1873                 goto out;
1874
1875         local_save_flags(flags);
1876         size = sizeof(*entry) + sizeof(u32) * len;
1877         buffer = tr->trace_buffer.buffer;
1878         event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1879                                           flags, pc);
1880         if (!event)
1881                 goto out;
1882         entry = ring_buffer_event_data(event);
1883         entry->ip                       = ip;
1884         entry->fmt                      = fmt;
1885
1886         memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1887         if (!filter_check_discard(call, entry, buffer, event)) {
1888                 __buffer_unlock_commit(buffer, event);
1889                 ftrace_trace_stack(buffer, flags, 6, pc);
1890         }
1891
1892 out:
1893         preempt_enable_notrace();
1894         unpause_graph_tracing();
1895
1896         return len;
1897 }
1898 EXPORT_SYMBOL_GPL(trace_vbprintk);
1899
1900 static int
1901 __trace_array_vprintk(struct ring_buffer *buffer,
1902                       unsigned long ip, const char *fmt, va_list args)
1903 {
1904         struct ftrace_event_call *call = &event_print;
1905         struct ring_buffer_event *event;
1906         int len = 0, size, pc;
1907         struct print_entry *entry;
1908         unsigned long flags;
1909         char *tbuffer;
1910
1911         if (tracing_disabled || tracing_selftest_running)
1912                 return 0;
1913
1914         /* Don't pollute graph traces with trace_vprintk internals */
1915         pause_graph_tracing();
1916
1917         pc = preempt_count();
1918         preempt_disable_notrace();
1919
1920
1921         tbuffer = get_trace_buf();
1922         if (!tbuffer) {
1923                 len = 0;
1924                 goto out;
1925         }
1926
1927         len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1928         if (len > TRACE_BUF_SIZE)
1929                 goto out;
1930
1931         local_save_flags(flags);
1932         size = sizeof(*entry) + len + 1;
1933         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1934                                           flags, pc);
1935         if (!event)
1936                 goto out;
1937         entry = ring_buffer_event_data(event);
1938         entry->ip = ip;
1939
1940         memcpy(&entry->buf, tbuffer, len);
1941         entry->buf[len] = '\0';
1942         if (!filter_check_discard(call, entry, buffer, event)) {
1943                 __buffer_unlock_commit(buffer, event);
1944                 ftrace_trace_stack(buffer, flags, 6, pc);
1945         }
1946  out:
1947         preempt_enable_notrace();
1948         unpause_graph_tracing();
1949
1950         return len;
1951 }
1952
1953 int trace_array_vprintk(struct trace_array *tr,
1954                         unsigned long ip, const char *fmt, va_list args)
1955 {
1956         return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
1957 }
1958
1959 int trace_array_printk(struct trace_array *tr,
1960                        unsigned long ip, const char *fmt, ...)
1961 {
1962         int ret;
1963         va_list ap;
1964
1965         if (!(trace_flags & TRACE_ITER_PRINTK))
1966                 return 0;
1967
1968         va_start(ap, fmt);
1969         ret = trace_array_vprintk(tr, ip, fmt, ap);
1970         va_end(ap);
1971         return ret;
1972 }
1973
1974 int trace_array_printk_buf(struct ring_buffer *buffer,
1975                            unsigned long ip, const char *fmt, ...)
1976 {
1977         int ret;
1978         va_list ap;
1979
1980         if (!(trace_flags & TRACE_ITER_PRINTK))
1981                 return 0;
1982
1983         va_start(ap, fmt);
1984         ret = __trace_array_vprintk(buffer, ip, fmt, ap);
1985         va_end(ap);
1986         return ret;
1987 }
1988
1989 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1990 {
1991         return trace_array_vprintk(&global_trace, ip, fmt, args);
1992 }
1993 EXPORT_SYMBOL_GPL(trace_vprintk);
1994
1995 static void trace_iterator_increment(struct trace_iterator *iter)
1996 {
1997         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
1998
1999         iter->idx++;
2000         if (buf_iter)
2001                 ring_buffer_read(buf_iter, NULL);
2002 }
2003
2004 static struct trace_entry *
2005 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2006                 unsigned long *lost_events)
2007 {
2008         struct ring_buffer_event *event;
2009         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2010
2011         if (buf_iter)
2012                 event = ring_buffer_iter_peek(buf_iter, ts);
2013         else
2014                 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2015                                          lost_events);
2016
2017         if (event) {
2018                 iter->ent_size = ring_buffer_event_length(event);
2019                 return ring_buffer_event_data(event);
2020         }
2021         iter->ent_size = 0;
2022         return NULL;
2023 }
2024
2025 static struct trace_entry *
2026 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2027                   unsigned long *missing_events, u64 *ent_ts)
2028 {
2029         struct ring_buffer *buffer = iter->trace_buffer->buffer;
2030         struct trace_entry *ent, *next = NULL;
2031         unsigned long lost_events = 0, next_lost = 0;
2032         int cpu_file = iter->cpu_file;
2033         u64 next_ts = 0, ts;
2034         int next_cpu = -1;
2035         int next_size = 0;
2036         int cpu;
2037
2038         /*
2039          * If we are in a per_cpu trace file, don't bother by iterating over
2040          * all cpu and peek directly.
2041          */
2042         if (cpu_file > RING_BUFFER_ALL_CPUS) {
2043                 if (ring_buffer_empty_cpu(buffer, cpu_file))
2044                         return NULL;
2045                 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2046                 if (ent_cpu)
2047                         *ent_cpu = cpu_file;
2048
2049                 return ent;
2050         }
2051
2052         for_each_tracing_cpu(cpu) {
2053
2054                 if (ring_buffer_empty_cpu(buffer, cpu))
2055                         continue;
2056
2057                 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2058
2059                 /*
2060                  * Pick the entry with the smallest timestamp:
2061                  */
2062                 if (ent && (!next || ts < next_ts)) {
2063                         next = ent;
2064                         next_cpu = cpu;
2065                         next_ts = ts;
2066                         next_lost = lost_events;
2067                         next_size = iter->ent_size;
2068                 }
2069         }
2070
2071         iter->ent_size = next_size;
2072
2073         if (ent_cpu)
2074                 *ent_cpu = next_cpu;
2075
2076         if (ent_ts)
2077                 *ent_ts = next_ts;
2078
2079         if (missing_events)
2080                 *missing_events = next_lost;
2081
2082         return next;
2083 }
2084
2085 /* Find the next real entry, without updating the iterator itself */
2086 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2087                                           int *ent_cpu, u64 *ent_ts)
2088 {
2089         return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2090 }
2091
2092 /* Find the next real entry, and increment the iterator to the next entry */
2093 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2094 {
2095         iter->ent = __find_next_entry(iter, &iter->cpu,
2096                                       &iter->lost_events, &iter->ts);
2097
2098         if (iter->ent)
2099                 trace_iterator_increment(iter);
2100
2101         return iter->ent ? iter : NULL;
2102 }
2103
2104 static void trace_consume(struct trace_iterator *iter)
2105 {
2106         ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2107                             &iter->lost_events);
2108 }
2109
2110 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2111 {
2112         struct trace_iterator *iter = m->private;
2113         int i = (int)*pos;
2114         void *ent;
2115
2116         WARN_ON_ONCE(iter->leftover);
2117
2118         (*pos)++;
2119
2120         /* can't go backwards */
2121         if (iter->idx > i)
2122                 return NULL;
2123
2124         if (iter->idx < 0)
2125                 ent = trace_find_next_entry_inc(iter);
2126         else
2127                 ent = iter;
2128
2129         while (ent && iter->idx < i)
2130                 ent = trace_find_next_entry_inc(iter);
2131
2132         iter->pos = *pos;
2133
2134         return ent;
2135 }
2136
2137 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2138 {
2139         struct ring_buffer_event *event;
2140         struct ring_buffer_iter *buf_iter;
2141         unsigned long entries = 0;
2142         u64 ts;
2143
2144         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2145
2146         buf_iter = trace_buffer_iter(iter, cpu);
2147         if (!buf_iter)
2148                 return;
2149
2150         ring_buffer_iter_reset(buf_iter);
2151
2152         /*
2153          * We could have the case with the max latency tracers
2154          * that a reset never took place on a cpu. This is evident
2155          * by the timestamp being before the start of the buffer.
2156          */
2157         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2158                 if (ts >= iter->trace_buffer->time_start)
2159                         break;
2160                 entries++;
2161                 ring_buffer_read(buf_iter, NULL);
2162         }
2163
2164         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2165 }
2166
2167 /*
2168  * The current tracer is copied to avoid a global locking
2169  * all around.
2170  */
2171 static void *s_start(struct seq_file *m, loff_t *pos)
2172 {
2173         struct trace_iterator *iter = m->private;
2174         struct trace_array *tr = iter->tr;
2175         int cpu_file = iter->cpu_file;
2176         void *p = NULL;
2177         loff_t l = 0;
2178         int cpu;
2179
2180         /*
2181          * copy the tracer to avoid using a global lock all around.
2182          * iter->trace is a copy of current_trace, the pointer to the
2183          * name may be used instead of a strcmp(), as iter->trace->name
2184          * will point to the same string as current_trace->name.
2185          */
2186         mutex_lock(&trace_types_lock);
2187         if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2188                 *iter->trace = *tr->current_trace;
2189         mutex_unlock(&trace_types_lock);
2190
2191 #ifdef CONFIG_TRACER_MAX_TRACE
2192         if (iter->snapshot && iter->trace->use_max_tr)
2193                 return ERR_PTR(-EBUSY);
2194 #endif
2195
2196         if (!iter->snapshot)
2197                 atomic_inc(&trace_record_cmdline_disabled);
2198
2199         if (*pos != iter->pos) {
2200                 iter->ent = NULL;
2201                 iter->cpu = 0;
2202                 iter->idx = -1;
2203
2204                 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2205                         for_each_tracing_cpu(cpu)
2206                                 tracing_iter_reset(iter, cpu);
2207                 } else
2208                         tracing_iter_reset(iter, cpu_file);
2209
2210                 iter->leftover = 0;
2211                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2212                         ;
2213
2214         } else {
2215                 /*
2216                  * If we overflowed the seq_file before, then we want
2217                  * to just reuse the trace_seq buffer again.
2218                  */
2219                 if (iter->leftover)
2220                         p = iter;
2221                 else {
2222                         l = *pos - 1;
2223                         p = s_next(m, p, &l);
2224                 }
2225         }
2226
2227         trace_event_read_lock();
2228         trace_access_lock(cpu_file);
2229         return p;
2230 }
2231
2232 static void s_stop(struct seq_file *m, void *p)
2233 {
2234         struct trace_iterator *iter = m->private;
2235
2236 #ifdef CONFIG_TRACER_MAX_TRACE
2237         if (iter->snapshot && iter->trace->use_max_tr)
2238                 return;
2239 #endif
2240
2241         if (!iter->snapshot)
2242                 atomic_dec(&trace_record_cmdline_disabled);
2243
2244         trace_access_unlock(iter->cpu_file);
2245         trace_event_read_unlock();
2246 }
2247
2248 static void
2249 get_total_entries(struct trace_buffer *buf,
2250                   unsigned long *total, unsigned long *entries)
2251 {
2252         unsigned long count;
2253         int cpu;
2254
2255         *total = 0;
2256         *entries = 0;
2257
2258         for_each_tracing_cpu(cpu) {
2259                 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2260                 /*
2261                  * If this buffer has skipped entries, then we hold all
2262                  * entries for the trace and we need to ignore the
2263                  * ones before the time stamp.
2264                  */
2265                 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2266                         count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2267                         /* total is the same as the entries */
2268                         *total += count;
2269                 } else
2270                         *total += count +
2271                                 ring_buffer_overrun_cpu(buf->buffer, cpu);
2272                 *entries += count;
2273         }
2274 }
2275
2276 static void print_lat_help_header(struct seq_file *m)
2277 {
2278         seq_puts(m, "#                  _------=> CPU#            \n");
2279         seq_puts(m, "#                 / _-----=> irqs-off        \n");
2280         seq_puts(m, "#                | / _----=> need-resched    \n");
2281         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
2282         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
2283         seq_puts(m, "#                |||| /     delay             \n");
2284         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
2285         seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
2286 }
2287
2288 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2289 {
2290         unsigned long total;
2291         unsigned long entries;
2292
2293         get_total_entries(buf, &total, &entries);
2294         seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
2295                    entries, total, num_online_cpus());
2296         seq_puts(m, "#\n");
2297 }
2298
2299 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2300 {
2301         print_event_info(buf, m);
2302         seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2303         seq_puts(m, "#              | |       |          |         |\n");
2304 }
2305
2306 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2307 {
2308         print_event_info(buf, m);
2309         seq_puts(m, "#                              _-----=> irqs-off\n");
2310         seq_puts(m, "#                             / _----=> need-resched\n");
2311         seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
2312         seq_puts(m, "#                            || / _--=> preempt-depth\n");
2313         seq_puts(m, "#                            ||| /     delay\n");
2314         seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
2315         seq_puts(m, "#              | |       |   ||||       |         |\n");
2316 }
2317
2318 void
2319 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2320 {
2321         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2322         struct trace_buffer *buf = iter->trace_buffer;
2323         struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2324         struct tracer *type = iter->trace;
2325         unsigned long entries;
2326         unsigned long total;
2327         const char *name = "preemption";
2328
2329         name = type->name;
2330
2331         get_total_entries(buf, &total, &entries);
2332
2333         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2334                    name, UTS_RELEASE);
2335         seq_puts(m, "# -----------------------------------"
2336                  "---------------------------------\n");
2337         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2338                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2339                    nsecs_to_usecs(data->saved_latency),
2340                    entries,
2341                    total,
2342                    buf->cpu,
2343 #if defined(CONFIG_PREEMPT_NONE)
2344                    "server",
2345 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2346                    "desktop",
2347 #elif defined(CONFIG_PREEMPT)
2348                    "preempt",
2349 #else
2350                    "unknown",
2351 #endif
2352                    /* These are reserved for later use */
2353                    0, 0, 0, 0);
2354 #ifdef CONFIG_SMP
2355         seq_printf(m, " #P:%d)\n", num_online_cpus());
2356 #else
2357         seq_puts(m, ")\n");
2358 #endif
2359         seq_puts(m, "#    -----------------\n");
2360         seq_printf(m, "#    | task: %.16s-%d "
2361                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2362                    data->comm, data->pid,
2363                    from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2364                    data->policy, data->rt_priority);
2365         seq_puts(m, "#    -----------------\n");
2366
2367         if (data->critical_start) {
2368                 seq_puts(m, "#  => started at: ");
2369                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2370                 trace_print_seq(m, &iter->seq);
2371                 seq_puts(m, "\n#  => ended at:   ");
2372                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2373                 trace_print_seq(m, &iter->seq);
2374                 seq_puts(m, "\n#\n");
2375         }
2376
2377         seq_puts(m, "#\n");
2378 }
2379
2380 static void test_cpu_buff_start(struct trace_iterator *iter)
2381 {
2382         struct trace_seq *s = &iter->seq;
2383
2384         if (!(trace_flags & TRACE_ITER_ANNOTATE))
2385                 return;
2386
2387         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2388                 return;
2389
2390         if (cpumask_test_cpu(iter->cpu, iter->started))
2391                 return;
2392
2393         if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2394                 return;
2395
2396         cpumask_set_cpu(iter->cpu, iter->started);
2397
2398         /* Don't print started cpu buffer for the first entry of the trace */
2399         if (iter->idx > 1)
2400                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2401                                 iter->cpu);
2402 }
2403
2404 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2405 {
2406         struct trace_seq *s = &iter->seq;
2407         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2408         struct trace_entry *entry;
2409         struct trace_event *event;
2410
2411         entry = iter->ent;
2412
2413         test_cpu_buff_start(iter);
2414
2415         event = ftrace_find_event(entry->type);
2416
2417         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2418                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2419                         if (!trace_print_lat_context(iter))
2420                                 goto partial;
2421                 } else {
2422                         if (!trace_print_context(iter))
2423                                 goto partial;
2424                 }
2425         }
2426
2427         if (event)
2428                 return event->funcs->trace(iter, sym_flags, event);
2429
2430         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2431                 goto partial;
2432
2433         return TRACE_TYPE_HANDLED;
2434 partial:
2435         return TRACE_TYPE_PARTIAL_LINE;
2436 }
2437
2438 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2439 {
2440         struct trace_seq *s = &iter->seq;
2441         struct trace_entry *entry;
2442         struct trace_event *event;
2443
2444         entry = iter->ent;
2445
2446         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2447                 if (!trace_seq_printf(s, "%d %d %llu ",
2448                                       entry->pid, iter->cpu, iter->ts))
2449                         goto partial;
2450         }
2451
2452         event = ftrace_find_event(entry->type);
2453         if (event)
2454                 return event->funcs->raw(iter, 0, event);
2455
2456         if (!trace_seq_printf(s, "%d ?\n", entry->type))
2457                 goto partial;
2458
2459         return TRACE_TYPE_HANDLED;
2460 partial:
2461         return TRACE_TYPE_PARTIAL_LINE;
2462 }
2463
2464 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2465 {
2466         struct trace_seq *s = &iter->seq;
2467         unsigned char newline = '\n';
2468         struct trace_entry *entry;
2469         struct trace_event *event;
2470
2471         entry = iter->ent;
2472
2473         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2474                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2475                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2476                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2477         }
2478
2479         event = ftrace_find_event(entry->type);
2480         if (event) {
2481                 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2482                 if (ret != TRACE_TYPE_HANDLED)
2483                         return ret;
2484         }
2485
2486         SEQ_PUT_FIELD_RET(s, newline);
2487
2488         return TRACE_TYPE_HANDLED;
2489 }
2490
2491 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2492 {
2493         struct trace_seq *s = &iter->seq;
2494         struct trace_entry *entry;
2495         struct trace_event *event;
2496
2497         entry = iter->ent;
2498
2499         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2500                 SEQ_PUT_FIELD_RET(s, entry->pid);
2501                 SEQ_PUT_FIELD_RET(s, iter->cpu);
2502                 SEQ_PUT_FIELD_RET(s, iter->ts);
2503         }
2504
2505         event = ftrace_find_event(entry->type);
2506         return event ? event->funcs->binary(iter, 0, event) :
2507                 TRACE_TYPE_HANDLED;
2508 }
2509
2510 int trace_empty(struct trace_iterator *iter)
2511 {
2512         struct ring_buffer_iter *buf_iter;
2513         int cpu;
2514
2515         /* If we are looking at one CPU buffer, only check that one */
2516         if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2517                 cpu = iter->cpu_file;
2518                 buf_iter = trace_buffer_iter(iter, cpu);
2519                 if (buf_iter) {
2520                         if (!ring_buffer_iter_empty(buf_iter))
2521                                 return 0;
2522                 } else {
2523                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2524                                 return 0;
2525                 }
2526                 return 1;
2527         }
2528
2529         for_each_tracing_cpu(cpu) {
2530                 buf_iter = trace_buffer_iter(iter, cpu);
2531                 if (buf_iter) {
2532                         if (!ring_buffer_iter_empty(buf_iter))
2533                                 return 0;
2534                 } else {
2535                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2536                                 return 0;
2537                 }
2538         }
2539
2540         return 1;
2541 }
2542
2543 /*  Called with trace_event_read_lock() held. */
2544 enum print_line_t print_trace_line(struct trace_iterator *iter)
2545 {
2546         enum print_line_t ret;
2547
2548         if (iter->lost_events &&
2549             !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2550                                  iter->cpu, iter->lost_events))
2551                 return TRACE_TYPE_PARTIAL_LINE;
2552
2553         if (iter->trace && iter->trace->print_line) {
2554                 ret = iter->trace->print_line(iter);
2555                 if (ret != TRACE_TYPE_UNHANDLED)
2556                         return ret;
2557         }
2558
2559         if (iter->ent->type == TRACE_BPUTS &&
2560                         trace_flags & TRACE_ITER_PRINTK &&
2561                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2562                 return trace_print_bputs_msg_only(iter);
2563
2564         if (iter->ent->type == TRACE_BPRINT &&
2565                         trace_flags & TRACE_ITER_PRINTK &&
2566                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2567                 return trace_print_bprintk_msg_only(iter);
2568
2569         if (iter->ent->type == TRACE_PRINT &&
2570                         trace_flags & TRACE_ITER_PRINTK &&
2571                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2572                 return trace_print_printk_msg_only(iter);
2573
2574         if (trace_flags & TRACE_ITER_BIN)
2575                 return print_bin_fmt(iter);
2576
2577         if (trace_flags & TRACE_ITER_HEX)
2578                 return print_hex_fmt(iter);
2579
2580         if (trace_flags & TRACE_ITER_RAW)
2581                 return print_raw_fmt(iter);
2582
2583         return print_trace_fmt(iter);
2584 }
2585
2586 void trace_latency_header(struct seq_file *m)
2587 {
2588         struct trace_iterator *iter = m->private;
2589
2590         /* print nothing if the buffers are empty */
2591         if (trace_empty(iter))
2592                 return;
2593
2594         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2595                 print_trace_header(m, iter);
2596
2597         if (!(trace_flags & TRACE_ITER_VERBOSE))
2598                 print_lat_help_header(m);
2599 }
2600
2601 void trace_default_header(struct seq_file *m)
2602 {
2603         struct trace_iterator *iter = m->private;
2604
2605         if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2606                 return;
2607
2608         if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2609                 /* print nothing if the buffers are empty */
2610                 if (trace_empty(iter))
2611                         return;
2612                 print_trace_header(m, iter);
2613                 if (!(trace_flags & TRACE_ITER_VERBOSE))
2614                         print_lat_help_header(m);
2615         } else {
2616                 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2617                         if (trace_flags & TRACE_ITER_IRQ_INFO)
2618                                 print_func_help_header_irq(iter->trace_buffer, m);
2619                         else
2620                                 print_func_help_header(iter->trace_buffer, m);
2621                 }
2622         }
2623 }
2624
2625 static void test_ftrace_alive(struct seq_file *m)
2626 {
2627         if (!ftrace_is_dead())
2628                 return;
2629         seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2630         seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
2631 }
2632
2633 #ifdef CONFIG_TRACER_MAX_TRACE
2634 static void show_snapshot_main_help(struct seq_file *m)
2635 {
2636         seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2637         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2638         seq_printf(m, "#                      Takes a snapshot of the main buffer.\n");
2639         seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2640         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2641         seq_printf(m, "#                       is not a '0' or '1')\n");
2642 }
2643
2644 static void show_snapshot_percpu_help(struct seq_file *m)
2645 {
2646         seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2647 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2648         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2649         seq_printf(m, "#                      Takes a snapshot of the main buffer for this cpu.\n");
2650 #else
2651         seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2652         seq_printf(m, "#                     Must use main snapshot file to allocate.\n");
2653 #endif
2654         seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2655         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2656         seq_printf(m, "#                       is not a '0' or '1')\n");
2657 }
2658
2659 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2660 {
2661         if (iter->tr->allocated_snapshot)
2662                 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2663         else
2664                 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2665
2666         seq_printf(m, "# Snapshot commands:\n");
2667         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2668                 show_snapshot_main_help(m);
2669         else
2670                 show_snapshot_percpu_help(m);
2671 }
2672 #else
2673 /* Should never be called */
2674 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2675 #endif
2676
2677 static int s_show(struct seq_file *m, void *v)
2678 {
2679         struct trace_iterator *iter = v;
2680         int ret;
2681
2682         if (iter->ent == NULL) {
2683                 if (iter->tr) {
2684                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
2685                         seq_puts(m, "#\n");
2686                         test_ftrace_alive(m);
2687                 }
2688                 if (iter->snapshot && trace_empty(iter))
2689                         print_snapshot_help(m, iter);
2690                 else if (iter->trace && iter->trace->print_header)
2691                         iter->trace->print_header(m);
2692                 else
2693                         trace_default_header(m);
2694
2695         } else if (iter->leftover) {
2696                 /*
2697                  * If we filled the seq_file buffer earlier, we
2698                  * want to just show it now.
2699                  */
2700                 ret = trace_print_seq(m, &iter->seq);
2701
2702                 /* ret should this time be zero, but you never know */
2703                 iter->leftover = ret;
2704
2705         } else {
2706                 print_trace_line(iter);
2707                 ret = trace_print_seq(m, &iter->seq);
2708                 /*
2709                  * If we overflow the seq_file buffer, then it will
2710                  * ask us for this data again at start up.
2711                  * Use that instead.
2712                  *  ret is 0 if seq_file write succeeded.
2713                  *        -1 otherwise.
2714                  */
2715                 iter->leftover = ret;
2716         }
2717
2718         return 0;
2719 }
2720
2721 static const struct seq_operations tracer_seq_ops = {
2722         .start          = s_start,
2723         .next           = s_next,
2724         .stop           = s_stop,
2725         .show           = s_show,
2726 };
2727
2728 static struct trace_iterator *
2729 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2730 {
2731         struct trace_cpu *tc = inode->i_private;
2732         struct trace_array *tr = tc->tr;
2733         struct trace_iterator *iter;
2734         int cpu;
2735
2736         if (tracing_disabled)
2737                 return ERR_PTR(-ENODEV);
2738
2739         iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2740         if (!iter)
2741                 return ERR_PTR(-ENOMEM);
2742
2743         iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2744                                     GFP_KERNEL);
2745         if (!iter->buffer_iter)
2746                 goto release;
2747
2748         /*
2749          * We make a copy of the current tracer to avoid concurrent
2750          * changes on it while we are reading.
2751          */
2752         mutex_lock(&trace_types_lock);
2753         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2754         if (!iter->trace)
2755                 goto fail;
2756
2757         *iter->trace = *tr->current_trace;
2758
2759         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2760                 goto fail;
2761
2762         iter->tr = tr;
2763
2764 #ifdef CONFIG_TRACER_MAX_TRACE
2765         /* Currently only the top directory has a snapshot */
2766         if (tr->current_trace->print_max || snapshot)
2767                 iter->trace_buffer = &tr->max_buffer;
2768         else
2769 #endif
2770                 iter->trace_buffer = &tr->trace_buffer;
2771         iter->snapshot = snapshot;
2772         iter->pos = -1;
2773         mutex_init(&iter->mutex);
2774         iter->cpu_file = tc->cpu;
2775
2776         /* Notify the tracer early; before we stop tracing. */
2777         if (iter->trace && iter->trace->open)
2778                 iter->trace->open(iter);
2779
2780         /* Annotate start of buffers if we had overruns */
2781         if (ring_buffer_overruns(iter->trace_buffer->buffer))
2782                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2783
2784         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2785         if (trace_clocks[trace_clock_id].in_ns)
2786                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2787
2788         /* stop the trace while dumping if we are not opening "snapshot" */
2789         if (!iter->snapshot)
2790                 tracing_stop_tr(tr);
2791
2792         if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2793                 for_each_tracing_cpu(cpu) {
2794                         iter->buffer_iter[cpu] =
2795                                 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2796                 }
2797                 ring_buffer_read_prepare_sync();
2798                 for_each_tracing_cpu(cpu) {
2799                         ring_buffer_read_start(iter->buffer_iter[cpu]);
2800                         tracing_iter_reset(iter, cpu);
2801                 }
2802         } else {
2803                 cpu = iter->cpu_file;
2804                 iter->buffer_iter[cpu] =
2805                         ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2806                 ring_buffer_read_prepare_sync();
2807                 ring_buffer_read_start(iter->buffer_iter[cpu]);
2808                 tracing_iter_reset(iter, cpu);
2809         }
2810
2811         tr->ref++;
2812
2813         mutex_unlock(&trace_types_lock);
2814
2815         return iter;
2816
2817  fail:
2818         mutex_unlock(&trace_types_lock);
2819         kfree(iter->trace);
2820         kfree(iter->buffer_iter);
2821 release:
2822         seq_release_private(inode, file);
2823         return ERR_PTR(-ENOMEM);
2824 }
2825
2826 int tracing_open_generic(struct inode *inode, struct file *filp)
2827 {
2828         if (tracing_disabled)
2829                 return -ENODEV;
2830
2831         filp->private_data = inode->i_private;
2832         return 0;
2833 }
2834
2835 static int tracing_release(struct inode *inode, struct file *file)
2836 {
2837         struct seq_file *m = file->private_data;
2838         struct trace_iterator *iter;
2839         struct trace_array *tr;
2840         int cpu;
2841
2842         if (!(file->f_mode & FMODE_READ))
2843                 return 0;
2844
2845         iter = m->private;
2846         tr = iter->tr;
2847
2848         mutex_lock(&trace_types_lock);
2849
2850         WARN_ON(!tr->ref);
2851         tr->ref--;
2852
2853         for_each_tracing_cpu(cpu) {
2854                 if (iter->buffer_iter[cpu])
2855                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2856         }
2857
2858         if (iter->trace && iter->trace->close)
2859                 iter->trace->close(iter);
2860
2861         if (!iter->snapshot)
2862                 /* reenable tracing if it was previously enabled */
2863                 tracing_start_tr(tr);
2864         mutex_unlock(&trace_types_lock);
2865
2866         mutex_destroy(&iter->mutex);
2867         free_cpumask_var(iter->started);
2868         kfree(iter->trace);
2869         kfree(iter->buffer_iter);
2870         seq_release_private(inode, file);
2871         return 0;
2872 }
2873
2874 static int tracing_open(struct inode *inode, struct file *file)
2875 {
2876         struct trace_iterator *iter;
2877         int ret = 0;
2878
2879         /* If this file was open for write, then erase contents */
2880         if ((file->f_mode & FMODE_WRITE) &&
2881             (file->f_flags & O_TRUNC)) {
2882                 struct trace_cpu *tc = inode->i_private;
2883                 struct trace_array *tr = tc->tr;
2884
2885                 if (tc->cpu == RING_BUFFER_ALL_CPUS)
2886                         tracing_reset_online_cpus(&tr->trace_buffer);
2887                 else
2888                         tracing_reset(&tr->trace_buffer, tc->cpu);
2889         }
2890
2891         if (file->f_mode & FMODE_READ) {
2892                 iter = __tracing_open(inode, file, false);
2893                 if (IS_ERR(iter))
2894                         ret = PTR_ERR(iter);
2895                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2896                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
2897         }
2898         return ret;
2899 }
2900
2901 static void *
2902 t_next(struct seq_file *m, void *v, loff_t *pos)
2903 {
2904         struct tracer *t = v;
2905
2906         (*pos)++;
2907
2908         if (t)
2909                 t = t->next;
2910
2911         return t;
2912 }
2913
2914 static void *t_start(struct seq_file *m, loff_t *pos)
2915 {
2916         struct tracer *t;
2917         loff_t l = 0;
2918
2919         mutex_lock(&trace_types_lock);
2920         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2921                 ;
2922
2923         return t;
2924 }
2925
2926 static void t_stop(struct seq_file *m, void *p)
2927 {
2928         mutex_unlock(&trace_types_lock);
2929 }
2930
2931 static int t_show(struct seq_file *m, void *v)
2932 {
2933         struct tracer *t = v;
2934
2935         if (!t)
2936                 return 0;
2937
2938         seq_printf(m, "%s", t->name);
2939         if (t->next)
2940                 seq_putc(m, ' ');
2941         else
2942                 seq_putc(m, '\n');
2943
2944         return 0;
2945 }
2946
2947 static const struct seq_operations show_traces_seq_ops = {
2948         .start          = t_start,
2949         .next           = t_next,
2950         .stop           = t_stop,
2951         .show           = t_show,
2952 };
2953
2954 static int show_traces_open(struct inode *inode, struct file *file)
2955 {
2956         if (tracing_disabled)
2957                 return -ENODEV;
2958
2959         return seq_open(file, &show_traces_seq_ops);
2960 }
2961
2962 static ssize_t
2963 tracing_write_stub(struct file *filp, const char __user *ubuf,
2964                    size_t count, loff_t *ppos)
2965 {
2966         return count;
2967 }
2968
2969 static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2970 {
2971         if (file->f_mode & FMODE_READ)
2972                 return seq_lseek(file, offset, origin);
2973         else
2974                 return 0;
2975 }
2976
2977 static const struct file_operations tracing_fops = {
2978         .open           = tracing_open,
2979         .read           = seq_read,
2980         .write          = tracing_write_stub,
2981         .llseek         = tracing_seek,
2982         .release        = tracing_release,
2983 };
2984
2985 static const struct file_operations show_traces_fops = {
2986         .open           = show_traces_open,
2987         .read           = seq_read,
2988         .release        = seq_release,
2989         .llseek         = seq_lseek,
2990 };
2991
2992 /*
2993  * Only trace on a CPU if the bitmask is set:
2994  */
2995 static cpumask_var_t tracing_cpumask;
2996
2997 /*
2998  * The tracer itself will not take this lock, but still we want
2999  * to provide a consistent cpumask to user-space:
3000  */
3001 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3002
3003 /*
3004  * Temporary storage for the character representation of the
3005  * CPU bitmask (and one more byte for the newline):
3006  */
3007 static char mask_str[NR_CPUS + 1];
3008
3009 static ssize_t
3010 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3011                      size_t count, loff_t *ppos)
3012 {
3013         int len;
3014
3015         mutex_lock(&tracing_cpumask_update_lock);
3016
3017         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
3018         if (count - len < 2) {
3019                 count = -EINVAL;
3020                 goto out_err;
3021         }
3022         len += sprintf(mask_str + len, "\n");
3023         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3024
3025 out_err:
3026         mutex_unlock(&tracing_cpumask_update_lock);
3027
3028         return count;
3029 }
3030
3031 static ssize_t
3032 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3033                       size_t count, loff_t *ppos)
3034 {
3035         struct trace_array *tr = filp->private_data;
3036         cpumask_var_t tracing_cpumask_new;
3037         int err, cpu;
3038
3039         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3040                 return -ENOMEM;
3041
3042         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3043         if (err)
3044                 goto err_unlock;
3045
3046         mutex_lock(&tracing_cpumask_update_lock);
3047
3048         local_irq_disable();
3049         arch_spin_lock(&ftrace_max_lock);
3050         for_each_tracing_cpu(cpu) {
3051                 /*
3052                  * Increase/decrease the disabled counter if we are
3053                  * about to flip a bit in the cpumask:
3054                  */
3055                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
3056                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3057                         atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3058                         ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3059                 }
3060                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
3061                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3062                         atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3063                         ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3064                 }
3065         }
3066         arch_spin_unlock(&ftrace_max_lock);
3067         local_irq_enable();
3068
3069         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
3070
3071         mutex_unlock(&tracing_cpumask_update_lock);
3072         free_cpumask_var(tracing_cpumask_new);
3073
3074         return count;
3075
3076 err_unlock:
3077         free_cpumask_var(tracing_cpumask_new);
3078
3079         return err;
3080 }
3081
3082 static const struct file_operations tracing_cpumask_fops = {
3083         .open           = tracing_open_generic,
3084         .read           = tracing_cpumask_read,
3085         .write          = tracing_cpumask_write,
3086         .llseek         = generic_file_llseek,
3087 };
3088
3089 static int tracing_trace_options_show(struct seq_file *m, void *v)
3090 {
3091         struct tracer_opt *trace_opts;
3092         struct trace_array *tr = m->private;
3093         u32 tracer_flags;
3094         int i;
3095
3096         mutex_lock(&trace_types_lock);
3097         tracer_flags = tr->current_trace->flags->val;
3098         trace_opts = tr->current_trace->flags->opts;
3099
3100         for (i = 0; trace_options[i]; i++) {
3101                 if (trace_flags & (1 << i))
3102                         seq_printf(m, "%s\n", trace_options[i]);
3103                 else
3104                         seq_printf(m, "no%s\n", trace_options[i]);
3105         }
3106
3107         for (i = 0; trace_opts[i].name; i++) {
3108                 if (tracer_flags & trace_opts[i].bit)
3109                         seq_printf(m, "%s\n", trace_opts[i].name);
3110                 else
3111                         seq_printf(m, "no%s\n", trace_opts[i].name);
3112         }
3113         mutex_unlock(&trace_types_lock);
3114
3115         return 0;
3116 }
3117
3118 static int __set_tracer_option(struct tracer *trace,
3119                                struct tracer_flags *tracer_flags,
3120                                struct tracer_opt *opts, int neg)
3121 {
3122         int ret;
3123
3124         ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3125         if (ret)
3126                 return ret;
3127
3128         if (neg)
3129                 tracer_flags->val &= ~opts->bit;
3130         else
3131                 tracer_flags->val |= opts->bit;
3132         return 0;
3133 }
3134
3135 /* Try to assign a tracer specific option */
3136 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3137 {
3138         struct tracer_flags *tracer_flags = trace->flags;
3139         struct tracer_opt *opts = NULL;
3140         int i;
3141
3142         for (i = 0; tracer_flags->opts[i].name; i++) {
3143                 opts = &tracer_flags->opts[i];
3144
3145                 if (strcmp(cmp, opts->name) == 0)
3146                         return __set_tracer_option(trace, trace->flags,
3147                                                    opts, neg);
3148         }
3149
3150         return -EINVAL;
3151 }
3152
3153 /* Some tracers require overwrite to stay enabled */
3154 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3155 {
3156         if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3157                 return -1;
3158
3159         return 0;
3160 }
3161
3162 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3163 {
3164         /* do nothing if flag is already set */
3165         if (!!(trace_flags & mask) == !!enabled)
3166                 return 0;
3167
3168         /* Give the tracer a chance to approve the change */
3169         if (tr->current_trace->flag_changed)
3170                 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
3171                         return -EINVAL;
3172
3173         if (enabled)
3174                 trace_flags |= mask;
3175         else
3176                 trace_flags &= ~mask;
3177
3178         if (mask == TRACE_ITER_RECORD_CMD)
3179                 trace_event_enable_cmd_record(enabled);
3180
3181         if (mask == TRACE_ITER_OVERWRITE) {
3182                 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3183 #ifdef CONFIG_TRACER_MAX_TRACE
3184                 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3185 #endif
3186         }
3187
3188         if (mask == TRACE_ITER_PRINTK)
3189                 trace_printk_start_stop_comm(enabled);
3190
3191         return 0;
3192 }
3193
3194 static int trace_set_options(struct trace_array *tr, char *option)
3195 {
3196         char *cmp;
3197         int neg = 0;
3198         int ret = -ENODEV;
3199         int i;
3200
3201         cmp = strstrip(option);
3202
3203         if (strncmp(cmp, "no", 2) == 0) {
3204                 neg = 1;
3205                 cmp += 2;
3206         }
3207
3208         mutex_lock(&trace_types_lock);
3209
3210         for (i = 0; trace_options[i]; i++) {
3211                 if (strcmp(cmp, trace_options[i]) == 0) {
3212                         ret = set_tracer_flag(tr, 1 << i, !neg);
3213                         break;
3214                 }
3215         }
3216
3217         /* If no option could be set, test the specific tracer options */
3218         if (!trace_options[i])
3219                 ret = set_tracer_option(tr->current_trace, cmp, neg);
3220
3221         mutex_unlock(&trace_types_lock);
3222
3223         return ret;
3224 }
3225
3226 static ssize_t
3227 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3228                         size_t cnt, loff_t *ppos)
3229 {
3230         struct seq_file *m = filp->private_data;
3231         struct trace_array *tr = m->private;
3232         char buf[64];
3233         int ret;
3234
3235         if (cnt >= sizeof(buf))
3236                 return -EINVAL;
3237
3238         if (copy_from_user(&buf, ubuf, cnt))
3239                 return -EFAULT;
3240
3241         buf[cnt] = 0;
3242
3243         ret = trace_set_options(tr, buf);
3244         if (ret < 0)
3245                 return ret;
3246
3247         *ppos += cnt;
3248
3249         return cnt;
3250 }
3251
3252 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3253 {
3254         if (tracing_disabled)
3255                 return -ENODEV;
3256
3257         return single_open(file, tracing_trace_options_show, inode->i_private);
3258 }
3259
3260 static const struct file_operations tracing_iter_fops = {
3261         .open           = tracing_trace_options_open,
3262         .read           = seq_read,
3263         .llseek         = seq_lseek,
3264         .release        = single_release,
3265         .write          = tracing_trace_options_write,
3266 };
3267
3268 static const char readme_msg[] =
3269         "tracing mini-HOWTO:\n\n"
3270         "# mount -t debugfs nodev /sys/kernel/debug\n\n"
3271         "# cat /sys/kernel/debug/tracing/available_tracers\n"
3272         "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
3273         "# cat /sys/kernel/debug/tracing/current_tracer\n"
3274         "nop\n"
3275         "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
3276         "# cat /sys/kernel/debug/tracing/current_tracer\n"
3277         "wakeup\n"
3278         "# cat /sys/kernel/debug/tracing/trace_options\n"
3279         "noprint-parent nosym-offset nosym-addr noverbose\n"
3280         "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
3281         "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
3282         "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
3283         "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
3284 ;
3285
3286 static ssize_t
3287 tracing_readme_read(struct file *filp, char __user *ubuf,
3288                        size_t cnt, loff_t *ppos)
3289 {
3290         return simple_read_from_buffer(ubuf, cnt, ppos,
3291                                         readme_msg, strlen(readme_msg));
3292 }
3293
3294 static const struct file_operations tracing_readme_fops = {
3295         .open           = tracing_open_generic,
3296         .read           = tracing_readme_read,
3297         .llseek         = generic_file_llseek,
3298 };
3299
3300 static ssize_t
3301 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3302                                 size_t cnt, loff_t *ppos)
3303 {
3304         char *buf_comm;
3305         char *file_buf;
3306         char *buf;
3307         int len = 0;
3308         int pid;
3309         int i;
3310
3311         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3312         if (!file_buf)
3313                 return -ENOMEM;
3314
3315         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3316         if (!buf_comm) {
3317                 kfree(file_buf);
3318                 return -ENOMEM;
3319         }
3320
3321         buf = file_buf;
3322
3323         for (i = 0; i < SAVED_CMDLINES; i++) {
3324                 int r;
3325
3326                 pid = map_cmdline_to_pid[i];
3327                 if (pid == -1 || pid == NO_CMDLINE_MAP)
3328                         continue;
3329
3330                 trace_find_cmdline(pid, buf_comm);
3331                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3332                 buf += r;
3333                 len += r;
3334         }
3335
3336         len = simple_read_from_buffer(ubuf, cnt, ppos,
3337                                       file_buf, len);
3338
3339         kfree(file_buf);
3340         kfree(buf_comm);
3341
3342         return len;
3343 }
3344
3345 static const struct file_operations tracing_saved_cmdlines_fops = {
3346     .open       = tracing_open_generic,
3347     .read       = tracing_saved_cmdlines_read,
3348     .llseek     = generic_file_llseek,
3349 };
3350
3351 static ssize_t
3352 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3353                        size_t cnt, loff_t *ppos)
3354 {
3355         struct trace_array *tr = filp->private_data;
3356         char buf[MAX_TRACER_SIZE+2];
3357         int r;
3358
3359         mutex_lock(&trace_types_lock);
3360         r = sprintf(buf, "%s\n", tr->current_trace->name);
3361         mutex_unlock(&trace_types_lock);
3362
3363         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3364 }
3365
3366 int tracer_init(struct tracer *t, struct trace_array *tr)
3367 {
3368         tracing_reset_online_cpus(&tr->trace_buffer);
3369         return t->init(tr);
3370 }
3371
3372 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3373 {
3374         int cpu;
3375
3376         for_each_tracing_cpu(cpu)
3377                 per_cpu_ptr(buf->data, cpu)->entries = val;
3378 }
3379
3380 #ifdef CONFIG_TRACER_MAX_TRACE
3381 /* resize @tr's buffer to the size of @size_tr's entries */
3382 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3383                                         struct trace_buffer *size_buf, int cpu_id)
3384 {
3385         int cpu, ret = 0;
3386
3387         if (cpu_id == RING_BUFFER_ALL_CPUS) {
3388                 for_each_tracing_cpu(cpu) {
3389                         ret = ring_buffer_resize(trace_buf->buffer,
3390                                  per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3391                         if (ret < 0)
3392                                 break;
3393                         per_cpu_ptr(trace_buf->data, cpu)->entries =
3394                                 per_cpu_ptr(size_buf->data, cpu)->entries;
3395                 }
3396         } else {
3397                 ret = ring_buffer_resize(trace_buf->buffer,
3398                                  per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3399                 if (ret == 0)
3400                         per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3401                                 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3402         }
3403
3404         return ret;
3405 }
3406 #endif /* CONFIG_TRACER_MAX_TRACE */
3407
3408 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3409                                         unsigned long size, int cpu)
3410 {
3411         int ret;
3412
3413         /*
3414          * If kernel or user changes the size of the ring buffer
3415          * we use the size that was given, and we can forget about
3416          * expanding it later.
3417          */
3418         ring_buffer_expanded = true;
3419
3420         /* May be called before buffers are initialized */
3421         if (!tr->trace_buffer.buffer)
3422                 return 0;
3423
3424         ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3425         if (ret < 0)
3426                 return ret;
3427
3428 #ifdef CONFIG_TRACER_MAX_TRACE
3429         if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3430             !tr->current_trace->use_max_tr)
3431                 goto out;
3432
3433         ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3434         if (ret < 0) {
3435                 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3436                                                      &tr->trace_buffer, cpu);
3437                 if (r < 0) {
3438                         /*
3439                          * AARGH! We are left with different
3440                          * size max buffer!!!!
3441                          * The max buffer is our "snapshot" buffer.
3442                          * When a tracer needs a snapshot (one of the
3443                          * latency tracers), it swaps the max buffer
3444                          * with the saved snap shot. We succeeded to
3445                          * update the size of the main buffer, but failed to
3446                          * update the size of the max buffer. But when we tried
3447                          * to reset the main buffer to the original size, we
3448                          * failed there too. This is very unlikely to
3449                          * happen, but if it does, warn and kill all
3450                          * tracing.
3451                          */
3452                         WARN_ON(1);
3453                         tracing_disabled = 1;
3454                 }
3455                 return ret;
3456         }
3457
3458         if (cpu == RING_BUFFER_ALL_CPUS)
3459                 set_buffer_entries(&tr->max_buffer, size);
3460         else
3461                 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3462
3463  out:
3464 #endif /* CONFIG_TRACER_MAX_TRACE */
3465
3466         if (cpu == RING_BUFFER_ALL_CPUS)
3467                 set_buffer_entries(&tr->trace_buffer, size);
3468         else
3469                 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
3470
3471         return ret;
3472 }
3473
3474 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3475                                           unsigned long size, int cpu_id)
3476 {
3477         int ret = size;
3478
3479         mutex_lock(&trace_types_lock);
3480
3481         if (cpu_id != RING_BUFFER_ALL_CPUS) {
3482                 /* make sure, this cpu is enabled in the mask */
3483                 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3484                         ret = -EINVAL;
3485                         goto out;
3486                 }
3487         }
3488
3489         ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3490         if (ret < 0)
3491                 ret = -ENOMEM;
3492
3493 out:
3494         mutex_unlock(&trace_types_lock);
3495
3496         return ret;
3497 }
3498
3499
3500 /**
3501  * tracing_update_buffers - used by tracing facility to expand ring buffers
3502  *
3503  * To save on memory when the tracing is never used on a system with it
3504  * configured in. The ring buffers are set to a minimum size. But once
3505  * a user starts to use the tracing facility, then they need to grow
3506  * to their default size.
3507  *
3508  * This function is to be called when a tracer is about to be used.
3509  */
3510 int tracing_update_buffers(void)
3511 {
3512         int ret = 0;
3513
3514         mutex_lock(&trace_types_lock);
3515         if (!ring_buffer_expanded)
3516                 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3517                                                 RING_BUFFER_ALL_CPUS);
3518         mutex_unlock(&trace_types_lock);
3519
3520         return ret;
3521 }
3522
3523 struct trace_option_dentry;
3524
3525 static struct trace_option_dentry *
3526 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
3527
3528 static void
3529 destroy_trace_option_files(struct trace_option_dentry *topts);
3530
3531 static int tracing_set_tracer(const char *buf)
3532 {
3533         static struct trace_option_dentry *topts;
3534         struct trace_array *tr = &global_trace;
3535         struct tracer *t;
3536 #ifdef CONFIG_TRACER_MAX_TRACE
3537         bool had_max_tr;
3538 #endif
3539         int ret = 0;
3540
3541         mutex_lock(&trace_types_lock);
3542
3543         if (!ring_buffer_expanded) {
3544                 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3545                                                 RING_BUFFER_ALL_CPUS);
3546                 if (ret < 0)
3547                         goto out;
3548                 ret = 0;
3549         }
3550
3551         for (t = trace_types; t; t = t->next) {
3552                 if (strcmp(t->name, buf) == 0)
3553                         break;
3554         }
3555         if (!t) {
3556                 ret = -EINVAL;
3557                 goto out;
3558         }
3559         if (t == tr->current_trace)
3560                 goto out;
3561
3562         trace_branch_disable();
3563
3564         tr->current_trace->enabled = false;
3565
3566         if (tr->current_trace->reset)
3567                 tr->current_trace->reset(tr);
3568
3569         /* Current trace needs to be nop_trace before synchronize_sched */
3570         tr->current_trace = &nop_trace;
3571
3572 #ifdef CONFIG_TRACER_MAX_TRACE
3573         had_max_tr = tr->allocated_snapshot;
3574
3575         if (had_max_tr && !t->use_max_tr) {
3576                 /*
3577                  * We need to make sure that the update_max_tr sees that
3578                  * current_trace changed to nop_trace to keep it from
3579                  * swapping the buffers after we resize it.
3580                  * The update_max_tr is called from interrupts disabled
3581                  * so a synchronized_sched() is sufficient.
3582                  */
3583                 synchronize_sched();
3584                 /*
3585                  * We don't free the ring buffer. instead, resize it because
3586                  * The max_tr ring buffer has some state (e.g. ring->clock) and
3587                  * we want preserve it.
3588                  */
3589                 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
3590                 set_buffer_entries(&tr->max_buffer, 1);
3591                 tracing_reset_online_cpus(&tr->max_buffer);
3592                 tr->allocated_snapshot = false;
3593         }
3594 #endif
3595         destroy_trace_option_files(topts);
3596
3597         topts = create_trace_option_files(tr, t);
3598
3599 #ifdef CONFIG_TRACER_MAX_TRACE
3600         if (t->use_max_tr && !had_max_tr) {
3601                 /* we need to make per cpu buffer sizes equivalent */
3602                 ret = resize_buffer_duplicate_size(&tr->max_buffer, &tr->trace_buffer,
3603                                                    RING_BUFFER_ALL_CPUS);
3604                 if (ret < 0)
3605                         goto out;
3606                 tr->allocated_snapshot = true;
3607         }
3608 #endif
3609
3610         if (t->init) {
3611                 ret = tracer_init(t, tr);
3612                 if (ret)
3613                         goto out;
3614         }
3615
3616         tr->current_trace = t;
3617         tr->current_trace->enabled = true;
3618         trace_branch_enable(tr);
3619  out:
3620         mutex_unlock(&trace_types_lock);
3621
3622         return ret;
3623 }
3624
3625 static ssize_t
3626 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3627                         size_t cnt, loff_t *ppos)
3628 {
3629         char buf[MAX_TRACER_SIZE+1];
3630         int i;
3631         size_t ret;
3632         int err;
3633
3634         ret = cnt;
3635
3636         if (cnt > MAX_TRACER_SIZE)
3637                 cnt = MAX_TRACER_SIZE;
3638
3639         if (copy_from_user(&buf, ubuf, cnt))
3640                 return -EFAULT;
3641
3642         buf[cnt] = 0;
3643
3644         /* strip ending whitespace. */
3645         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3646                 buf[i] = 0;
3647
3648         err = tracing_set_tracer(buf);
3649         if (err)
3650                 return err;
3651
3652         *ppos += ret;
3653
3654         return ret;
3655 }
3656
3657 static ssize_t
3658 tracing_max_lat_read(struct file *filp, char __user *ubuf,
3659                      size_t cnt, loff_t *ppos)
3660 {
3661         unsigned long *ptr = filp->private_data;
3662         char buf[64];
3663         int r;
3664
3665         r = snprintf(buf, sizeof(buf), "%ld\n",
3666                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3667         if (r > sizeof(buf))
3668                 r = sizeof(buf);
3669         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3670 }
3671
3672 static ssize_t
3673 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3674                       size_t cnt, loff_t *ppos)
3675 {
3676         unsigned long *ptr = filp->private_data;
3677         unsigned long val;
3678         int ret;
3679
3680         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3681         if (ret)
3682                 return ret;
3683
3684         *ptr = val * 1000;
3685
3686         return cnt;
3687 }
3688
3689 static int tracing_open_pipe(struct inode *inode, struct file *filp)
3690 {
3691         struct trace_cpu *tc = inode->i_private;
3692         struct trace_array *tr = tc->tr;
3693         struct trace_iterator *iter;
3694         int ret = 0;
3695
3696         if (tracing_disabled)
3697                 return -ENODEV;
3698
3699         mutex_lock(&trace_types_lock);
3700
3701         /* create a buffer to store the information to pass to userspace */
3702         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3703         if (!iter) {
3704                 ret = -ENOMEM;
3705                 goto out;
3706         }
3707
3708         /*
3709          * We make a copy of the current tracer to avoid concurrent
3710          * changes on it while we are reading.
3711          */
3712         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3713         if (!iter->trace) {
3714                 ret = -ENOMEM;
3715                 goto fail;
3716         }
3717         *iter->trace = *tr->current_trace;
3718
3719         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3720                 ret = -ENOMEM;
3721                 goto fail;
3722         }
3723
3724         /* trace pipe does not show start of buffer */
3725         cpumask_setall(iter->started);
3726
3727         if (trace_flags & TRACE_ITER_LATENCY_FMT)
3728                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3729
3730         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3731         if (trace_clocks[trace_clock_id].in_ns)
3732                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3733
3734         iter->cpu_file = tc->cpu;
3735         iter->tr = tc->tr;
3736         iter->trace_buffer = &tc->tr->trace_buffer;
3737         mutex_init(&iter->mutex);
3738         filp->private_data = iter;
3739
3740         if (iter->trace->pipe_open)
3741                 iter->trace->pipe_open(iter);
3742
3743         nonseekable_open(inode, filp);
3744 out:
3745         mutex_unlock(&trace_types_lock);
3746         return ret;
3747
3748 fail:
3749         kfree(iter->trace);
3750         kfree(iter);
3751         mutex_unlock(&trace_types_lock);
3752         return ret;
3753 }
3754
3755 static int tracing_release_pipe(struct inode *inode, struct file *file)
3756 {
3757         struct trace_iterator *iter = file->private_data;
3758
3759         mutex_lock(&trace_types_lock);
3760
3761         if (iter->trace->pipe_close)
3762                 iter->trace->pipe_close(iter);
3763
3764         mutex_unlock(&trace_types_lock);
3765
3766         free_cpumask_var(iter->started);
3767         mutex_destroy(&iter->mutex);
3768         kfree(iter->trace);
3769         kfree(iter);
3770
3771         return 0;
3772 }
3773
3774 static unsigned int
3775 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
3776 {
3777         /* Iterators are static, they should be filled or empty */
3778         if (trace_buffer_iter(iter, iter->cpu_file))
3779                 return POLLIN | POLLRDNORM;
3780
3781         if (trace_flags & TRACE_ITER_BLOCK)
3782                 /*
3783                  * Always select as readable when in blocking mode
3784                  */
3785                 return POLLIN | POLLRDNORM;
3786         else
3787                 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
3788                                              filp, poll_table);
3789 }
3790
3791 static unsigned int
3792 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3793 {
3794         struct trace_iterator *iter = filp->private_data;
3795
3796         return trace_poll(iter, filp, poll_table);
3797 }
3798
3799 /*
3800  * This is a make-shift waitqueue.
3801  * A tracer might use this callback on some rare cases:
3802  *
3803  *  1) the current tracer might hold the runqueue lock when it wakes up
3804  *     a reader, hence a deadlock (sched, function, and function graph tracers)
3805  *  2) the function tracers, trace all functions, we don't want
3806  *     the overhead of calling wake_up and friends
3807  *     (and tracing them too)
3808  *
3809  *     Anyway, this is really very primitive wakeup.
3810  */
3811 void poll_wait_pipe(struct trace_iterator *iter)
3812 {
3813         set_current_state(TASK_INTERRUPTIBLE);
3814         /* sleep for 100 msecs, and try again. */
3815         schedule_timeout(HZ / 10);
3816 }
3817
3818 /* Must be called with trace_types_lock mutex held. */
3819 static int tracing_wait_pipe(struct file *filp)
3820 {
3821         struct trace_iterator *iter = filp->private_data;
3822
3823         while (trace_empty(iter)) {
3824
3825                 if ((filp->f_flags & O_NONBLOCK)) {
3826                         return -EAGAIN;
3827                 }
3828
3829                 mutex_unlock(&iter->mutex);
3830
3831                 iter->trace->wait_pipe(iter);
3832
3833                 mutex_lock(&iter->mutex);
3834
3835                 if (signal_pending(current))
3836                         return -EINTR;
3837
3838                 /*
3839                  * We block until we read something and tracing is disabled.
3840                  * We still block if tracing is disabled, but we have never
3841                  * read anything. This allows a user to cat this file, and
3842                  * then enable tracing. But after we have read something,
3843                  * we give an EOF when tracing is again disabled.
3844                  *
3845                  * iter->pos will be 0 if we haven't read anything.
3846                  */
3847                 if (!tracing_is_enabled() && iter->pos)
3848                         break;
3849         }
3850
3851         return 1;
3852 }
3853
3854 /*
3855  * Consumer reader.
3856  */
3857 static ssize_t
3858 tracing_read_pipe(struct file *filp, char __user *ubuf,
3859                   size_t cnt, loff_t *ppos)
3860 {
3861         struct trace_iterator *iter = filp->private_data;
3862         struct trace_array *tr = iter->tr;
3863         ssize_t sret;
3864
3865         /* return any leftover data */
3866         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3867         if (sret != -EBUSY)
3868                 return sret;
3869
3870         trace_seq_init(&iter->seq);
3871
3872         /* copy the tracer to avoid using a global lock all around */
3873         mutex_lock(&trace_types_lock);
3874         if (unlikely(iter->trace->name != tr->current_trace->name))
3875                 *iter->trace = *tr->current_trace;
3876         mutex_unlock(&trace_types_lock);
3877
3878         /*
3879          * Avoid more than one consumer on a single file descriptor
3880          * This is just a matter of traces coherency, the ring buffer itself
3881          * is protected.
3882          */
3883         mutex_lock(&iter->mutex);
3884         if (iter->trace->read) {
3885                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3886                 if (sret)
3887                         goto out;
3888         }
3889
3890 waitagain:
3891         sret = tracing_wait_pipe(filp);
3892         if (sret <= 0)
3893                 goto out;
3894
3895         /* stop when tracing is finished */
3896         if (trace_empty(iter)) {
3897                 sret = 0;
3898                 goto out;
3899         }
3900
3901         if (cnt >= PAGE_SIZE)
3902                 cnt = PAGE_SIZE - 1;
3903
3904         /* reset all but tr, trace, and overruns */
3905         memset(&iter->seq, 0,
3906                sizeof(struct trace_iterator) -
3907                offsetof(struct trace_iterator, seq));
3908         iter->pos = -1;
3909
3910         trace_event_read_lock();
3911         trace_access_lock(iter->cpu_file);
3912         while (trace_find_next_entry_inc(iter) != NULL) {
3913                 enum print_line_t ret;
3914                 int len = iter->seq.len;
3915
3916                 ret = print_trace_line(iter);
3917                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3918                         /* don't print partial lines */
3919                         iter->seq.len = len;
3920                         break;
3921                 }
3922                 if (ret != TRACE_TYPE_NO_CONSUME)
3923                         trace_consume(iter);
3924
3925                 if (iter->seq.len >= cnt)
3926                         break;
3927
3928                 /*
3929                  * Setting the full flag means we reached the trace_seq buffer
3930                  * size and we should leave by partial output condition above.
3931                  * One of the trace_seq_* functions is not used properly.
3932                  */
3933                 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3934                           iter->ent->type);
3935         }
3936         trace_access_unlock(iter->cpu_file);
3937         trace_event_read_unlock();
3938
3939         /* Now copy what we have to the user */
3940         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3941         if (iter->seq.readpos >= iter->seq.len)
3942                 trace_seq_init(&iter->seq);
3943
3944         /*
3945          * If there was nothing to send to user, in spite of consuming trace
3946          * entries, go back to wait for more entries.
3947          */
3948         if (sret == -EBUSY)
3949                 goto waitagain;
3950
3951 out:
3952         mutex_unlock(&iter->mutex);
3953
3954         return sret;
3955 }
3956
3957 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3958                                      struct pipe_buffer *buf)
3959 {
3960         __free_page(buf->page);
3961 }
3962
3963 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3964                                      unsigned int idx)
3965 {
3966         __free_page(spd->pages[idx]);
3967 }
3968
3969 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3970         .can_merge              = 0,
3971         .map                    = generic_pipe_buf_map,
3972         .unmap                  = generic_pipe_buf_unmap,
3973         .confirm                = generic_pipe_buf_confirm,
3974         .release                = tracing_pipe_buf_release,
3975         .steal                  = generic_pipe_buf_steal,
3976         .get                    = generic_pipe_buf_get,
3977 };
3978
3979 static size_t
3980 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
3981 {
3982         size_t count;
3983         int ret;
3984
3985         /* Seq buffer is page-sized, exactly what we need. */
3986         for (;;) {
3987                 count = iter->seq.len;
3988                 ret = print_trace_line(iter);
3989                 count = iter->seq.len - count;
3990                 if (rem < count) {
3991                         rem = 0;
3992                         iter->seq.len -= count;
3993                         break;
3994                 }
3995                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3996                         iter->seq.len -= count;
3997                         break;
3998                 }
3999
4000                 if (ret != TRACE_TYPE_NO_CONSUME)
4001                         trace_consume(iter);
4002                 rem -= count;
4003                 if (!trace_find_next_entry_inc(iter))   {
4004                         rem = 0;
4005                         iter->ent = NULL;
4006                         break;
4007                 }
4008         }
4009
4010         return rem;
4011 }
4012
4013 static ssize_t tracing_splice_read_pipe(struct file *filp,
4014                                         loff_t *ppos,
4015                                         struct pipe_inode_info *pipe,
4016                                         size_t len,
4017                                         unsigned int flags)
4018 {
4019         struct page *pages_def[PIPE_DEF_BUFFERS];
4020         struct partial_page partial_def[PIPE_DEF_BUFFERS];
4021         struct trace_iterator *iter = filp->private_data;
4022         struct splice_pipe_desc spd = {
4023                 .pages          = pages_def,
4024                 .partial        = partial_def,
4025                 .nr_pages       = 0, /* This gets updated below. */
4026                 .nr_pages_max   = PIPE_DEF_BUFFERS,
4027                 .flags          = flags,
4028                 .ops            = &tracing_pipe_buf_ops,
4029                 .spd_release    = tracing_spd_release_pipe,
4030         };
4031         struct trace_array *tr = iter->tr;
4032         ssize_t ret;
4033         size_t rem;
4034         unsigned int i;
4035
4036         if (splice_grow_spd(pipe, &spd))
4037                 return -ENOMEM;
4038
4039         /* copy the tracer to avoid using a global lock all around */
4040         mutex_lock(&trace_types_lock);
4041         if (unlikely(iter->trace->name != tr->current_trace->name))
4042                 *iter->trace = *tr->current_trace;
4043         mutex_unlock(&trace_types_lock);
4044
4045         mutex_lock(&iter->mutex);
4046
4047         if (iter->trace->splice_read) {
4048                 ret = iter->trace->splice_read(iter, filp,
4049                                                ppos, pipe, len, flags);
4050                 if (ret)
4051                         goto out_err;
4052         }
4053
4054         ret = tracing_wait_pipe(filp);
4055         if (ret <= 0)
4056                 goto out_err;
4057
4058         if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4059                 ret = -EFAULT;
4060                 goto out_err;
4061         }
4062
4063         trace_event_read_lock();
4064         trace_access_lock(iter->cpu_file);
4065
4066         /* Fill as many pages as possible. */
4067         for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4068                 spd.pages[i] = alloc_page(GFP_KERNEL);
4069                 if (!spd.pages[i])
4070                         break;
4071
4072                 rem = tracing_fill_pipe_page(rem, iter);
4073
4074                 /* Copy the data into the page, so we can start over. */
4075                 ret = trace_seq_to_buffer(&iter->seq,
4076                                           page_address(spd.pages[i]),
4077                                           iter->seq.len);
4078                 if (ret < 0) {
4079                         __free_page(spd.pages[i]);
4080                         break;
4081                 }
4082                 spd.partial[i].offset = 0;
4083                 spd.partial[i].len = iter->seq.len;
4084
4085                 trace_seq_init(&iter->seq);
4086         }
4087
4088         trace_access_unlock(iter->cpu_file);
4089         trace_event_read_unlock();
4090         mutex_unlock(&iter->mutex);
4091
4092         spd.nr_pages = i;
4093
4094         ret = splice_to_pipe(pipe, &spd);
4095 out:
4096         splice_shrink_spd(&spd);
4097         return ret;
4098
4099 out_err:
4100         mutex_unlock(&iter->mutex);
4101         goto out;
4102 }
4103
4104 static ssize_t
4105 tracing_entries_read(struct file *filp, char __user *ubuf,
4106                      size_t cnt, loff_t *ppos)
4107 {
4108         struct trace_cpu *tc = filp->private_data;
4109         struct trace_array *tr = tc->tr;
4110         char buf[64];
4111         int r = 0;
4112         ssize_t ret;
4113
4114         mutex_lock(&trace_types_lock);
4115
4116         if (tc->cpu == RING_BUFFER_ALL_CPUS) {
4117                 int cpu, buf_size_same;
4118                 unsigned long size;
4119
4120                 size = 0;
4121                 buf_size_same = 1;
4122                 /* check if all cpu sizes are same */
4123                 for_each_tracing_cpu(cpu) {
4124                         /* fill in the size from first enabled cpu */
4125                         if (size == 0)
4126                                 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4127                         if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4128                                 buf_size_same = 0;
4129                                 break;
4130                         }
4131                 }
4132
4133                 if (buf_size_same) {
4134                         if (!ring_buffer_expanded)
4135                                 r = sprintf(buf, "%lu (expanded: %lu)\n",
4136                                             size >> 10,
4137                                             trace_buf_size >> 10);
4138                         else
4139                                 r = sprintf(buf, "%lu\n", size >> 10);
4140                 } else
4141                         r = sprintf(buf, "X\n");
4142         } else
4143                 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, tc->cpu)->entries >> 10);
4144
4145         mutex_unlock(&trace_types_lock);
4146
4147         ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4148         return ret;
4149 }
4150
4151 static ssize_t
4152 tracing_entries_write(struct file *filp, const char __user *ubuf,
4153                       size_t cnt, loff_t *ppos)
4154 {
4155         struct trace_cpu *tc = filp->private_data;
4156         unsigned long val;
4157         int ret;
4158
4159         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4160         if (ret)
4161                 return ret;
4162
4163         /* must have at least 1 entry */
4164         if (!val)
4165                 return -EINVAL;
4166
4167         /* value is in KB */
4168         val <<= 10;
4169
4170         ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
4171         if (ret < 0)
4172                 return ret;
4173
4174         *ppos += cnt;
4175
4176         return cnt;
4177 }
4178
4179 static ssize_t
4180 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4181                                 size_t cnt, loff_t *ppos)
4182 {
4183         struct trace_array *tr = filp->private_data;
4184         char buf[64];
4185         int r, cpu;
4186         unsigned long size = 0, expanded_size = 0;
4187
4188         mutex_lock(&trace_types_lock);
4189         for_each_tracing_cpu(cpu) {
4190                 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4191                 if (!ring_buffer_expanded)
4192                         expanded_size += trace_buf_size >> 10;
4193         }
4194         if (ring_buffer_expanded)
4195                 r = sprintf(buf, "%lu\n", size);
4196         else
4197                 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4198         mutex_unlock(&trace_types_lock);
4199
4200         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4201 }
4202
4203 static ssize_t
4204 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4205                           size_t cnt, loff_t *ppos)
4206 {
4207         /*
4208          * There is no need to read what the user has written, this function
4209          * is just to make sure that there is no error when "echo" is used
4210          */
4211
4212         *ppos += cnt;
4213
4214         return cnt;
4215 }
4216
4217 static int
4218 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4219 {
4220         struct trace_array *tr = inode->i_private;
4221
4222         /* disable tracing ? */
4223         if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4224                 tracing_off();
4225         /* resize the ring buffer to 0 */
4226         tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4227
4228         return 0;
4229 }
4230
4231 static ssize_t
4232 tracing_mark_write(struct file *filp, const char __user *ubuf,
4233                                         size_t cnt, loff_t *fpos)
4234 {
4235         unsigned long addr = (unsigned long)ubuf;
4236         struct ring_buffer_event *event;
4237         struct ring_buffer *buffer;
4238         struct print_entry *entry;
4239         unsigned long irq_flags;
4240         struct page *pages[2];
4241         void *map_page[2];
4242         int nr_pages = 1;
4243         ssize_t written;
4244         int offset;
4245         int size;
4246         int len;
4247         int ret;
4248         int i;
4249
4250         if (tracing_disabled)
4251                 return -EINVAL;
4252
4253         if (!(trace_flags & TRACE_ITER_MARKERS))
4254                 return -EINVAL;
4255
4256         if (cnt > TRACE_BUF_SIZE)
4257                 cnt = TRACE_BUF_SIZE;
4258
4259         /*
4260          * Userspace is injecting traces into the kernel trace buffer.
4261          * We want to be as non intrusive as possible.
4262          * To do so, we do not want to allocate any special buffers
4263          * or take any locks, but instead write the userspace data
4264          * straight into the ring buffer.
4265          *
4266          * First we need to pin the userspace buffer into memory,
4267          * which, most likely it is, because it just referenced it.
4268          * But there's no guarantee that it is. By using get_user_pages_fast()
4269          * and kmap_atomic/kunmap_atomic() we can get access to the
4270          * pages directly. We then write the data directly into the
4271          * ring buffer.
4272          */
4273         BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4274
4275         /* check if we cross pages */
4276         if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4277                 nr_pages = 2;
4278
4279         offset = addr & (PAGE_SIZE - 1);
4280         addr &= PAGE_MASK;
4281
4282         ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4283         if (ret < nr_pages) {
4284                 while (--ret >= 0)
4285                         put_page(pages[ret]);
4286                 written = -EFAULT;
4287                 goto out;
4288         }
4289
4290         for (i = 0; i < nr_pages; i++)
4291                 map_page[i] = kmap_atomic(pages[i]);
4292
4293         local_save_flags(irq_flags);
4294         size = sizeof(*entry) + cnt + 2; /* possible \n added */
4295         buffer = global_trace.trace_buffer.buffer;
4296         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4297                                           irq_flags, preempt_count());
4298         if (!event) {
4299                 /* Ring buffer disabled, return as if not open for write */
4300                 written = -EBADF;
4301                 goto out_unlock;
4302         }
4303
4304         entry = ring_buffer_event_data(event);
4305         entry->ip = _THIS_IP_;
4306
4307         if (nr_pages == 2) {
4308                 len = PAGE_SIZE - offset;
4309                 memcpy(&entry->buf, map_page[0] + offset, len);
4310                 memcpy(&entry->buf[len], map_page[1], cnt - len);
4311         } else
4312                 memcpy(&entry->buf, map_page[0] + offset, cnt);
4313
4314         if (entry->buf[cnt - 1] != '\n') {
4315                 entry->buf[cnt] = '\n';
4316                 entry->buf[cnt + 1] = '\0';
4317         } else
4318                 entry->buf[cnt] = '\0';
4319
4320         __buffer_unlock_commit(buffer, event);
4321
4322         written = cnt;
4323
4324         *fpos += written;
4325
4326  out_unlock:
4327         for (i = 0; i < nr_pages; i++){
4328                 kunmap_atomic(map_page[i]);
4329                 put_page(pages[i]);
4330         }
4331  out:
4332         return written;
4333 }
4334
4335 static int tracing_clock_show(struct seq_file *m, void *v)
4336 {
4337         struct trace_array *tr = m->private;
4338         int i;
4339
4340         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4341                 seq_printf(m,
4342                         "%s%s%s%s", i ? " " : "",
4343                         i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4344                         i == tr->clock_id ? "]" : "");
4345         seq_putc(m, '\n');
4346
4347         return 0;
4348 }
4349
4350 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4351                                    size_t cnt, loff_t *fpos)
4352 {
4353         struct seq_file *m = filp->private_data;
4354         struct trace_array *tr = m->private;
4355         char buf[64];
4356         const char *clockstr;
4357         int i;
4358
4359         if (cnt >= sizeof(buf))
4360                 return -EINVAL;
4361
4362         if (copy_from_user(&buf, ubuf, cnt))
4363                 return -EFAULT;
4364
4365         buf[cnt] = 0;
4366
4367         clockstr = strstrip(buf);
4368
4369         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4370                 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4371                         break;
4372         }
4373         if (i == ARRAY_SIZE(trace_clocks))
4374                 return -EINVAL;
4375
4376         mutex_lock(&trace_types_lock);
4377
4378         tr->clock_id = i;
4379
4380         ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4381
4382         /*
4383          * New clock may not be consistent with the previous clock.
4384          * Reset the buffer so that it doesn't have incomparable timestamps.
4385          */
4386         tracing_reset_online_cpus(&global_trace.trace_buffer);
4387
4388 #ifdef CONFIG_TRACER_MAX_TRACE
4389         if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4390                 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4391         tracing_reset_online_cpus(&global_trace.max_buffer);
4392 #endif
4393
4394         mutex_unlock(&trace_types_lock);
4395
4396         *fpos += cnt;
4397
4398         return cnt;
4399 }
4400
4401 static int tracing_clock_open(struct inode *inode, struct file *file)
4402 {
4403         if (tracing_disabled)
4404                 return -ENODEV;
4405
4406         return single_open(file, tracing_clock_show, inode->i_private);
4407 }
4408
4409 struct ftrace_buffer_info {
4410         struct trace_iterator   iter;
4411         void                    *spare;
4412         unsigned int            read;
4413 };
4414
4415 #ifdef CONFIG_TRACER_SNAPSHOT
4416 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4417 {
4418         struct trace_cpu *tc = inode->i_private;
4419         struct trace_iterator *iter;
4420         struct seq_file *m;
4421         int ret = 0;
4422
4423         if (file->f_mode & FMODE_READ) {
4424                 iter = __tracing_open(inode, file, true);
4425                 if (IS_ERR(iter))
4426                         ret = PTR_ERR(iter);
4427         } else {
4428                 /* Writes still need the seq_file to hold the private data */
4429                 m = kzalloc(sizeof(*m), GFP_KERNEL);
4430                 if (!m)
4431                         return -ENOMEM;
4432                 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4433                 if (!iter) {
4434                         kfree(m);
4435                         return -ENOMEM;
4436                 }
4437                 iter->tr = tc->tr;
4438                 iter->trace_buffer = &tc->tr->max_buffer;
4439                 iter->cpu_file = tc->cpu;
4440                 m->private = iter;
4441                 file->private_data = m;
4442         }
4443
4444         return ret;
4445 }
4446
4447 static ssize_t
4448 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4449                        loff_t *ppos)
4450 {
4451         struct seq_file *m = filp->private_data;
4452         struct trace_iterator *iter = m->private;
4453         struct trace_array *tr = iter->tr;
4454         unsigned long val;
4455         int ret;
4456
4457         ret = tracing_update_buffers();
4458         if (ret < 0)
4459                 return ret;
4460
4461         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4462         if (ret)
4463                 return ret;
4464
4465         mutex_lock(&trace_types_lock);
4466
4467         if (tr->current_trace->use_max_tr) {
4468                 ret = -EBUSY;
4469                 goto out;
4470         }
4471
4472         switch (val) {
4473         case 0:
4474                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4475                         ret = -EINVAL;
4476                         break;
4477                 }
4478                 if (tr->allocated_snapshot) {
4479                         /* free spare buffer */
4480                         ring_buffer_resize(tr->max_buffer.buffer, 1,
4481                                            RING_BUFFER_ALL_CPUS);
4482                         set_buffer_entries(&tr->max_buffer, 1);
4483                         tracing_reset_online_cpus(&tr->max_buffer);
4484                         tr->allocated_snapshot = false;
4485                 }
4486                 break;
4487         case 1:
4488 /* Only allow per-cpu swap if the ring buffer supports it */
4489 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4490                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4491                         ret = -EINVAL;
4492                         break;
4493                 }
4494 #endif
4495                 if (!tr->allocated_snapshot) {
4496                         /* allocate spare buffer */
4497                         ret = resize_buffer_duplicate_size(&tr->max_buffer,
4498                                         &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
4499                         if (ret < 0)
4500                                 break;
4501                         tr->allocated_snapshot = true;
4502                 }
4503                 local_irq_disable();
4504                 /* Now, we're going to swap */
4505                 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4506                         update_max_tr(tr, current, smp_processor_id());
4507                 else
4508                         update_max_tr_single(tr, current, iter->cpu_file);
4509                 local_irq_enable();
4510                 break;
4511         default:
4512                 if (tr->allocated_snapshot) {
4513                         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4514                                 tracing_reset_online_cpus(&tr->max_buffer);
4515                         else
4516                                 tracing_reset(&tr->max_buffer, iter->cpu_file);
4517                 }
4518                 break;
4519         }
4520
4521         if (ret >= 0) {
4522                 *ppos += cnt;
4523                 ret = cnt;
4524         }
4525 out:
4526         mutex_unlock(&trace_types_lock);
4527         return ret;
4528 }
4529
4530 static int tracing_snapshot_release(struct inode *inode, struct file *file)
4531 {
4532         struct seq_file *m = file->private_data;
4533
4534         if (file->f_mode & FMODE_READ)
4535                 return tracing_release(inode, file);
4536
4537         /* If write only, the seq_file is just a stub */
4538         if (m)
4539                 kfree(m->private);
4540         kfree(m);
4541
4542         return 0;
4543 }
4544
4545 static int tracing_buffers_open(struct inode *inode, struct file *filp);
4546 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4547                                     size_t count, loff_t *ppos);
4548 static int tracing_buffers_release(struct inode *inode, struct file *file);
4549 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4550                    struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4551
4552 static int snapshot_raw_open(struct inode *inode, struct file *filp)
4553 {
4554         struct ftrace_buffer_info *info;
4555         int ret;
4556
4557         ret = tracing_buffers_open(inode, filp);
4558         if (ret < 0)
4559                 return ret;
4560
4561         info = filp->private_data;
4562
4563         if (info->iter.trace->use_max_tr) {
4564                 tracing_buffers_release(inode, filp);
4565                 return -EBUSY;
4566         }
4567
4568         info->iter.snapshot = true;
4569         info->iter.trace_buffer = &info->iter.tr->max_buffer;
4570
4571         return ret;
4572 }
4573
4574 #endif /* CONFIG_TRACER_SNAPSHOT */
4575
4576
4577 static const struct file_operations tracing_max_lat_fops = {
4578         .open           = tracing_open_generic,
4579         .read           = tracing_max_lat_read,
4580         .write          = tracing_max_lat_write,
4581         .llseek         = generic_file_llseek,
4582 };
4583
4584 static const struct file_operations set_tracer_fops = {
4585         .open           = tracing_open_generic,
4586         .read           = tracing_set_trace_read,
4587         .write          = tracing_set_trace_write,
4588         .llseek         = generic_file_llseek,
4589 };
4590
4591 static const struct file_operations tracing_pipe_fops = {
4592         .open           = tracing_open_pipe,
4593         .poll           = tracing_poll_pipe,
4594         .read           = tracing_read_pipe,
4595         .splice_read    = tracing_splice_read_pipe,
4596         .release        = tracing_release_pipe,
4597         .llseek         = no_llseek,
4598 };
4599
4600 static const struct file_operations tracing_entries_fops = {
4601         .open           = tracing_open_generic,
4602         .read           = tracing_entries_read,
4603         .write          = tracing_entries_write,
4604         .llseek         = generic_file_llseek,
4605 };
4606
4607 static const struct file_operations tracing_total_entries_fops = {
4608         .open           = tracing_open_generic,
4609         .read           = tracing_total_entries_read,
4610         .llseek         = generic_file_llseek,
4611 };
4612
4613 static const struct file_operations tracing_free_buffer_fops = {
4614         .write          = tracing_free_buffer_write,
4615         .release        = tracing_free_buffer_release,
4616 };
4617
4618 static const struct file_operations tracing_mark_fops = {
4619         .open           = tracing_open_generic,
4620         .write          = tracing_mark_write,
4621         .llseek         = generic_file_llseek,
4622 };
4623
4624 static const struct file_operations trace_clock_fops = {
4625         .open           = tracing_clock_open,
4626         .read           = seq_read,
4627         .llseek         = seq_lseek,
4628         .release        = single_release,
4629         .write          = tracing_clock_write,
4630 };
4631
4632 #ifdef CONFIG_TRACER_SNAPSHOT
4633 static const struct file_operations snapshot_fops = {
4634         .open           = tracing_snapshot_open,
4635         .read           = seq_read,
4636         .write          = tracing_snapshot_write,
4637         .llseek         = tracing_seek,
4638         .release        = tracing_snapshot_release,
4639 };
4640
4641 static const struct file_operations snapshot_raw_fops = {
4642         .open           = snapshot_raw_open,
4643         .read           = tracing_buffers_read,
4644         .release        = tracing_buffers_release,
4645         .splice_read    = tracing_buffers_splice_read,
4646         .llseek         = no_llseek,
4647 };
4648
4649 #endif /* CONFIG_TRACER_SNAPSHOT */
4650
4651 static int tracing_buffers_open(struct inode *inode, struct file *filp)
4652 {
4653         struct trace_cpu *tc = inode->i_private;
4654         struct trace_array *tr = tc->tr;
4655         struct ftrace_buffer_info *info;
4656
4657         if (tracing_disabled)
4658                 return -ENODEV;
4659
4660         info = kzalloc(sizeof(*info), GFP_KERNEL);
4661         if (!info)
4662                 return -ENOMEM;
4663
4664         mutex_lock(&trace_types_lock);
4665
4666         tr->ref++;
4667
4668         info->iter.tr           = tr;
4669         info->iter.cpu_file     = tc->cpu;
4670         info->iter.trace        = tr->current_trace;
4671         info->iter.trace_buffer = &tr->trace_buffer;
4672         info->spare             = NULL;
4673         /* Force reading ring buffer for first read */
4674         info->read              = (unsigned int)-1;
4675
4676         filp->private_data = info;
4677
4678         mutex_unlock(&trace_types_lock);
4679
4680         return nonseekable_open(inode, filp);
4681 }
4682
4683 static unsigned int
4684 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4685 {
4686         struct ftrace_buffer_info *info = filp->private_data;
4687         struct trace_iterator *iter = &info->iter;
4688
4689         return trace_poll(iter, filp, poll_table);
4690 }
4691
4692 static ssize_t
4693 tracing_buffers_read(struct file *filp, char __user *ubuf,
4694                      size_t count, loff_t *ppos)
4695 {
4696         struct ftrace_buffer_info *info = filp->private_data;
4697         struct trace_iterator *iter = &info->iter;
4698         ssize_t ret;
4699         ssize_t size;
4700
4701         if (!count)
4702                 return 0;
4703
4704         mutex_lock(&trace_types_lock);
4705
4706 #ifdef CONFIG_TRACER_MAX_TRACE
4707         if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4708                 size = -EBUSY;
4709                 goto out_unlock;
4710         }
4711 #endif
4712
4713         if (!info->spare)
4714                 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4715                                                           iter->cpu_file);
4716         size = -ENOMEM;
4717         if (!info->spare)
4718                 goto out_unlock;
4719
4720         /* Do we have previous read data to read? */
4721         if (info->read < PAGE_SIZE)
4722                 goto read;
4723
4724  again:
4725         trace_access_lock(iter->cpu_file);
4726         ret = ring_buffer_read_page(iter->trace_buffer->buffer,
4727                                     &info->spare,
4728                                     count,
4729                                     iter->cpu_file, 0);
4730         trace_access_unlock(iter->cpu_file);
4731
4732         if (ret < 0) {
4733                 if (trace_empty(iter)) {
4734                         if ((filp->f_flags & O_NONBLOCK)) {
4735                                 size = -EAGAIN;
4736                                 goto out_unlock;
4737                         }
4738                         mutex_unlock(&trace_types_lock);
4739                         iter->trace->wait_pipe(iter);
4740                         mutex_lock(&trace_types_lock);
4741                         if (signal_pending(current)) {
4742                                 size = -EINTR;
4743                                 goto out_unlock;
4744                         }
4745                         goto again;
4746                 }
4747                 size = 0;
4748                 goto out_unlock;
4749         }
4750
4751         info->read = 0;
4752  read:
4753         size = PAGE_SIZE - info->read;
4754         if (size > count)
4755                 size = count;
4756
4757         ret = copy_to_user(ubuf, info->spare + info->read, size);
4758         if (ret == size) {
4759                 size = -EFAULT;
4760                 goto out_unlock;
4761         }
4762         size -= ret;
4763
4764         *ppos += size;
4765         info->read += size;
4766
4767  out_unlock:
4768         mutex_unlock(&trace_types_lock);
4769
4770         return size;
4771 }
4772
4773 static int tracing_buffers_release(struct inode *inode, struct file *file)
4774 {
4775         struct ftrace_buffer_info *info = file->private_data;
4776         struct trace_iterator *iter = &info->iter;
4777
4778         mutex_lock(&trace_types_lock);
4779
4780         WARN_ON(!iter->tr->ref);
4781         iter->tr->ref--;
4782
4783         if (info->spare)
4784                 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
4785         kfree(info);
4786
4787         mutex_unlock(&trace_types_lock);
4788
4789         return 0;
4790 }
4791
4792 struct buffer_ref {
4793         struct ring_buffer      *buffer;
4794         void                    *page;
4795         int                     ref;
4796 };
4797
4798 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4799                                     struct pipe_buffer *buf)
4800 {
4801         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4802
4803         if (--ref->ref)
4804                 return;
4805
4806         ring_buffer_free_read_page(ref->buffer, ref->page);
4807         kfree(ref);
4808         buf->private = 0;
4809 }
4810
4811 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4812                                 struct pipe_buffer *buf)
4813 {
4814         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4815
4816         ref->ref++;
4817 }
4818
4819 /* Pipe buffer operations for a buffer. */
4820 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4821         .can_merge              = 0,
4822         .map                    = generic_pipe_buf_map,
4823         .unmap                  = generic_pipe_buf_unmap,
4824         .confirm                = generic_pipe_buf_confirm,
4825         .release                = buffer_pipe_buf_release,
4826         .steal                  = generic_pipe_buf_steal,
4827         .get                    = buffer_pipe_buf_get,
4828 };
4829
4830 /*
4831  * Callback from splice_to_pipe(), if we need to release some pages
4832  * at the end of the spd in case we error'ed out in filling the pipe.
4833  */
4834 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4835 {
4836         struct buffer_ref *ref =
4837                 (struct buffer_ref *)spd->partial[i].private;
4838
4839         if (--ref->ref)
4840                 return;
4841
4842         ring_buffer_free_read_page(ref->buffer, ref->page);
4843         kfree(ref);
4844         spd->partial[i].private = 0;
4845 }
4846
4847 static ssize_t
4848 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4849                             struct pipe_inode_info *pipe, size_t len,
4850                             unsigned int flags)
4851 {
4852         struct ftrace_buffer_info *info = file->private_data;
4853         struct trace_iterator *iter = &info->iter;
4854         struct partial_page partial_def[PIPE_DEF_BUFFERS];
4855         struct page *pages_def[PIPE_DEF_BUFFERS];
4856         struct splice_pipe_desc spd = {
4857                 .pages          = pages_def,
4858                 .partial        = partial_def,
4859                 .nr_pages_max   = PIPE_DEF_BUFFERS,
4860                 .flags          = flags,
4861                 .ops            = &buffer_pipe_buf_ops,
4862                 .spd_release    = buffer_spd_release,
4863         };
4864         struct buffer_ref *ref;
4865         int entries, size, i;
4866         ssize_t ret;
4867
4868         mutex_lock(&trace_types_lock);
4869
4870 #ifdef CONFIG_TRACER_MAX_TRACE
4871         if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4872                 ret = -EBUSY;
4873                 goto out;
4874         }
4875 #endif
4876
4877         if (splice_grow_spd(pipe, &spd)) {
4878                 ret = -ENOMEM;
4879                 goto out;
4880         }
4881
4882         if (*ppos & (PAGE_SIZE - 1)) {
4883                 ret = -EINVAL;
4884                 goto out;
4885         }
4886
4887         if (len & (PAGE_SIZE - 1)) {
4888                 if (len < PAGE_SIZE) {
4889                         ret = -EINVAL;
4890                         goto out;
4891                 }
4892                 len &= PAGE_MASK;
4893         }
4894
4895  again:
4896         trace_access_lock(iter->cpu_file);
4897         entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
4898
4899         for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4900                 struct page *page;
4901                 int r;
4902
4903                 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4904                 if (!ref)
4905                         break;
4906
4907                 ref->ref = 1;
4908                 ref->buffer = iter->trace_buffer->buffer;
4909                 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
4910                 if (!ref->page) {
4911                         kfree(ref);
4912                         break;
4913                 }
4914
4915                 r = ring_buffer_read_page(ref->buffer, &ref->page,
4916                                           len, iter->cpu_file, 1);
4917                 if (r < 0) {
4918                         ring_buffer_free_read_page(ref->buffer, ref->page);
4919                         kfree(ref);
4920                         break;
4921                 }
4922
4923                 /*
4924                  * zero out any left over data, this is going to
4925                  * user land.
4926                  */
4927                 size = ring_buffer_page_len(ref->page);
4928                 if (size < PAGE_SIZE)
4929                         memset(ref->page + size, 0, PAGE_SIZE - size);
4930
4931                 page = virt_to_page(ref->page);
4932
4933                 spd.pages[i] = page;
4934                 spd.partial[i].len = PAGE_SIZE;
4935                 spd.partial[i].offset = 0;
4936                 spd.partial[i].private = (unsigned long)ref;
4937                 spd.nr_pages++;
4938                 *ppos += PAGE_SIZE;
4939
4940                 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
4941         }
4942
4943         trace_access_unlock(iter->cpu_file);
4944         spd.nr_pages = i;
4945
4946         /* did we read anything? */
4947         if (!spd.nr_pages) {
4948                 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
4949                         ret = -EAGAIN;
4950                         goto out;
4951                 }
4952                 mutex_unlock(&trace_types_lock);
4953                 iter->trace->wait_pipe(iter);
4954                 mutex_lock(&trace_types_lock);
4955                 if (signal_pending(current)) {
4956                         ret = -EINTR;
4957                         goto out;
4958                 }
4959                 goto again;
4960         }
4961
4962         ret = splice_to_pipe(pipe, &spd);
4963         splice_shrink_spd(&spd);
4964 out:
4965         mutex_unlock(&trace_types_lock);
4966
4967         return ret;
4968 }
4969
4970 static const struct file_operations tracing_buffers_fops = {
4971         .open           = tracing_buffers_open,
4972         .read           = tracing_buffers_read,
4973         .poll           = tracing_buffers_poll,
4974         .release        = tracing_buffers_release,
4975         .splice_read    = tracing_buffers_splice_read,
4976         .llseek         = no_llseek,
4977 };
4978
4979 static ssize_t
4980 tracing_stats_read(struct file *filp, char __user *ubuf,
4981                    size_t count, loff_t *ppos)
4982 {
4983         struct trace_cpu *tc = filp->private_data;
4984         struct trace_array *tr = tc->tr;
4985         struct trace_buffer *trace_buf = &tr->trace_buffer;
4986         struct trace_seq *s;
4987         unsigned long cnt;
4988         unsigned long long t;
4989         unsigned long usec_rem;
4990         int cpu = tc->cpu;
4991
4992         s = kmalloc(sizeof(*s), GFP_KERNEL);
4993         if (!s)
4994                 return -ENOMEM;
4995
4996         trace_seq_init(s);
4997
4998         cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
4999         trace_seq_printf(s, "entries: %ld\n", cnt);
5000
5001         cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5002         trace_seq_printf(s, "overrun: %ld\n", cnt);
5003
5004         cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5005         trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5006
5007         cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5008         trace_seq_printf(s, "bytes: %ld\n", cnt);
5009
5010         if (trace_clocks[trace_clock_id].in_ns) {
5011                 /* local or global for trace_clock */
5012                 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5013                 usec_rem = do_div(t, USEC_PER_SEC);
5014                 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5015                                                                 t, usec_rem);
5016
5017                 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5018                 usec_rem = do_div(t, USEC_PER_SEC);
5019                 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5020         } else {
5021                 /* counter or tsc mode for trace_clock */
5022                 trace_seq_printf(s, "oldest event ts: %llu\n",
5023                                 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5024
5025                 trace_seq_printf(s, "now ts: %llu\n",
5026                                 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5027         }
5028
5029         cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5030         trace_seq_printf(s, "dropped events: %ld\n", cnt);
5031
5032         cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5033         trace_seq_printf(s, "read events: %ld\n", cnt);
5034
5035         count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5036
5037         kfree(s);
5038
5039         return count;
5040 }
5041
5042 static const struct file_operations tracing_stats_fops = {
5043         .open           = tracing_open_generic,
5044         .read           = tracing_stats_read,
5045         .llseek         = generic_file_llseek,
5046 };
5047
5048 #ifdef CONFIG_DYNAMIC_FTRACE
5049
5050 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5051 {
5052         return 0;
5053 }
5054
5055 static ssize_t
5056 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5057                   size_t cnt, loff_t *ppos)
5058 {
5059         static char ftrace_dyn_info_buffer[1024];
5060         static DEFINE_MUTEX(dyn_info_mutex);
5061         unsigned long *p = filp->private_data;
5062         char *buf = ftrace_dyn_info_buffer;
5063         int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5064         int r;
5065
5066         mutex_lock(&dyn_info_mutex);
5067         r = sprintf(buf, "%ld ", *p);
5068
5069         r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5070         buf[r++] = '\n';
5071
5072         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5073
5074         mutex_unlock(&dyn_info_mutex);
5075
5076         return r;
5077 }
5078
5079 static const struct file_operations tracing_dyn_info_fops = {
5080         .open           = tracing_open_generic,
5081         .read           = tracing_read_dyn_info,
5082         .llseek         = generic_file_llseek,
5083 };
5084 #endif
5085
5086 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5087 {
5088         static int once;
5089
5090         if (tr->dir)
5091                 return tr->dir;
5092
5093         if (!debugfs_initialized())
5094                 return NULL;
5095
5096         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5097                 tr->dir = debugfs_create_dir("tracing", NULL);
5098
5099         if (!tr->dir && !once) {
5100                 once = 1;
5101                 pr_warning("Could not create debugfs directory 'tracing'\n");
5102                 return NULL;
5103         }
5104
5105         return tr->dir;
5106 }
5107
5108 struct dentry *tracing_init_dentry(void)
5109 {
5110         return tracing_init_dentry_tr(&global_trace);
5111 }
5112
5113 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5114 {
5115         struct dentry *d_tracer;
5116
5117         if (tr->percpu_dir)
5118                 return tr->percpu_dir;
5119
5120         d_tracer = tracing_init_dentry_tr(tr);
5121         if (!d_tracer)
5122                 return NULL;
5123
5124         tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5125
5126         WARN_ONCE(!tr->percpu_dir,
5127                   "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5128
5129         return tr->percpu_dir;
5130 }
5131
5132 static void
5133 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5134 {
5135         struct trace_array_cpu *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
5136         struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5137         struct dentry *d_cpu;
5138         char cpu_dir[30]; /* 30 characters should be more than enough */
5139
5140         if (!d_percpu)
5141                 return;
5142
5143         snprintf(cpu_dir, 30, "cpu%ld", cpu);
5144         d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5145         if (!d_cpu) {
5146                 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5147                 return;
5148         }
5149
5150         /* per cpu trace_pipe */
5151         trace_create_file("trace_pipe", 0444, d_cpu,
5152                         (void *)&data->trace_cpu, &tracing_pipe_fops);
5153
5154         /* per cpu trace */
5155         trace_create_file("trace", 0644, d_cpu,
5156                         (void *)&data->trace_cpu, &tracing_fops);
5157
5158         trace_create_file("trace_pipe_raw", 0444, d_cpu,
5159                         (void *)&data->trace_cpu, &tracing_buffers_fops);
5160
5161         trace_create_file("stats", 0444, d_cpu,
5162                         (void *)&data->trace_cpu, &tracing_stats_fops);
5163
5164         trace_create_file("buffer_size_kb", 0444, d_cpu,
5165                         (void *)&data->trace_cpu, &tracing_entries_fops);
5166
5167 #ifdef CONFIG_TRACER_SNAPSHOT
5168         trace_create_file("snapshot", 0644, d_cpu,
5169                           (void *)&data->trace_cpu, &snapshot_fops);
5170
5171         trace_create_file("snapshot_raw", 0444, d_cpu,
5172                         (void *)&data->trace_cpu, &snapshot_raw_fops);
5173 #endif
5174 }
5175
5176 #ifdef CONFIG_FTRACE_SELFTEST
5177 /* Let selftest have access to static functions in this file */
5178 #include "trace_selftest.c"
5179 #endif
5180
5181 struct trace_option_dentry {
5182         struct tracer_opt               *opt;
5183         struct tracer_flags             *flags;
5184         struct trace_array              *tr;
5185         struct dentry                   *entry;
5186 };
5187
5188 static ssize_t
5189 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5190                         loff_t *ppos)
5191 {
5192         struct trace_option_dentry *topt = filp->private_data;
5193         char *buf;
5194
5195         if (topt->flags->val & topt->opt->bit)
5196                 buf = "1\n";
5197         else
5198                 buf = "0\n";
5199
5200         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5201 }
5202
5203 static ssize_t
5204 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5205                          loff_t *ppos)
5206 {
5207         struct trace_option_dentry *topt = filp->private_data;
5208         unsigned long val;
5209         int ret;
5210
5211         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5212         if (ret)
5213                 return ret;
5214
5215         if (val != 0 && val != 1)
5216                 return -EINVAL;
5217
5218         if (!!(topt->flags->val & topt->opt->bit) != val) {
5219                 mutex_lock(&trace_types_lock);
5220                 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
5221                                           topt->opt, !val);
5222                 mutex_unlock(&trace_types_lock);
5223                 if (ret)
5224                         return ret;
5225         }
5226
5227         *ppos += cnt;
5228
5229         return cnt;
5230 }
5231
5232
5233 static const struct file_operations trace_options_fops = {
5234         .open = tracing_open_generic,
5235         .read = trace_options_read,
5236         .write = trace_options_write,
5237         .llseek = generic_file_llseek,
5238 };
5239
5240 static ssize_t
5241 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5242                         loff_t *ppos)
5243 {
5244         long index = (long)filp->private_data;
5245         char *buf;
5246
5247         if (trace_flags & (1 << index))
5248                 buf = "1\n";
5249         else
5250                 buf = "0\n";
5251
5252         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5253 }
5254
5255 static ssize_t
5256 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5257                          loff_t *ppos)
5258 {
5259         struct trace_array *tr = &global_trace;
5260         long index = (long)filp->private_data;
5261         unsigned long val;
5262         int ret;
5263
5264         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5265         if (ret)
5266                 return ret;
5267
5268         if (val != 0 && val != 1)
5269                 return -EINVAL;
5270
5271         mutex_lock(&trace_types_lock);
5272         ret = set_tracer_flag(tr, 1 << index, val);
5273         mutex_unlock(&trace_types_lock);
5274
5275         if (ret < 0)
5276                 return ret;
5277
5278         *ppos += cnt;
5279
5280         return cnt;
5281 }
5282
5283 static const struct file_operations trace_options_core_fops = {
5284         .open = tracing_open_generic,
5285         .read = trace_options_core_read,
5286         .write = trace_options_core_write,
5287         .llseek = generic_file_llseek,
5288 };
5289
5290 struct dentry *trace_create_file(const char *name,
5291                                  umode_t mode,
5292                                  struct dentry *parent,
5293                                  void *data,
5294                                  const struct file_operations *fops)
5295 {
5296         struct dentry *ret;
5297
5298         ret = debugfs_create_file(name, mode, parent, data, fops);
5299         if (!ret)
5300                 pr_warning("Could not create debugfs '%s' entry\n", name);
5301
5302         return ret;
5303 }
5304
5305
5306 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
5307 {
5308         struct dentry *d_tracer;
5309
5310         if (tr->options)
5311                 return tr->options;
5312
5313         d_tracer = tracing_init_dentry_tr(tr);
5314         if (!d_tracer)
5315                 return NULL;
5316
5317         tr->options = debugfs_create_dir("options", d_tracer);
5318         if (!tr->options) {
5319                 pr_warning("Could not create debugfs directory 'options'\n");
5320                 return NULL;
5321         }
5322
5323         return tr->options;
5324 }
5325
5326 static void
5327 create_trace_option_file(struct trace_array *tr,
5328                          struct trace_option_dentry *topt,
5329                          struct tracer_flags *flags,
5330                          struct tracer_opt *opt)
5331 {
5332         struct dentry *t_options;
5333
5334         t_options = trace_options_init_dentry(tr);
5335         if (!t_options)
5336                 return;
5337
5338         topt->flags = flags;
5339         topt->opt = opt;
5340         topt->tr = tr;
5341
5342         topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
5343                                     &trace_options_fops);
5344
5345 }
5346
5347 static struct trace_option_dentry *
5348 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
5349 {
5350         struct trace_option_dentry *topts;
5351         struct tracer_flags *flags;
5352         struct tracer_opt *opts;
5353         int cnt;
5354
5355         if (!tracer)
5356                 return NULL;
5357
5358         flags = tracer->flags;
5359
5360         if (!flags || !flags->opts)
5361                 return NULL;
5362
5363         opts = flags->opts;
5364
5365         for (cnt = 0; opts[cnt].name; cnt++)
5366                 ;
5367
5368         topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
5369         if (!topts)
5370                 return NULL;
5371
5372         for (cnt = 0; opts[cnt].name; cnt++)
5373                 create_trace_option_file(tr, &topts[cnt], flags,
5374                                          &opts[cnt]);
5375
5376         return topts;
5377 }
5378
5379 static void
5380 destroy_trace_option_files(struct trace_option_dentry *topts)
5381 {
5382         int cnt;
5383
5384         if (!topts)
5385                 return;
5386
5387         for (cnt = 0; topts[cnt].opt; cnt++) {
5388                 if (topts[cnt].entry)
5389                         debugfs_remove(topts[cnt].entry);
5390         }
5391
5392         kfree(topts);
5393 }
5394
5395 static struct dentry *
5396 create_trace_option_core_file(struct trace_array *tr,
5397                               const char *option, long index)
5398 {
5399         struct dentry *t_options;
5400
5401         t_options = trace_options_init_dentry(tr);
5402         if (!t_options)
5403                 return NULL;
5404
5405         return trace_create_file(option, 0644, t_options, (void *)index,
5406                                     &trace_options_core_fops);
5407 }
5408
5409 static __init void create_trace_options_dir(struct trace_array *tr)
5410 {
5411         struct dentry *t_options;
5412         int i;
5413
5414         t_options = trace_options_init_dentry(tr);
5415         if (!t_options)
5416                 return;
5417
5418         for (i = 0; trace_options[i]; i++)
5419                 create_trace_option_core_file(tr, trace_options[i], i);
5420 }
5421
5422 static ssize_t
5423 rb_simple_read(struct file *filp, char __user *ubuf,
5424                size_t cnt, loff_t *ppos)
5425 {
5426         struct trace_array *tr = filp->private_data;
5427         struct ring_buffer *buffer = tr->trace_buffer.buffer;
5428         char buf[64];
5429         int r;
5430
5431         if (buffer)
5432                 r = ring_buffer_record_is_on(buffer);
5433         else
5434                 r = 0;
5435
5436         r = sprintf(buf, "%d\n", r);
5437
5438         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5439 }
5440
5441 static ssize_t
5442 rb_simple_write(struct file *filp, const char __user *ubuf,
5443                 size_t cnt, loff_t *ppos)
5444 {
5445         struct trace_array *tr = filp->private_data;
5446         struct ring_buffer *buffer = tr->trace_buffer.buffer;
5447         unsigned long val;
5448         int ret;
5449
5450         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5451         if (ret)
5452                 return ret;
5453
5454         if (buffer) {
5455                 mutex_lock(&trace_types_lock);
5456                 if (val) {
5457                         ring_buffer_record_on(buffer);
5458                         if (tr->current_trace->start)
5459                                 tr->current_trace->start(tr);
5460                 } else {
5461                         ring_buffer_record_off(buffer);
5462                         if (tr->current_trace->stop)
5463                                 tr->current_trace->stop(tr);
5464                 }
5465                 mutex_unlock(&trace_types_lock);
5466         }
5467
5468         (*ppos)++;
5469
5470         return cnt;
5471 }
5472
5473 static const struct file_operations rb_simple_fops = {
5474         .open           = tracing_open_generic,
5475         .read           = rb_simple_read,
5476         .write          = rb_simple_write,
5477         .llseek         = default_llseek,
5478 };
5479
5480 struct dentry *trace_instance_dir;
5481
5482 static void
5483 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5484
5485 static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
5486 {
5487         int cpu;
5488
5489         for_each_tracing_cpu(cpu) {
5490                 memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
5491                 per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
5492                 per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
5493         }
5494 }
5495
5496 static int
5497 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
5498 {
5499         enum ring_buffer_flags rb_flags;
5500
5501         rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5502
5503         buf->buffer = ring_buffer_alloc(size, rb_flags);
5504         if (!buf->buffer)
5505                 return -ENOMEM;
5506
5507         buf->data = alloc_percpu(struct trace_array_cpu);
5508         if (!buf->data) {
5509                 ring_buffer_free(buf->buffer);
5510                 return -ENOMEM;
5511         }
5512
5513         init_trace_buffers(tr, buf);
5514
5515         /* Allocate the first page for all buffers */
5516         set_buffer_entries(&tr->trace_buffer,
5517                            ring_buffer_size(tr->trace_buffer.buffer, 0));
5518
5519         return 0;
5520 }
5521
5522 static int allocate_trace_buffers(struct trace_array *tr, int size)
5523 {
5524         int ret;
5525
5526         ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5527         if (ret)
5528                 return ret;
5529
5530 #ifdef CONFIG_TRACER_MAX_TRACE
5531         ret = allocate_trace_buffer(tr, &tr->max_buffer,
5532                                     allocate_snapshot ? size : 1);
5533         if (WARN_ON(ret)) {
5534                 ring_buffer_free(tr->trace_buffer.buffer);
5535                 free_percpu(tr->trace_buffer.data);
5536                 return -ENOMEM;
5537         }
5538         tr->allocated_snapshot = allocate_snapshot;
5539
5540         /*
5541          * Only the top level trace array gets its snapshot allocated
5542          * from the kernel command line.
5543          */
5544         allocate_snapshot = false;
5545 #endif
5546         return 0;
5547 }
5548
5549 static int new_instance_create(const char *name)
5550 {
5551         struct trace_array *tr;
5552         int ret;
5553
5554         mutex_lock(&trace_types_lock);
5555
5556         ret = -EEXIST;
5557         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5558                 if (tr->name && strcmp(tr->name, name) == 0)
5559                         goto out_unlock;
5560         }
5561
5562         ret = -ENOMEM;
5563         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5564         if (!tr)
5565                 goto out_unlock;
5566
5567         tr->name = kstrdup(name, GFP_KERNEL);
5568         if (!tr->name)
5569                 goto out_free_tr;
5570
5571         raw_spin_lock_init(&tr->start_lock);
5572
5573         tr->current_trace = &nop_trace;
5574
5575         INIT_LIST_HEAD(&tr->systems);
5576         INIT_LIST_HEAD(&tr->events);
5577
5578         if (allocate_trace_buffers(tr, trace_buf_size) < 0)
5579                 goto out_free_tr;
5580
5581         /* Holder for file callbacks */
5582         tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5583         tr->trace_cpu.tr = tr;
5584
5585         tr->dir = debugfs_create_dir(name, trace_instance_dir);
5586         if (!tr->dir)
5587                 goto out_free_tr;
5588
5589         ret = event_trace_add_tracer(tr->dir, tr);
5590         if (ret)
5591                 goto out_free_tr;
5592
5593         init_tracer_debugfs(tr, tr->dir);
5594
5595         list_add(&tr->list, &ftrace_trace_arrays);
5596
5597         mutex_unlock(&trace_types_lock);
5598
5599         return 0;
5600
5601  out_free_tr:
5602         if (tr->trace_buffer.buffer)
5603                 ring_buffer_free(tr->trace_buffer.buffer);
5604         kfree(tr->name);
5605         kfree(tr);
5606
5607  out_unlock:
5608         mutex_unlock(&trace_types_lock);
5609
5610         return ret;
5611
5612 }
5613
5614 static int instance_delete(const char *name)
5615 {
5616         struct trace_array *tr;
5617         int found = 0;
5618         int ret;
5619
5620         mutex_lock(&trace_types_lock);
5621
5622         ret = -ENODEV;
5623         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5624                 if (tr->name && strcmp(tr->name, name) == 0) {
5625                         found = 1;
5626                         break;
5627                 }
5628         }
5629         if (!found)
5630                 goto out_unlock;
5631
5632         ret = -EBUSY;
5633         if (tr->ref)
5634                 goto out_unlock;
5635
5636         list_del(&tr->list);
5637
5638         event_trace_del_tracer(tr);
5639         debugfs_remove_recursive(tr->dir);
5640         free_percpu(tr->trace_buffer.data);
5641         ring_buffer_free(tr->trace_buffer.buffer);
5642
5643         kfree(tr->name);
5644         kfree(tr);
5645
5646         ret = 0;
5647
5648  out_unlock:
5649         mutex_unlock(&trace_types_lock);
5650
5651         return ret;
5652 }
5653
5654 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
5655 {
5656         struct dentry *parent;
5657         int ret;
5658
5659         /* Paranoid: Make sure the parent is the "instances" directory */
5660         parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5661         if (WARN_ON_ONCE(parent != trace_instance_dir))
5662                 return -ENOENT;
5663
5664         /*
5665          * The inode mutex is locked, but debugfs_create_dir() will also
5666          * take the mutex. As the instances directory can not be destroyed
5667          * or changed in any other way, it is safe to unlock it, and
5668          * let the dentry try. If two users try to make the same dir at
5669          * the same time, then the new_instance_create() will determine the
5670          * winner.
5671          */
5672         mutex_unlock(&inode->i_mutex);
5673
5674         ret = new_instance_create(dentry->d_iname);
5675
5676         mutex_lock(&inode->i_mutex);
5677
5678         return ret;
5679 }
5680
5681 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
5682 {
5683         struct dentry *parent;
5684         int ret;
5685
5686         /* Paranoid: Make sure the parent is the "instances" directory */
5687         parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5688         if (WARN_ON_ONCE(parent != trace_instance_dir))
5689                 return -ENOENT;
5690
5691         /* The caller did a dget() on dentry */
5692         mutex_unlock(&dentry->d_inode->i_mutex);
5693
5694         /*
5695          * The inode mutex is locked, but debugfs_create_dir() will also
5696          * take the mutex. As the instances directory can not be destroyed
5697          * or changed in any other way, it is safe to unlock it, and
5698          * let the dentry try. If two users try to make the same dir at
5699          * the same time, then the instance_delete() will determine the
5700          * winner.
5701          */
5702         mutex_unlock(&inode->i_mutex);
5703
5704         ret = instance_delete(dentry->d_iname);
5705
5706         mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
5707         mutex_lock(&dentry->d_inode->i_mutex);
5708
5709         return ret;
5710 }
5711
5712 static const struct inode_operations instance_dir_inode_operations = {
5713         .lookup         = simple_lookup,
5714         .mkdir          = instance_mkdir,
5715         .rmdir          = instance_rmdir,
5716 };
5717
5718 static __init void create_trace_instances(struct dentry *d_tracer)
5719 {
5720         trace_instance_dir = debugfs_create_dir("instances", d_tracer);
5721         if (WARN_ON(!trace_instance_dir))
5722                 return;
5723
5724         /* Hijack the dir inode operations, to allow mkdir */
5725         trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
5726 }
5727
5728 static void
5729 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
5730 {
5731         int cpu;
5732
5733         trace_create_file("trace_options", 0644, d_tracer,
5734                           tr, &tracing_iter_fops);
5735
5736         trace_create_file("trace", 0644, d_tracer,
5737                         (void *)&tr->trace_cpu, &tracing_fops);
5738
5739         trace_create_file("trace_pipe", 0444, d_tracer,
5740                         (void *)&tr->trace_cpu, &tracing_pipe_fops);
5741
5742         trace_create_file("buffer_size_kb", 0644, d_tracer,
5743                         (void *)&tr->trace_cpu, &tracing_entries_fops);
5744
5745         trace_create_file("buffer_total_size_kb", 0444, d_tracer,
5746                           tr, &tracing_total_entries_fops);
5747
5748         trace_create_file("free_buffer", 0644, d_tracer,
5749                           tr, &tracing_free_buffer_fops);
5750
5751         trace_create_file("trace_marker", 0220, d_tracer,
5752                           tr, &tracing_mark_fops);
5753
5754         trace_create_file("trace_clock", 0644, d_tracer, tr,
5755                           &trace_clock_fops);
5756
5757         trace_create_file("tracing_on", 0644, d_tracer,
5758                             tr, &rb_simple_fops);
5759
5760 #ifdef CONFIG_TRACER_SNAPSHOT
5761         trace_create_file("snapshot", 0644, d_tracer,
5762                           (void *)&tr->trace_cpu, &snapshot_fops);
5763 #endif
5764
5765         for_each_tracing_cpu(cpu)
5766                 tracing_init_debugfs_percpu(tr, cpu);
5767
5768 }
5769
5770 static __init int tracer_init_debugfs(void)
5771 {
5772         struct dentry *d_tracer;
5773
5774         trace_access_lock_init();
5775
5776         d_tracer = tracing_init_dentry();
5777
5778         init_tracer_debugfs(&global_trace, d_tracer);
5779
5780         trace_create_file("tracing_cpumask", 0644, d_tracer,
5781                         &global_trace, &tracing_cpumask_fops);
5782
5783         trace_create_file("available_tracers", 0444, d_tracer,
5784                         &global_trace, &show_traces_fops);
5785
5786         trace_create_file("current_tracer", 0644, d_tracer,
5787                         &global_trace, &set_tracer_fops);
5788
5789 #ifdef CONFIG_TRACER_MAX_TRACE
5790         trace_create_file("tracing_max_latency", 0644, d_tracer,
5791                         &tracing_max_latency, &tracing_max_lat_fops);
5792 #endif
5793
5794         trace_create_file("tracing_thresh", 0644, d_tracer,
5795                         &tracing_thresh, &tracing_max_lat_fops);
5796
5797         trace_create_file("README", 0444, d_tracer,
5798                         NULL, &tracing_readme_fops);
5799
5800         trace_create_file("saved_cmdlines", 0444, d_tracer,
5801                         NULL, &tracing_saved_cmdlines_fops);
5802
5803 #ifdef CONFIG_DYNAMIC_FTRACE
5804         trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
5805                         &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
5806 #endif
5807
5808         create_trace_instances(d_tracer);
5809
5810         create_trace_options_dir(&global_trace);
5811
5812         return 0;
5813 }
5814
5815 static int trace_panic_handler(struct notifier_block *this,
5816                                unsigned long event, void *unused)
5817 {
5818         if (ftrace_dump_on_oops)
5819                 ftrace_dump(ftrace_dump_on_oops);
5820         return NOTIFY_OK;
5821 }
5822
5823 static struct notifier_block trace_panic_notifier = {
5824         .notifier_call  = trace_panic_handler,
5825         .next           = NULL,
5826         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
5827 };
5828
5829 static int trace_die_handler(struct notifier_block *self,
5830                              unsigned long val,
5831                              void *data)
5832 {
5833         switch (val) {
5834         case DIE_OOPS:
5835                 if (ftrace_dump_on_oops)
5836                         ftrace_dump(ftrace_dump_on_oops);
5837                 break;
5838         default:
5839                 break;
5840         }
5841         return NOTIFY_OK;
5842 }
5843
5844 static struct notifier_block trace_die_notifier = {
5845         .notifier_call = trace_die_handler,
5846         .priority = 200
5847 };
5848
5849 /*
5850  * printk is set to max of 1024, we really don't need it that big.
5851  * Nothing should be printing 1000 characters anyway.
5852  */
5853 #define TRACE_MAX_PRINT         1000
5854
5855 /*
5856  * Define here KERN_TRACE so that we have one place to modify
5857  * it if we decide to change what log level the ftrace dump
5858  * should be at.
5859  */
5860 #define KERN_TRACE              KERN_EMERG
5861
5862 void
5863 trace_printk_seq(struct trace_seq *s)
5864 {
5865         /* Probably should print a warning here. */
5866         if (s->len >= 1000)
5867                 s->len = 1000;
5868
5869         /* should be zero ended, but we are paranoid. */
5870         s->buffer[s->len] = 0;
5871
5872         printk(KERN_TRACE "%s", s->buffer);
5873
5874         trace_seq_init(s);
5875 }
5876
5877 void trace_init_global_iter(struct trace_iterator *iter)
5878 {
5879         iter->tr = &global_trace;
5880         iter->trace = iter->tr->current_trace;
5881         iter->cpu_file = RING_BUFFER_ALL_CPUS;
5882         iter->trace_buffer = &global_trace.trace_buffer;
5883 }
5884
5885 static void
5886 __ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
5887 {
5888         static arch_spinlock_t ftrace_dump_lock =
5889                 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
5890         /* use static because iter can be a bit big for the stack */
5891         static struct trace_iterator iter;
5892         unsigned int old_userobj;
5893         static int dump_ran;
5894         unsigned long flags;
5895         int cnt = 0, cpu;
5896
5897         /* only one dump */
5898         local_irq_save(flags);
5899         arch_spin_lock(&ftrace_dump_lock);
5900         if (dump_ran)
5901                 goto out;
5902
5903         dump_ran = 1;
5904
5905         tracing_off();
5906
5907         /* Did function tracer already get disabled? */
5908         if (ftrace_is_dead()) {
5909                 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
5910                 printk("#          MAY BE MISSING FUNCTION EVENTS\n");
5911         }
5912
5913         if (disable_tracing)
5914                 ftrace_kill();
5915
5916         /* Simulate the iterator */
5917         trace_init_global_iter(&iter);
5918
5919         for_each_tracing_cpu(cpu) {
5920                 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
5921         }
5922
5923         old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
5924
5925         /* don't look at user memory in panic mode */
5926         trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
5927
5928         switch (oops_dump_mode) {
5929         case DUMP_ALL:
5930                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
5931                 break;
5932         case DUMP_ORIG:
5933                 iter.cpu_file = raw_smp_processor_id();
5934                 break;
5935         case DUMP_NONE:
5936                 goto out_enable;
5937         default:
5938                 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
5939                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
5940         }
5941
5942         printk(KERN_TRACE "Dumping ftrace buffer:\n");
5943
5944         /*
5945          * We need to stop all tracing on all CPUS to read the
5946          * the next buffer. This is a bit expensive, but is
5947          * not done often. We fill all what we can read,
5948          * and then release the locks again.
5949          */
5950
5951         while (!trace_empty(&iter)) {
5952
5953                 if (!cnt)
5954                         printk(KERN_TRACE "---------------------------------\n");
5955
5956                 cnt++;
5957
5958                 /* reset all but tr, trace, and overruns */
5959                 memset(&iter.seq, 0,
5960                        sizeof(struct trace_iterator) -
5961                        offsetof(struct trace_iterator, seq));
5962                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
5963                 iter.pos = -1;
5964
5965                 if (trace_find_next_entry_inc(&iter) != NULL) {
5966                         int ret;
5967
5968                         ret = print_trace_line(&iter);
5969                         if (ret != TRACE_TYPE_NO_CONSUME)
5970                                 trace_consume(&iter);
5971                 }
5972                 touch_nmi_watchdog();
5973
5974                 trace_printk_seq(&iter.seq);
5975         }
5976
5977         if (!cnt)
5978                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
5979         else
5980                 printk(KERN_TRACE "---------------------------------\n");
5981
5982  out_enable:
5983         /* Re-enable tracing if requested */
5984         if (!disable_tracing) {
5985                 trace_flags |= old_userobj;
5986
5987                 for_each_tracing_cpu(cpu) {
5988                         atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
5989                 }
5990                 tracing_on();
5991         }
5992
5993  out:
5994         arch_spin_unlock(&ftrace_dump_lock);
5995         local_irq_restore(flags);
5996 }
5997
5998 /* By default: disable tracing after the dump */
5999 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6000 {
6001         __ftrace_dump(true, oops_dump_mode);
6002 }
6003 EXPORT_SYMBOL_GPL(ftrace_dump);
6004
6005 __init static int tracer_alloc_buffers(void)
6006 {
6007         int ring_buf_size;
6008         int ret = -ENOMEM;
6009
6010
6011         if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6012                 goto out;
6013
6014         if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
6015                 goto out_free_buffer_mask;
6016
6017         /* Only allocate trace_printk buffers if a trace_printk exists */
6018         if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6019                 /* Must be called before global_trace.buffer is allocated */
6020                 trace_printk_init_buffers();
6021
6022         /* To save memory, keep the ring buffer size to its minimum */
6023         if (ring_buffer_expanded)
6024                 ring_buf_size = trace_buf_size;
6025         else
6026                 ring_buf_size = 1;
6027
6028         cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6029         cpumask_copy(tracing_cpumask, cpu_all_mask);
6030
6031         raw_spin_lock_init(&global_trace.start_lock);
6032
6033         /* TODO: make the number of buffers hot pluggable with CPUS */
6034         if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6035                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6036                 WARN_ON(1);
6037                 goto out_free_cpumask;
6038         }
6039
6040         if (global_trace.buffer_disabled)
6041                 tracing_off();
6042
6043         trace_init_cmdlines();
6044
6045         register_tracer(&nop_trace);
6046
6047         global_trace.current_trace = &nop_trace;
6048
6049         /* All seems OK, enable tracing */
6050         tracing_disabled = 0;
6051
6052         atomic_notifier_chain_register(&panic_notifier_list,
6053                                        &trace_panic_notifier);
6054
6055         register_die_notifier(&trace_die_notifier);
6056
6057         global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6058
6059         /* Holder for file callbacks */
6060         global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6061         global_trace.trace_cpu.tr = &global_trace;
6062
6063         INIT_LIST_HEAD(&global_trace.systems);
6064         INIT_LIST_HEAD(&global_trace.events);
6065         list_add(&global_trace.list, &ftrace_trace_arrays);
6066
6067         while (trace_boot_options) {
6068                 char *option;
6069
6070                 option = strsep(&trace_boot_options, ",");
6071                 trace_set_options(&global_trace, option);
6072         }
6073
6074         return 0;
6075
6076 out_free_cpumask:
6077         free_percpu(global_trace.trace_buffer.data);
6078 #ifdef CONFIG_TRACER_MAX_TRACE
6079         free_percpu(global_trace.max_buffer.data);
6080 #endif
6081         free_cpumask_var(tracing_cpumask);
6082 out_free_buffer_mask:
6083         free_cpumask_var(tracing_buffer_mask);
6084 out:
6085         return ret;
6086 }
6087
6088 __init static int clear_boot_tracer(void)
6089 {
6090         /*
6091          * The default tracer at boot buffer is an init section.
6092          * This function is called in lateinit. If we did not
6093          * find the boot tracer, then clear it out, to prevent
6094          * later registration from accessing the buffer that is
6095          * about to be freed.
6096          */
6097         if (!default_bootup_tracer)
6098                 return 0;
6099
6100         printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6101                default_bootup_tracer);
6102         default_bootup_tracer = NULL;
6103
6104         return 0;
6105 }
6106
6107 early_initcall(tracer_alloc_buffers);
6108 fs_initcall(tracer_init_debugfs);
6109 late_initcall(clear_boot_tracer);