tracing: Add "perf" trace_clock
[linux-3.10.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 Nadia Yvette Chambers
13  */
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
40 #include <linux/fs.h>
41 #include <linux/sched/rt.h>
42
43 #include "trace.h"
44 #include "trace_output.h"
45
46 /*
47  * On boot up, the ring buffer is set to the minimum size, so that
48  * we do not waste memory on systems that are not using tracing.
49  */
50 bool ring_buffer_expanded;
51
52 /*
53  * We need to change this state when a selftest is running.
54  * A selftest will lurk into the ring-buffer to count the
55  * entries inserted during the selftest although some concurrent
56  * insertions into the ring-buffer such as trace_printk could occurred
57  * at the same time, giving false positive or negative results.
58  */
59 static bool __read_mostly tracing_selftest_running;
60
61 /*
62  * If a tracer is running, we do not want to run SELFTEST.
63  */
64 bool __read_mostly tracing_selftest_disabled;
65
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
68         { }
69 };
70
71 static struct tracer_flags dummy_tracer_flags = {
72         .val = 0,
73         .opts = dummy_tracer_opt
74 };
75
76 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77 {
78         return 0;
79 }
80
81 /*
82  * To prevent the comm cache from being overwritten when no
83  * tracing is active, only save the comm when a trace event
84  * occurred.
85  */
86 static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88 /*
89  * Kill all tracing for good (never come back).
90  * It is initialized to 1 but will turn to zero if the initialization
91  * of the tracer is successful. But that is the only place that sets
92  * this back to zero.
93  */
94 static int tracing_disabled = 1;
95
96 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
97
98 cpumask_var_t __read_mostly     tracing_buffer_mask;
99
100 /*
101  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102  *
103  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104  * is set, then ftrace_dump is called. This will output the contents
105  * of the ftrace buffers to the console.  This is very useful for
106  * capturing traces that lead to crashes and outputing it to a
107  * serial console.
108  *
109  * It is default off, but you can enable it with either specifying
110  * "ftrace_dump_on_oops" in the kernel command line, or setting
111  * /proc/sys/kernel/ftrace_dump_on_oops
112  * Set 1 if you want to dump buffers of all CPUs
113  * Set 2 if you want to dump the buffer of the CPU that triggered oops
114  */
115
116 enum ftrace_dump_mode ftrace_dump_on_oops;
117
118 static int tracing_set_tracer(const char *buf);
119
120 #define MAX_TRACER_SIZE         100
121 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
122 static char *default_bootup_tracer;
123
124 static bool allocate_snapshot;
125
126 static int __init set_cmdline_ftrace(char *str)
127 {
128         strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
129         default_bootup_tracer = bootup_tracer_buf;
130         /* We are using ftrace early, expand it */
131         ring_buffer_expanded = true;
132         return 1;
133 }
134 __setup("ftrace=", set_cmdline_ftrace);
135
136 static int __init set_ftrace_dump_on_oops(char *str)
137 {
138         if (*str++ != '=' || !*str) {
139                 ftrace_dump_on_oops = DUMP_ALL;
140                 return 1;
141         }
142
143         if (!strcmp("orig_cpu", str)) {
144                 ftrace_dump_on_oops = DUMP_ORIG;
145                 return 1;
146         }
147
148         return 0;
149 }
150 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
151
152 static int __init boot_alloc_snapshot(char *str)
153 {
154         allocate_snapshot = true;
155         /* We also need the main ring buffer expanded */
156         ring_buffer_expanded = true;
157         return 1;
158 }
159 __setup("alloc_snapshot", boot_alloc_snapshot);
160
161
162 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
163 static char *trace_boot_options __initdata;
164
165 static int __init set_trace_boot_options(char *str)
166 {
167         strncpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
168         trace_boot_options = trace_boot_options_buf;
169         return 0;
170 }
171 __setup("trace_options=", set_trace_boot_options);
172
173 unsigned long long ns2usecs(cycle_t nsec)
174 {
175         nsec += 500;
176         do_div(nsec, 1000);
177         return nsec;
178 }
179
180 /*
181  * The global_trace is the descriptor that holds the tracing
182  * buffers for the live tracing. For each CPU, it contains
183  * a link list of pages that will store trace entries. The
184  * page descriptor of the pages in the memory is used to hold
185  * the link list by linking the lru item in the page descriptor
186  * to each of the pages in the buffer per CPU.
187  *
188  * For each active CPU there is a data field that holds the
189  * pages for the buffer for that CPU. Each CPU has the same number
190  * of pages allocated for its buffer.
191  */
192 static struct trace_array       global_trace;
193
194 LIST_HEAD(ftrace_trace_arrays);
195
196 int filter_current_check_discard(struct ring_buffer *buffer,
197                                  struct ftrace_event_call *call, void *rec,
198                                  struct ring_buffer_event *event)
199 {
200         return filter_check_discard(call, rec, buffer, event);
201 }
202 EXPORT_SYMBOL_GPL(filter_current_check_discard);
203
204 cycle_t ftrace_now(int cpu)
205 {
206         u64 ts;
207
208         /* Early boot up does not have a buffer yet */
209         if (!global_trace.trace_buffer.buffer)
210                 return trace_clock_local();
211
212         ts = ring_buffer_time_stamp(global_trace.trace_buffer.buffer, cpu);
213         ring_buffer_normalize_time_stamp(global_trace.trace_buffer.buffer, cpu, &ts);
214
215         return ts;
216 }
217
218 int tracing_is_enabled(void)
219 {
220         return tracing_is_on();
221 }
222
223 /*
224  * trace_buf_size is the size in bytes that is allocated
225  * for a buffer. Note, the number of bytes is always rounded
226  * to page size.
227  *
228  * This number is purposely set to a low number of 16384.
229  * If the dump on oops happens, it will be much appreciated
230  * to not have to wait for all that output. Anyway this can be
231  * boot time and run time configurable.
232  */
233 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
234
235 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
236
237 /* trace_types holds a link list of available tracers. */
238 static struct tracer            *trace_types __read_mostly;
239
240 /*
241  * trace_types_lock is used to protect the trace_types list.
242  */
243 static DEFINE_MUTEX(trace_types_lock);
244
245 /*
246  * serialize the access of the ring buffer
247  *
248  * ring buffer serializes readers, but it is low level protection.
249  * The validity of the events (which returns by ring_buffer_peek() ..etc)
250  * are not protected by ring buffer.
251  *
252  * The content of events may become garbage if we allow other process consumes
253  * these events concurrently:
254  *   A) the page of the consumed events may become a normal page
255  *      (not reader page) in ring buffer, and this page will be rewrited
256  *      by events producer.
257  *   B) The page of the consumed events may become a page for splice_read,
258  *      and this page will be returned to system.
259  *
260  * These primitives allow multi process access to different cpu ring buffer
261  * concurrently.
262  *
263  * These primitives don't distinguish read-only and read-consume access.
264  * Multi read-only access are also serialized.
265  */
266
267 #ifdef CONFIG_SMP
268 static DECLARE_RWSEM(all_cpu_access_lock);
269 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
270
271 static inline void trace_access_lock(int cpu)
272 {
273         if (cpu == RING_BUFFER_ALL_CPUS) {
274                 /* gain it for accessing the whole ring buffer. */
275                 down_write(&all_cpu_access_lock);
276         } else {
277                 /* gain it for accessing a cpu ring buffer. */
278
279                 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
280                 down_read(&all_cpu_access_lock);
281
282                 /* Secondly block other access to this @cpu ring buffer. */
283                 mutex_lock(&per_cpu(cpu_access_lock, cpu));
284         }
285 }
286
287 static inline void trace_access_unlock(int cpu)
288 {
289         if (cpu == RING_BUFFER_ALL_CPUS) {
290                 up_write(&all_cpu_access_lock);
291         } else {
292                 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
293                 up_read(&all_cpu_access_lock);
294         }
295 }
296
297 static inline void trace_access_lock_init(void)
298 {
299         int cpu;
300
301         for_each_possible_cpu(cpu)
302                 mutex_init(&per_cpu(cpu_access_lock, cpu));
303 }
304
305 #else
306
307 static DEFINE_MUTEX(access_lock);
308
309 static inline void trace_access_lock(int cpu)
310 {
311         (void)cpu;
312         mutex_lock(&access_lock);
313 }
314
315 static inline void trace_access_unlock(int cpu)
316 {
317         (void)cpu;
318         mutex_unlock(&access_lock);
319 }
320
321 static inline void trace_access_lock_init(void)
322 {
323 }
324
325 #endif
326
327 /* trace_flags holds trace_options default values */
328 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
329         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
330         TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
331         TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
332
333 /**
334  * tracing_on - enable tracing buffers
335  *
336  * This function enables tracing buffers that may have been
337  * disabled with tracing_off.
338  */
339 void tracing_on(void)
340 {
341         if (global_trace.trace_buffer.buffer)
342                 ring_buffer_record_on(global_trace.trace_buffer.buffer);
343         /*
344          * This flag is only looked at when buffers haven't been
345          * allocated yet. We don't really care about the race
346          * between setting this flag and actually turning
347          * on the buffer.
348          */
349         global_trace.buffer_disabled = 0;
350 }
351 EXPORT_SYMBOL_GPL(tracing_on);
352
353 /**
354  * __trace_puts - write a constant string into the trace buffer.
355  * @ip:    The address of the caller
356  * @str:   The constant string to write
357  * @size:  The size of the string.
358  */
359 int __trace_puts(unsigned long ip, const char *str, int size)
360 {
361         struct ring_buffer_event *event;
362         struct ring_buffer *buffer;
363         struct print_entry *entry;
364         unsigned long irq_flags;
365         int alloc;
366
367         alloc = sizeof(*entry) + size + 2; /* possible \n added */
368
369         local_save_flags(irq_flags);
370         buffer = global_trace.trace_buffer.buffer;
371         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
372                                           irq_flags, preempt_count());
373         if (!event)
374                 return 0;
375
376         entry = ring_buffer_event_data(event);
377         entry->ip = ip;
378
379         memcpy(&entry->buf, str, size);
380
381         /* Add a newline if necessary */
382         if (entry->buf[size - 1] != '\n') {
383                 entry->buf[size] = '\n';
384                 entry->buf[size + 1] = '\0';
385         } else
386                 entry->buf[size] = '\0';
387
388         __buffer_unlock_commit(buffer, event);
389
390         return size;
391 }
392 EXPORT_SYMBOL_GPL(__trace_puts);
393
394 /**
395  * __trace_bputs - write the pointer to a constant string into trace buffer
396  * @ip:    The address of the caller
397  * @str:   The constant string to write to the buffer to
398  */
399 int __trace_bputs(unsigned long ip, const char *str)
400 {
401         struct ring_buffer_event *event;
402         struct ring_buffer *buffer;
403         struct bputs_entry *entry;
404         unsigned long irq_flags;
405         int size = sizeof(struct bputs_entry);
406
407         local_save_flags(irq_flags);
408         buffer = global_trace.trace_buffer.buffer;
409         event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
410                                           irq_flags, preempt_count());
411         if (!event)
412                 return 0;
413
414         entry = ring_buffer_event_data(event);
415         entry->ip                       = ip;
416         entry->str                      = str;
417
418         __buffer_unlock_commit(buffer, event);
419
420         return 1;
421 }
422 EXPORT_SYMBOL_GPL(__trace_bputs);
423
424 #ifdef CONFIG_TRACER_SNAPSHOT
425 /**
426  * trace_snapshot - take a snapshot of the current buffer.
427  *
428  * This causes a swap between the snapshot buffer and the current live
429  * tracing buffer. You can use this to take snapshots of the live
430  * trace when some condition is triggered, but continue to trace.
431  *
432  * Note, make sure to allocate the snapshot with either
433  * a tracing_snapshot_alloc(), or by doing it manually
434  * with: echo 1 > /sys/kernel/debug/tracing/snapshot
435  *
436  * If the snapshot buffer is not allocated, it will stop tracing.
437  * Basically making a permanent snapshot.
438  */
439 void tracing_snapshot(void)
440 {
441         struct trace_array *tr = &global_trace;
442         struct tracer *tracer = tr->current_trace;
443         unsigned long flags;
444
445         if (in_nmi()) {
446                 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
447                 internal_trace_puts("*** snapshot is being ignored        ***\n");
448                 return;
449         }
450
451         if (!tr->allocated_snapshot) {
452                 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
453                 internal_trace_puts("*** stopping trace here!   ***\n");
454                 tracing_off();
455                 return;
456         }
457
458         /* Note, snapshot can not be used when the tracer uses it */
459         if (tracer->use_max_tr) {
460                 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
461                 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
462                 return;
463         }
464
465         local_irq_save(flags);
466         update_max_tr(tr, current, smp_processor_id());
467         local_irq_restore(flags);
468 }
469 EXPORT_SYMBOL_GPL(tracing_snapshot);
470
471 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
472                                         struct trace_buffer *size_buf, int cpu_id);
473 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
474
475 static int alloc_snapshot(struct trace_array *tr)
476 {
477         int ret;
478
479         if (!tr->allocated_snapshot) {
480
481                 /* allocate spare buffer */
482                 ret = resize_buffer_duplicate_size(&tr->max_buffer,
483                                    &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
484                 if (ret < 0)
485                         return ret;
486
487                 tr->allocated_snapshot = true;
488         }
489
490         return 0;
491 }
492
493 void free_snapshot(struct trace_array *tr)
494 {
495         /*
496          * We don't free the ring buffer. instead, resize it because
497          * The max_tr ring buffer has some state (e.g. ring->clock) and
498          * we want preserve it.
499          */
500         ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
501         set_buffer_entries(&tr->max_buffer, 1);
502         tracing_reset_online_cpus(&tr->max_buffer);
503         tr->allocated_snapshot = false;
504 }
505
506 /**
507  * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
508  *
509  * This is similar to trace_snapshot(), but it will allocate the
510  * snapshot buffer if it isn't already allocated. Use this only
511  * where it is safe to sleep, as the allocation may sleep.
512  *
513  * This causes a swap between the snapshot buffer and the current live
514  * tracing buffer. You can use this to take snapshots of the live
515  * trace when some condition is triggered, but continue to trace.
516  */
517 void tracing_snapshot_alloc(void)
518 {
519         struct trace_array *tr = &global_trace;
520         int ret;
521
522         ret = alloc_snapshot(tr);
523         if (WARN_ON(ret < 0))
524                 return;
525
526         tracing_snapshot();
527 }
528 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
529 #else
530 void tracing_snapshot(void)
531 {
532         WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
533 }
534 EXPORT_SYMBOL_GPL(tracing_snapshot);
535 void tracing_snapshot_alloc(void)
536 {
537         /* Give warning */
538         tracing_snapshot();
539 }
540 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
541 #endif /* CONFIG_TRACER_SNAPSHOT */
542
543 /**
544  * tracing_off - turn off tracing buffers
545  *
546  * This function stops the tracing buffers from recording data.
547  * It does not disable any overhead the tracers themselves may
548  * be causing. This function simply causes all recording to
549  * the ring buffers to fail.
550  */
551 void tracing_off(void)
552 {
553         if (global_trace.trace_buffer.buffer)
554                 ring_buffer_record_off(global_trace.trace_buffer.buffer);
555         /*
556          * This flag is only looked at when buffers haven't been
557          * allocated yet. We don't really care about the race
558          * between setting this flag and actually turning
559          * on the buffer.
560          */
561         global_trace.buffer_disabled = 1;
562 }
563 EXPORT_SYMBOL_GPL(tracing_off);
564
565 /**
566  * tracing_is_on - show state of ring buffers enabled
567  */
568 int tracing_is_on(void)
569 {
570         if (global_trace.trace_buffer.buffer)
571                 return ring_buffer_record_is_on(global_trace.trace_buffer.buffer);
572         return !global_trace.buffer_disabled;
573 }
574 EXPORT_SYMBOL_GPL(tracing_is_on);
575
576 static int __init set_buf_size(char *str)
577 {
578         unsigned long buf_size;
579
580         if (!str)
581                 return 0;
582         buf_size = memparse(str, &str);
583         /* nr_entries can not be zero */
584         if (buf_size == 0)
585                 return 0;
586         trace_buf_size = buf_size;
587         return 1;
588 }
589 __setup("trace_buf_size=", set_buf_size);
590
591 static int __init set_tracing_thresh(char *str)
592 {
593         unsigned long threshold;
594         int ret;
595
596         if (!str)
597                 return 0;
598         ret = kstrtoul(str, 0, &threshold);
599         if (ret < 0)
600                 return 0;
601         tracing_thresh = threshold * 1000;
602         return 1;
603 }
604 __setup("tracing_thresh=", set_tracing_thresh);
605
606 unsigned long nsecs_to_usecs(unsigned long nsecs)
607 {
608         return nsecs / 1000;
609 }
610
611 /* These must match the bit postions in trace_iterator_flags */
612 static const char *trace_options[] = {
613         "print-parent",
614         "sym-offset",
615         "sym-addr",
616         "verbose",
617         "raw",
618         "hex",
619         "bin",
620         "block",
621         "stacktrace",
622         "trace_printk",
623         "ftrace_preempt",
624         "branch",
625         "annotate",
626         "userstacktrace",
627         "sym-userobj",
628         "printk-msg-only",
629         "context-info",
630         "latency-format",
631         "sleep-time",
632         "graph-time",
633         "record-cmd",
634         "overwrite",
635         "disable_on_free",
636         "irq-info",
637         "markers",
638         "function-trace",
639         NULL
640 };
641
642 static struct {
643         u64 (*func)(void);
644         const char *name;
645         int in_ns;              /* is this clock in nanoseconds? */
646 } trace_clocks[] = {
647         { trace_clock_local,    "local",        1 },
648         { trace_clock_global,   "global",       1 },
649         { trace_clock_counter,  "counter",      0 },
650         { trace_clock_jiffies,  "uptime",       1 },
651         { trace_clock,          "perf",         1 },
652         ARCH_TRACE_CLOCKS
653 };
654
655 int trace_clock_id;
656
657 /*
658  * trace_parser_get_init - gets the buffer for trace parser
659  */
660 int trace_parser_get_init(struct trace_parser *parser, int size)
661 {
662         memset(parser, 0, sizeof(*parser));
663
664         parser->buffer = kmalloc(size, GFP_KERNEL);
665         if (!parser->buffer)
666                 return 1;
667
668         parser->size = size;
669         return 0;
670 }
671
672 /*
673  * trace_parser_put - frees the buffer for trace parser
674  */
675 void trace_parser_put(struct trace_parser *parser)
676 {
677         kfree(parser->buffer);
678 }
679
680 /*
681  * trace_get_user - reads the user input string separated by  space
682  * (matched by isspace(ch))
683  *
684  * For each string found the 'struct trace_parser' is updated,
685  * and the function returns.
686  *
687  * Returns number of bytes read.
688  *
689  * See kernel/trace/trace.h for 'struct trace_parser' details.
690  */
691 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
692         size_t cnt, loff_t *ppos)
693 {
694         char ch;
695         size_t read = 0;
696         ssize_t ret;
697
698         if (!*ppos)
699                 trace_parser_clear(parser);
700
701         ret = get_user(ch, ubuf++);
702         if (ret)
703                 goto out;
704
705         read++;
706         cnt--;
707
708         /*
709          * The parser is not finished with the last write,
710          * continue reading the user input without skipping spaces.
711          */
712         if (!parser->cont) {
713                 /* skip white space */
714                 while (cnt && isspace(ch)) {
715                         ret = get_user(ch, ubuf++);
716                         if (ret)
717                                 goto out;
718                         read++;
719                         cnt--;
720                 }
721
722                 /* only spaces were written */
723                 if (isspace(ch)) {
724                         *ppos += read;
725                         ret = read;
726                         goto out;
727                 }
728
729                 parser->idx = 0;
730         }
731
732         /* read the non-space input */
733         while (cnt && !isspace(ch)) {
734                 if (parser->idx < parser->size - 1)
735                         parser->buffer[parser->idx++] = ch;
736                 else {
737                         ret = -EINVAL;
738                         goto out;
739                 }
740                 ret = get_user(ch, ubuf++);
741                 if (ret)
742                         goto out;
743                 read++;
744                 cnt--;
745         }
746
747         /* We either got finished input or we have to wait for another call. */
748         if (isspace(ch)) {
749                 parser->buffer[parser->idx] = 0;
750                 parser->cont = false;
751         } else {
752                 parser->cont = true;
753                 parser->buffer[parser->idx++] = ch;
754         }
755
756         *ppos += read;
757         ret = read;
758
759 out:
760         return ret;
761 }
762
763 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
764 {
765         int len;
766         int ret;
767
768         if (!cnt)
769                 return 0;
770
771         if (s->len <= s->readpos)
772                 return -EBUSY;
773
774         len = s->len - s->readpos;
775         if (cnt > len)
776                 cnt = len;
777         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
778         if (ret == cnt)
779                 return -EFAULT;
780
781         cnt -= ret;
782
783         s->readpos += cnt;
784         return cnt;
785 }
786
787 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
788 {
789         int len;
790
791         if (s->len <= s->readpos)
792                 return -EBUSY;
793
794         len = s->len - s->readpos;
795         if (cnt > len)
796                 cnt = len;
797         memcpy(buf, s->buffer + s->readpos, cnt);
798
799         s->readpos += cnt;
800         return cnt;
801 }
802
803 /*
804  * ftrace_max_lock is used to protect the swapping of buffers
805  * when taking a max snapshot. The buffers themselves are
806  * protected by per_cpu spinlocks. But the action of the swap
807  * needs its own lock.
808  *
809  * This is defined as a arch_spinlock_t in order to help
810  * with performance when lockdep debugging is enabled.
811  *
812  * It is also used in other places outside the update_max_tr
813  * so it needs to be defined outside of the
814  * CONFIG_TRACER_MAX_TRACE.
815  */
816 static arch_spinlock_t ftrace_max_lock =
817         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
818
819 unsigned long __read_mostly     tracing_thresh;
820
821 #ifdef CONFIG_TRACER_MAX_TRACE
822 unsigned long __read_mostly     tracing_max_latency;
823
824 /*
825  * Copy the new maximum trace into the separate maximum-trace
826  * structure. (this way the maximum trace is permanently saved,
827  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
828  */
829 static void
830 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
831 {
832         struct trace_buffer *trace_buf = &tr->trace_buffer;
833         struct trace_buffer *max_buf = &tr->max_buffer;
834         struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
835         struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
836
837         max_buf->cpu = cpu;
838         max_buf->time_start = data->preempt_timestamp;
839
840         max_data->saved_latency = tracing_max_latency;
841         max_data->critical_start = data->critical_start;
842         max_data->critical_end = data->critical_end;
843
844         memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
845         max_data->pid = tsk->pid;
846         max_data->uid = task_uid(tsk);
847         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
848         max_data->policy = tsk->policy;
849         max_data->rt_priority = tsk->rt_priority;
850
851         /* record this tasks comm */
852         tracing_record_cmdline(tsk);
853 }
854
855 /**
856  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
857  * @tr: tracer
858  * @tsk: the task with the latency
859  * @cpu: The cpu that initiated the trace.
860  *
861  * Flip the buffers between the @tr and the max_tr and record information
862  * about which task was the cause of this latency.
863  */
864 void
865 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
866 {
867         struct ring_buffer *buf;
868
869         if (tr->stop_count)
870                 return;
871
872         WARN_ON_ONCE(!irqs_disabled());
873
874         if (!tr->allocated_snapshot) {
875                 /* Only the nop tracer should hit this when disabling */
876                 WARN_ON_ONCE(tr->current_trace != &nop_trace);
877                 return;
878         }
879
880         arch_spin_lock(&ftrace_max_lock);
881
882         buf = tr->trace_buffer.buffer;
883         tr->trace_buffer.buffer = tr->max_buffer.buffer;
884         tr->max_buffer.buffer = buf;
885
886         __update_max_tr(tr, tsk, cpu);
887         arch_spin_unlock(&ftrace_max_lock);
888 }
889
890 /**
891  * update_max_tr_single - only copy one trace over, and reset the rest
892  * @tr - tracer
893  * @tsk - task with the latency
894  * @cpu - the cpu of the buffer to copy.
895  *
896  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
897  */
898 void
899 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
900 {
901         int ret;
902
903         if (tr->stop_count)
904                 return;
905
906         WARN_ON_ONCE(!irqs_disabled());
907         if (WARN_ON_ONCE(!tr->allocated_snapshot))
908                 return;
909
910         arch_spin_lock(&ftrace_max_lock);
911
912         ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
913
914         if (ret == -EBUSY) {
915                 /*
916                  * We failed to swap the buffer due to a commit taking
917                  * place on this CPU. We fail to record, but we reset
918                  * the max trace buffer (no one writes directly to it)
919                  * and flag that it failed.
920                  */
921                 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
922                         "Failed to swap buffers due to commit in progress\n");
923         }
924
925         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
926
927         __update_max_tr(tr, tsk, cpu);
928         arch_spin_unlock(&ftrace_max_lock);
929 }
930 #endif /* CONFIG_TRACER_MAX_TRACE */
931
932 static void default_wait_pipe(struct trace_iterator *iter)
933 {
934         /* Iterators are static, they should be filled or empty */
935         if (trace_buffer_iter(iter, iter->cpu_file))
936                 return;
937
938         ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
939 }
940
941 #ifdef CONFIG_FTRACE_STARTUP_TEST
942 static int run_tracer_selftest(struct tracer *type)
943 {
944         struct trace_array *tr = &global_trace;
945         struct tracer *saved_tracer = tr->current_trace;
946         int ret;
947
948         if (!type->selftest || tracing_selftest_disabled)
949                 return 0;
950
951         /*
952          * Run a selftest on this tracer.
953          * Here we reset the trace buffer, and set the current
954          * tracer to be this tracer. The tracer can then run some
955          * internal tracing to verify that everything is in order.
956          * If we fail, we do not register this tracer.
957          */
958         tracing_reset_online_cpus(&tr->trace_buffer);
959
960         tr->current_trace = type;
961
962 #ifdef CONFIG_TRACER_MAX_TRACE
963         if (type->use_max_tr) {
964                 /* If we expanded the buffers, make sure the max is expanded too */
965                 if (ring_buffer_expanded)
966                         ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
967                                            RING_BUFFER_ALL_CPUS);
968                 tr->allocated_snapshot = true;
969         }
970 #endif
971
972         /* the test is responsible for initializing and enabling */
973         pr_info("Testing tracer %s: ", type->name);
974         ret = type->selftest(type, tr);
975         /* the test is responsible for resetting too */
976         tr->current_trace = saved_tracer;
977         if (ret) {
978                 printk(KERN_CONT "FAILED!\n");
979                 /* Add the warning after printing 'FAILED' */
980                 WARN_ON(1);
981                 return -1;
982         }
983         /* Only reset on passing, to avoid touching corrupted buffers */
984         tracing_reset_online_cpus(&tr->trace_buffer);
985
986 #ifdef CONFIG_TRACER_MAX_TRACE
987         if (type->use_max_tr) {
988                 tr->allocated_snapshot = false;
989
990                 /* Shrink the max buffer again */
991                 if (ring_buffer_expanded)
992                         ring_buffer_resize(tr->max_buffer.buffer, 1,
993                                            RING_BUFFER_ALL_CPUS);
994         }
995 #endif
996
997         printk(KERN_CONT "PASSED\n");
998         return 0;
999 }
1000 #else
1001 static inline int run_tracer_selftest(struct tracer *type)
1002 {
1003         return 0;
1004 }
1005 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1006
1007 /**
1008  * register_tracer - register a tracer with the ftrace system.
1009  * @type - the plugin for the tracer
1010  *
1011  * Register a new plugin tracer.
1012  */
1013 int register_tracer(struct tracer *type)
1014 {
1015         struct tracer *t;
1016         int ret = 0;
1017
1018         if (!type->name) {
1019                 pr_info("Tracer must have a name\n");
1020                 return -1;
1021         }
1022
1023         if (strlen(type->name) >= MAX_TRACER_SIZE) {
1024                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1025                 return -1;
1026         }
1027
1028         mutex_lock(&trace_types_lock);
1029
1030         tracing_selftest_running = true;
1031
1032         for (t = trace_types; t; t = t->next) {
1033                 if (strcmp(type->name, t->name) == 0) {
1034                         /* already found */
1035                         pr_info("Tracer %s already registered\n",
1036                                 type->name);
1037                         ret = -1;
1038                         goto out;
1039                 }
1040         }
1041
1042         if (!type->set_flag)
1043                 type->set_flag = &dummy_set_flag;
1044         if (!type->flags)
1045                 type->flags = &dummy_tracer_flags;
1046         else
1047                 if (!type->flags->opts)
1048                         type->flags->opts = dummy_tracer_opt;
1049         if (!type->wait_pipe)
1050                 type->wait_pipe = default_wait_pipe;
1051
1052         ret = run_tracer_selftest(type);
1053         if (ret < 0)
1054                 goto out;
1055
1056         type->next = trace_types;
1057         trace_types = type;
1058
1059  out:
1060         tracing_selftest_running = false;
1061         mutex_unlock(&trace_types_lock);
1062
1063         if (ret || !default_bootup_tracer)
1064                 goto out_unlock;
1065
1066         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1067                 goto out_unlock;
1068
1069         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1070         /* Do we want this tracer to start on bootup? */
1071         tracing_set_tracer(type->name);
1072         default_bootup_tracer = NULL;
1073         /* disable other selftests, since this will break it. */
1074         tracing_selftest_disabled = true;
1075 #ifdef CONFIG_FTRACE_STARTUP_TEST
1076         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1077                type->name);
1078 #endif
1079
1080  out_unlock:
1081         return ret;
1082 }
1083
1084 void tracing_reset(struct trace_buffer *buf, int cpu)
1085 {
1086         struct ring_buffer *buffer = buf->buffer;
1087
1088         if (!buffer)
1089                 return;
1090
1091         ring_buffer_record_disable(buffer);
1092
1093         /* Make sure all commits have finished */
1094         synchronize_sched();
1095         ring_buffer_reset_cpu(buffer, cpu);
1096
1097         ring_buffer_record_enable(buffer);
1098 }
1099
1100 void tracing_reset_online_cpus(struct trace_buffer *buf)
1101 {
1102         struct ring_buffer *buffer = buf->buffer;
1103         int cpu;
1104
1105         if (!buffer)
1106                 return;
1107
1108         ring_buffer_record_disable(buffer);
1109
1110         /* Make sure all commits have finished */
1111         synchronize_sched();
1112
1113         buf->time_start = ftrace_now(buf->cpu);
1114
1115         for_each_online_cpu(cpu)
1116                 ring_buffer_reset_cpu(buffer, cpu);
1117
1118         ring_buffer_record_enable(buffer);
1119 }
1120
1121 void tracing_reset_current(int cpu)
1122 {
1123         tracing_reset(&global_trace.trace_buffer, cpu);
1124 }
1125
1126 void tracing_reset_all_online_cpus(void)
1127 {
1128         struct trace_array *tr;
1129
1130         mutex_lock(&trace_types_lock);
1131         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1132                 tracing_reset_online_cpus(&tr->trace_buffer);
1133 #ifdef CONFIG_TRACER_MAX_TRACE
1134                 tracing_reset_online_cpus(&tr->max_buffer);
1135 #endif
1136         }
1137         mutex_unlock(&trace_types_lock);
1138 }
1139
1140 #define SAVED_CMDLINES 128
1141 #define NO_CMDLINE_MAP UINT_MAX
1142 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1143 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1144 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1145 static int cmdline_idx;
1146 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1147
1148 /* temporary disable recording */
1149 static atomic_t trace_record_cmdline_disabled __read_mostly;
1150
1151 static void trace_init_cmdlines(void)
1152 {
1153         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1154         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
1155         cmdline_idx = 0;
1156 }
1157
1158 int is_tracing_stopped(void)
1159 {
1160         return global_trace.stop_count;
1161 }
1162
1163 /**
1164  * ftrace_off_permanent - disable all ftrace code permanently
1165  *
1166  * This should only be called when a serious anomally has
1167  * been detected.  This will turn off the function tracing,
1168  * ring buffers, and other tracing utilites. It takes no
1169  * locks and can be called from any context.
1170  */
1171 void ftrace_off_permanent(void)
1172 {
1173         tracing_disabled = 1;
1174         ftrace_stop();
1175         tracing_off_permanent();
1176 }
1177
1178 /**
1179  * tracing_start - quick start of the tracer
1180  *
1181  * If tracing is enabled but was stopped by tracing_stop,
1182  * this will start the tracer back up.
1183  */
1184 void tracing_start(void)
1185 {
1186         struct ring_buffer *buffer;
1187         unsigned long flags;
1188
1189         if (tracing_disabled)
1190                 return;
1191
1192         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1193         if (--global_trace.stop_count) {
1194                 if (global_trace.stop_count < 0) {
1195                         /* Someone screwed up their debugging */
1196                         WARN_ON_ONCE(1);
1197                         global_trace.stop_count = 0;
1198                 }
1199                 goto out;
1200         }
1201
1202         /* Prevent the buffers from switching */
1203         arch_spin_lock(&ftrace_max_lock);
1204
1205         buffer = global_trace.trace_buffer.buffer;
1206         if (buffer)
1207                 ring_buffer_record_enable(buffer);
1208
1209 #ifdef CONFIG_TRACER_MAX_TRACE
1210         buffer = global_trace.max_buffer.buffer;
1211         if (buffer)
1212                 ring_buffer_record_enable(buffer);
1213 #endif
1214
1215         arch_spin_unlock(&ftrace_max_lock);
1216
1217         ftrace_start();
1218  out:
1219         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1220 }
1221
1222 static void tracing_start_tr(struct trace_array *tr)
1223 {
1224         struct ring_buffer *buffer;
1225         unsigned long flags;
1226
1227         if (tracing_disabled)
1228                 return;
1229
1230         /* If global, we need to also start the max tracer */
1231         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1232                 return tracing_start();
1233
1234         raw_spin_lock_irqsave(&tr->start_lock, flags);
1235
1236         if (--tr->stop_count) {
1237                 if (tr->stop_count < 0) {
1238                         /* Someone screwed up their debugging */
1239                         WARN_ON_ONCE(1);
1240                         tr->stop_count = 0;
1241                 }
1242                 goto out;
1243         }
1244
1245         buffer = tr->trace_buffer.buffer;
1246         if (buffer)
1247                 ring_buffer_record_enable(buffer);
1248
1249  out:
1250         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1251 }
1252
1253 /**
1254  * tracing_stop - quick stop of the tracer
1255  *
1256  * Light weight way to stop tracing. Use in conjunction with
1257  * tracing_start.
1258  */
1259 void tracing_stop(void)
1260 {
1261         struct ring_buffer *buffer;
1262         unsigned long flags;
1263
1264         ftrace_stop();
1265         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1266         if (global_trace.stop_count++)
1267                 goto out;
1268
1269         /* Prevent the buffers from switching */
1270         arch_spin_lock(&ftrace_max_lock);
1271
1272         buffer = global_trace.trace_buffer.buffer;
1273         if (buffer)
1274                 ring_buffer_record_disable(buffer);
1275
1276 #ifdef CONFIG_TRACER_MAX_TRACE
1277         buffer = global_trace.max_buffer.buffer;
1278         if (buffer)
1279                 ring_buffer_record_disable(buffer);
1280 #endif
1281
1282         arch_spin_unlock(&ftrace_max_lock);
1283
1284  out:
1285         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1286 }
1287
1288 static void tracing_stop_tr(struct trace_array *tr)
1289 {
1290         struct ring_buffer *buffer;
1291         unsigned long flags;
1292
1293         /* If global, we need to also stop the max tracer */
1294         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1295                 return tracing_stop();
1296
1297         raw_spin_lock_irqsave(&tr->start_lock, flags);
1298         if (tr->stop_count++)
1299                 goto out;
1300
1301         buffer = tr->trace_buffer.buffer;
1302         if (buffer)
1303                 ring_buffer_record_disable(buffer);
1304
1305  out:
1306         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1307 }
1308
1309 void trace_stop_cmdline_recording(void);
1310
1311 static void trace_save_cmdline(struct task_struct *tsk)
1312 {
1313         unsigned pid, idx;
1314
1315         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1316                 return;
1317
1318         /*
1319          * It's not the end of the world if we don't get
1320          * the lock, but we also don't want to spin
1321          * nor do we want to disable interrupts,
1322          * so if we miss here, then better luck next time.
1323          */
1324         if (!arch_spin_trylock(&trace_cmdline_lock))
1325                 return;
1326
1327         idx = map_pid_to_cmdline[tsk->pid];
1328         if (idx == NO_CMDLINE_MAP) {
1329                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1330
1331                 /*
1332                  * Check whether the cmdline buffer at idx has a pid
1333                  * mapped. We are going to overwrite that entry so we
1334                  * need to clear the map_pid_to_cmdline. Otherwise we
1335                  * would read the new comm for the old pid.
1336                  */
1337                 pid = map_cmdline_to_pid[idx];
1338                 if (pid != NO_CMDLINE_MAP)
1339                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1340
1341                 map_cmdline_to_pid[idx] = tsk->pid;
1342                 map_pid_to_cmdline[tsk->pid] = idx;
1343
1344                 cmdline_idx = idx;
1345         }
1346
1347         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1348
1349         arch_spin_unlock(&trace_cmdline_lock);
1350 }
1351
1352 void trace_find_cmdline(int pid, char comm[])
1353 {
1354         unsigned map;
1355
1356         if (!pid) {
1357                 strcpy(comm, "<idle>");
1358                 return;
1359         }
1360
1361         if (WARN_ON_ONCE(pid < 0)) {
1362                 strcpy(comm, "<XXX>");
1363                 return;
1364         }
1365
1366         if (pid > PID_MAX_DEFAULT) {
1367                 strcpy(comm, "<...>");
1368                 return;
1369         }
1370
1371         preempt_disable();
1372         arch_spin_lock(&trace_cmdline_lock);
1373         map = map_pid_to_cmdline[pid];
1374         if (map != NO_CMDLINE_MAP)
1375                 strcpy(comm, saved_cmdlines[map]);
1376         else
1377                 strcpy(comm, "<...>");
1378
1379         arch_spin_unlock(&trace_cmdline_lock);
1380         preempt_enable();
1381 }
1382
1383 void tracing_record_cmdline(struct task_struct *tsk)
1384 {
1385         if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1386                 return;
1387
1388         if (!__this_cpu_read(trace_cmdline_save))
1389                 return;
1390
1391         __this_cpu_write(trace_cmdline_save, false);
1392
1393         trace_save_cmdline(tsk);
1394 }
1395
1396 void
1397 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1398                              int pc)
1399 {
1400         struct task_struct *tsk = current;
1401
1402         entry->preempt_count            = pc & 0xff;
1403         entry->pid                      = (tsk) ? tsk->pid : 0;
1404         entry->flags =
1405 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1406                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1407 #else
1408                 TRACE_FLAG_IRQS_NOSUPPORT |
1409 #endif
1410                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1411                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1412                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1413 }
1414 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1415
1416 struct ring_buffer_event *
1417 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1418                           int type,
1419                           unsigned long len,
1420                           unsigned long flags, int pc)
1421 {
1422         struct ring_buffer_event *event;
1423
1424         event = ring_buffer_lock_reserve(buffer, len);
1425         if (event != NULL) {
1426                 struct trace_entry *ent = ring_buffer_event_data(event);
1427
1428                 tracing_generic_entry_update(ent, flags, pc);
1429                 ent->type = type;
1430         }
1431
1432         return event;
1433 }
1434
1435 void
1436 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1437 {
1438         __this_cpu_write(trace_cmdline_save, true);
1439         ring_buffer_unlock_commit(buffer, event);
1440 }
1441
1442 static inline void
1443 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1444                              struct ring_buffer_event *event,
1445                              unsigned long flags, int pc)
1446 {
1447         __buffer_unlock_commit(buffer, event);
1448
1449         ftrace_trace_stack(buffer, flags, 6, pc);
1450         ftrace_trace_userstack(buffer, flags, pc);
1451 }
1452
1453 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1454                                 struct ring_buffer_event *event,
1455                                 unsigned long flags, int pc)
1456 {
1457         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1458 }
1459 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1460
1461 struct ring_buffer_event *
1462 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1463                           struct ftrace_event_file *ftrace_file,
1464                           int type, unsigned long len,
1465                           unsigned long flags, int pc)
1466 {
1467         *current_rb = ftrace_file->tr->trace_buffer.buffer;
1468         return trace_buffer_lock_reserve(*current_rb,
1469                                          type, len, flags, pc);
1470 }
1471 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1472
1473 struct ring_buffer_event *
1474 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1475                                   int type, unsigned long len,
1476                                   unsigned long flags, int pc)
1477 {
1478         *current_rb = global_trace.trace_buffer.buffer;
1479         return trace_buffer_lock_reserve(*current_rb,
1480                                          type, len, flags, pc);
1481 }
1482 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1483
1484 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1485                                         struct ring_buffer_event *event,
1486                                         unsigned long flags, int pc)
1487 {
1488         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1489 }
1490 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1491
1492 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1493                                      struct ring_buffer_event *event,
1494                                      unsigned long flags, int pc,
1495                                      struct pt_regs *regs)
1496 {
1497         __buffer_unlock_commit(buffer, event);
1498
1499         ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1500         ftrace_trace_userstack(buffer, flags, pc);
1501 }
1502 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1503
1504 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1505                                          struct ring_buffer_event *event)
1506 {
1507         ring_buffer_discard_commit(buffer, event);
1508 }
1509 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1510
1511 void
1512 trace_function(struct trace_array *tr,
1513                unsigned long ip, unsigned long parent_ip, unsigned long flags,
1514                int pc)
1515 {
1516         struct ftrace_event_call *call = &event_function;
1517         struct ring_buffer *buffer = tr->trace_buffer.buffer;
1518         struct ring_buffer_event *event;
1519         struct ftrace_entry *entry;
1520
1521         /* If we are reading the ring buffer, don't trace */
1522         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1523                 return;
1524
1525         event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1526                                           flags, pc);
1527         if (!event)
1528                 return;
1529         entry   = ring_buffer_event_data(event);
1530         entry->ip                       = ip;
1531         entry->parent_ip                = parent_ip;
1532
1533         if (!filter_check_discard(call, entry, buffer, event))
1534                 __buffer_unlock_commit(buffer, event);
1535 }
1536
1537 void
1538 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1539        unsigned long ip, unsigned long parent_ip, unsigned long flags,
1540        int pc)
1541 {
1542         if (likely(!atomic_read(&data->disabled)))
1543                 trace_function(tr, ip, parent_ip, flags, pc);
1544 }
1545
1546 #ifdef CONFIG_STACKTRACE
1547
1548 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1549 struct ftrace_stack {
1550         unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
1551 };
1552
1553 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1554 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1555
1556 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1557                                  unsigned long flags,
1558                                  int skip, int pc, struct pt_regs *regs)
1559 {
1560         struct ftrace_event_call *call = &event_kernel_stack;
1561         struct ring_buffer_event *event;
1562         struct stack_entry *entry;
1563         struct stack_trace trace;
1564         int use_stack;
1565         int size = FTRACE_STACK_ENTRIES;
1566
1567         trace.nr_entries        = 0;
1568         trace.skip              = skip;
1569
1570         /*
1571          * Since events can happen in NMIs there's no safe way to
1572          * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1573          * or NMI comes in, it will just have to use the default
1574          * FTRACE_STACK_SIZE.
1575          */
1576         preempt_disable_notrace();
1577
1578         use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1579         /*
1580          * We don't need any atomic variables, just a barrier.
1581          * If an interrupt comes in, we don't care, because it would
1582          * have exited and put the counter back to what we want.
1583          * We just need a barrier to keep gcc from moving things
1584          * around.
1585          */
1586         barrier();
1587         if (use_stack == 1) {
1588                 trace.entries           = &__get_cpu_var(ftrace_stack).calls[0];
1589                 trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
1590
1591                 if (regs)
1592                         save_stack_trace_regs(regs, &trace);
1593                 else
1594                         save_stack_trace(&trace);
1595
1596                 if (trace.nr_entries > size)
1597                         size = trace.nr_entries;
1598         } else
1599                 /* From now on, use_stack is a boolean */
1600                 use_stack = 0;
1601
1602         size *= sizeof(unsigned long);
1603
1604         event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1605                                           sizeof(*entry) + size, flags, pc);
1606         if (!event)
1607                 goto out;
1608         entry = ring_buffer_event_data(event);
1609
1610         memset(&entry->caller, 0, size);
1611
1612         if (use_stack)
1613                 memcpy(&entry->caller, trace.entries,
1614                        trace.nr_entries * sizeof(unsigned long));
1615         else {
1616                 trace.max_entries       = FTRACE_STACK_ENTRIES;
1617                 trace.entries           = entry->caller;
1618                 if (regs)
1619                         save_stack_trace_regs(regs, &trace);
1620                 else
1621                         save_stack_trace(&trace);
1622         }
1623
1624         entry->size = trace.nr_entries;
1625
1626         if (!filter_check_discard(call, entry, buffer, event))
1627                 __buffer_unlock_commit(buffer, event);
1628
1629  out:
1630         /* Again, don't let gcc optimize things here */
1631         barrier();
1632         __this_cpu_dec(ftrace_stack_reserve);
1633         preempt_enable_notrace();
1634
1635 }
1636
1637 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1638                              int skip, int pc, struct pt_regs *regs)
1639 {
1640         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1641                 return;
1642
1643         __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1644 }
1645
1646 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1647                         int skip, int pc)
1648 {
1649         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1650                 return;
1651
1652         __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1653 }
1654
1655 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1656                    int pc)
1657 {
1658         __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1659 }
1660
1661 /**
1662  * trace_dump_stack - record a stack back trace in the trace buffer
1663  * @skip: Number of functions to skip (helper handlers)
1664  */
1665 void trace_dump_stack(int skip)
1666 {
1667         unsigned long flags;
1668
1669         if (tracing_disabled || tracing_selftest_running)
1670                 return;
1671
1672         local_save_flags(flags);
1673
1674         /*
1675          * Skip 3 more, seems to get us at the caller of
1676          * this function.
1677          */
1678         skip += 3;
1679         __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1680                              flags, skip, preempt_count(), NULL);
1681 }
1682
1683 static DEFINE_PER_CPU(int, user_stack_count);
1684
1685 void
1686 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1687 {
1688         struct ftrace_event_call *call = &event_user_stack;
1689         struct ring_buffer_event *event;
1690         struct userstack_entry *entry;
1691         struct stack_trace trace;
1692
1693         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1694                 return;
1695
1696         /*
1697          * NMIs can not handle page faults, even with fix ups.
1698          * The save user stack can (and often does) fault.
1699          */
1700         if (unlikely(in_nmi()))
1701                 return;
1702
1703         /*
1704          * prevent recursion, since the user stack tracing may
1705          * trigger other kernel events.
1706          */
1707         preempt_disable();
1708         if (__this_cpu_read(user_stack_count))
1709                 goto out;
1710
1711         __this_cpu_inc(user_stack_count);
1712
1713         event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1714                                           sizeof(*entry), flags, pc);
1715         if (!event)
1716                 goto out_drop_count;
1717         entry   = ring_buffer_event_data(event);
1718
1719         entry->tgid             = current->tgid;
1720         memset(&entry->caller, 0, sizeof(entry->caller));
1721
1722         trace.nr_entries        = 0;
1723         trace.max_entries       = FTRACE_STACK_ENTRIES;
1724         trace.skip              = 0;
1725         trace.entries           = entry->caller;
1726
1727         save_stack_trace_user(&trace);
1728         if (!filter_check_discard(call, entry, buffer, event))
1729                 __buffer_unlock_commit(buffer, event);
1730
1731  out_drop_count:
1732         __this_cpu_dec(user_stack_count);
1733  out:
1734         preempt_enable();
1735 }
1736
1737 #ifdef UNUSED
1738 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1739 {
1740         ftrace_trace_userstack(tr, flags, preempt_count());
1741 }
1742 #endif /* UNUSED */
1743
1744 #endif /* CONFIG_STACKTRACE */
1745
1746 /* created for use with alloc_percpu */
1747 struct trace_buffer_struct {
1748         char buffer[TRACE_BUF_SIZE];
1749 };
1750
1751 static struct trace_buffer_struct *trace_percpu_buffer;
1752 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1753 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1754 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1755
1756 /*
1757  * The buffer used is dependent on the context. There is a per cpu
1758  * buffer for normal context, softirq contex, hard irq context and
1759  * for NMI context. Thise allows for lockless recording.
1760  *
1761  * Note, if the buffers failed to be allocated, then this returns NULL
1762  */
1763 static char *get_trace_buf(void)
1764 {
1765         struct trace_buffer_struct *percpu_buffer;
1766
1767         /*
1768          * If we have allocated per cpu buffers, then we do not
1769          * need to do any locking.
1770          */
1771         if (in_nmi())
1772                 percpu_buffer = trace_percpu_nmi_buffer;
1773         else if (in_irq())
1774                 percpu_buffer = trace_percpu_irq_buffer;
1775         else if (in_softirq())
1776                 percpu_buffer = trace_percpu_sirq_buffer;
1777         else
1778                 percpu_buffer = trace_percpu_buffer;
1779
1780         if (!percpu_buffer)
1781                 return NULL;
1782
1783         return this_cpu_ptr(&percpu_buffer->buffer[0]);
1784 }
1785
1786 static int alloc_percpu_trace_buffer(void)
1787 {
1788         struct trace_buffer_struct *buffers;
1789         struct trace_buffer_struct *sirq_buffers;
1790         struct trace_buffer_struct *irq_buffers;
1791         struct trace_buffer_struct *nmi_buffers;
1792
1793         buffers = alloc_percpu(struct trace_buffer_struct);
1794         if (!buffers)
1795                 goto err_warn;
1796
1797         sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1798         if (!sirq_buffers)
1799                 goto err_sirq;
1800
1801         irq_buffers = alloc_percpu(struct trace_buffer_struct);
1802         if (!irq_buffers)
1803                 goto err_irq;
1804
1805         nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1806         if (!nmi_buffers)
1807                 goto err_nmi;
1808
1809         trace_percpu_buffer = buffers;
1810         trace_percpu_sirq_buffer = sirq_buffers;
1811         trace_percpu_irq_buffer = irq_buffers;
1812         trace_percpu_nmi_buffer = nmi_buffers;
1813
1814         return 0;
1815
1816  err_nmi:
1817         free_percpu(irq_buffers);
1818  err_irq:
1819         free_percpu(sirq_buffers);
1820  err_sirq:
1821         free_percpu(buffers);
1822  err_warn:
1823         WARN(1, "Could not allocate percpu trace_printk buffer");
1824         return -ENOMEM;
1825 }
1826
1827 static int buffers_allocated;
1828
1829 void trace_printk_init_buffers(void)
1830 {
1831         if (buffers_allocated)
1832                 return;
1833
1834         if (alloc_percpu_trace_buffer())
1835                 return;
1836
1837         pr_info("ftrace: Allocated trace_printk buffers\n");
1838
1839         /* Expand the buffers to set size */
1840         tracing_update_buffers();
1841
1842         buffers_allocated = 1;
1843
1844         /*
1845          * trace_printk_init_buffers() can be called by modules.
1846          * If that happens, then we need to start cmdline recording
1847          * directly here. If the global_trace.buffer is already
1848          * allocated here, then this was called by module code.
1849          */
1850         if (global_trace.trace_buffer.buffer)
1851                 tracing_start_cmdline_record();
1852 }
1853
1854 void trace_printk_start_comm(void)
1855 {
1856         /* Start tracing comms if trace printk is set */
1857         if (!buffers_allocated)
1858                 return;
1859         tracing_start_cmdline_record();
1860 }
1861
1862 static void trace_printk_start_stop_comm(int enabled)
1863 {
1864         if (!buffers_allocated)
1865                 return;
1866
1867         if (enabled)
1868                 tracing_start_cmdline_record();
1869         else
1870                 tracing_stop_cmdline_record();
1871 }
1872
1873 /**
1874  * trace_vbprintk - write binary msg to tracing buffer
1875  *
1876  */
1877 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1878 {
1879         struct ftrace_event_call *call = &event_bprint;
1880         struct ring_buffer_event *event;
1881         struct ring_buffer *buffer;
1882         struct trace_array *tr = &global_trace;
1883         struct bprint_entry *entry;
1884         unsigned long flags;
1885         char *tbuffer;
1886         int len = 0, size, pc;
1887
1888         if (unlikely(tracing_selftest_running || tracing_disabled))
1889                 return 0;
1890
1891         /* Don't pollute graph traces with trace_vprintk internals */
1892         pause_graph_tracing();
1893
1894         pc = preempt_count();
1895         preempt_disable_notrace();
1896
1897         tbuffer = get_trace_buf();
1898         if (!tbuffer) {
1899                 len = 0;
1900                 goto out;
1901         }
1902
1903         len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1904
1905         if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1906                 goto out;
1907
1908         local_save_flags(flags);
1909         size = sizeof(*entry) + sizeof(u32) * len;
1910         buffer = tr->trace_buffer.buffer;
1911         event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1912                                           flags, pc);
1913         if (!event)
1914                 goto out;
1915         entry = ring_buffer_event_data(event);
1916         entry->ip                       = ip;
1917         entry->fmt                      = fmt;
1918
1919         memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1920         if (!filter_check_discard(call, entry, buffer, event)) {
1921                 __buffer_unlock_commit(buffer, event);
1922                 ftrace_trace_stack(buffer, flags, 6, pc);
1923         }
1924
1925 out:
1926         preempt_enable_notrace();
1927         unpause_graph_tracing();
1928
1929         return len;
1930 }
1931 EXPORT_SYMBOL_GPL(trace_vbprintk);
1932
1933 static int
1934 __trace_array_vprintk(struct ring_buffer *buffer,
1935                       unsigned long ip, const char *fmt, va_list args)
1936 {
1937         struct ftrace_event_call *call = &event_print;
1938         struct ring_buffer_event *event;
1939         int len = 0, size, pc;
1940         struct print_entry *entry;
1941         unsigned long flags;
1942         char *tbuffer;
1943
1944         if (tracing_disabled || tracing_selftest_running)
1945                 return 0;
1946
1947         /* Don't pollute graph traces with trace_vprintk internals */
1948         pause_graph_tracing();
1949
1950         pc = preempt_count();
1951         preempt_disable_notrace();
1952
1953
1954         tbuffer = get_trace_buf();
1955         if (!tbuffer) {
1956                 len = 0;
1957                 goto out;
1958         }
1959
1960         len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1961         if (len > TRACE_BUF_SIZE)
1962                 goto out;
1963
1964         local_save_flags(flags);
1965         size = sizeof(*entry) + len + 1;
1966         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1967                                           flags, pc);
1968         if (!event)
1969                 goto out;
1970         entry = ring_buffer_event_data(event);
1971         entry->ip = ip;
1972
1973         memcpy(&entry->buf, tbuffer, len);
1974         entry->buf[len] = '\0';
1975         if (!filter_check_discard(call, entry, buffer, event)) {
1976                 __buffer_unlock_commit(buffer, event);
1977                 ftrace_trace_stack(buffer, flags, 6, pc);
1978         }
1979  out:
1980         preempt_enable_notrace();
1981         unpause_graph_tracing();
1982
1983         return len;
1984 }
1985
1986 int trace_array_vprintk(struct trace_array *tr,
1987                         unsigned long ip, const char *fmt, va_list args)
1988 {
1989         return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
1990 }
1991
1992 int trace_array_printk(struct trace_array *tr,
1993                        unsigned long ip, const char *fmt, ...)
1994 {
1995         int ret;
1996         va_list ap;
1997
1998         if (!(trace_flags & TRACE_ITER_PRINTK))
1999                 return 0;
2000
2001         va_start(ap, fmt);
2002         ret = trace_array_vprintk(tr, ip, fmt, ap);
2003         va_end(ap);
2004         return ret;
2005 }
2006
2007 int trace_array_printk_buf(struct ring_buffer *buffer,
2008                            unsigned long ip, const char *fmt, ...)
2009 {
2010         int ret;
2011         va_list ap;
2012
2013         if (!(trace_flags & TRACE_ITER_PRINTK))
2014                 return 0;
2015
2016         va_start(ap, fmt);
2017         ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2018         va_end(ap);
2019         return ret;
2020 }
2021
2022 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2023 {
2024         return trace_array_vprintk(&global_trace, ip, fmt, args);
2025 }
2026 EXPORT_SYMBOL_GPL(trace_vprintk);
2027
2028 static void trace_iterator_increment(struct trace_iterator *iter)
2029 {
2030         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2031
2032         iter->idx++;
2033         if (buf_iter)
2034                 ring_buffer_read(buf_iter, NULL);
2035 }
2036
2037 static struct trace_entry *
2038 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2039                 unsigned long *lost_events)
2040 {
2041         struct ring_buffer_event *event;
2042         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2043
2044         if (buf_iter)
2045                 event = ring_buffer_iter_peek(buf_iter, ts);
2046         else
2047                 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2048                                          lost_events);
2049
2050         if (event) {
2051                 iter->ent_size = ring_buffer_event_length(event);
2052                 return ring_buffer_event_data(event);
2053         }
2054         iter->ent_size = 0;
2055         return NULL;
2056 }
2057
2058 static struct trace_entry *
2059 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2060                   unsigned long *missing_events, u64 *ent_ts)
2061 {
2062         struct ring_buffer *buffer = iter->trace_buffer->buffer;
2063         struct trace_entry *ent, *next = NULL;
2064         unsigned long lost_events = 0, next_lost = 0;
2065         int cpu_file = iter->cpu_file;
2066         u64 next_ts = 0, ts;
2067         int next_cpu = -1;
2068         int next_size = 0;
2069         int cpu;
2070
2071         /*
2072          * If we are in a per_cpu trace file, don't bother by iterating over
2073          * all cpu and peek directly.
2074          */
2075         if (cpu_file > RING_BUFFER_ALL_CPUS) {
2076                 if (ring_buffer_empty_cpu(buffer, cpu_file))
2077                         return NULL;
2078                 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2079                 if (ent_cpu)
2080                         *ent_cpu = cpu_file;
2081
2082                 return ent;
2083         }
2084
2085         for_each_tracing_cpu(cpu) {
2086
2087                 if (ring_buffer_empty_cpu(buffer, cpu))
2088                         continue;
2089
2090                 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2091
2092                 /*
2093                  * Pick the entry with the smallest timestamp:
2094                  */
2095                 if (ent && (!next || ts < next_ts)) {
2096                         next = ent;
2097                         next_cpu = cpu;
2098                         next_ts = ts;
2099                         next_lost = lost_events;
2100                         next_size = iter->ent_size;
2101                 }
2102         }
2103
2104         iter->ent_size = next_size;
2105
2106         if (ent_cpu)
2107                 *ent_cpu = next_cpu;
2108
2109         if (ent_ts)
2110                 *ent_ts = next_ts;
2111
2112         if (missing_events)
2113                 *missing_events = next_lost;
2114
2115         return next;
2116 }
2117
2118 /* Find the next real entry, without updating the iterator itself */
2119 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2120                                           int *ent_cpu, u64 *ent_ts)
2121 {
2122         return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2123 }
2124
2125 /* Find the next real entry, and increment the iterator to the next entry */
2126 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2127 {
2128         iter->ent = __find_next_entry(iter, &iter->cpu,
2129                                       &iter->lost_events, &iter->ts);
2130
2131         if (iter->ent)
2132                 trace_iterator_increment(iter);
2133
2134         return iter->ent ? iter : NULL;
2135 }
2136
2137 static void trace_consume(struct trace_iterator *iter)
2138 {
2139         ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2140                             &iter->lost_events);
2141 }
2142
2143 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2144 {
2145         struct trace_iterator *iter = m->private;
2146         int i = (int)*pos;
2147         void *ent;
2148
2149         WARN_ON_ONCE(iter->leftover);
2150
2151         (*pos)++;
2152
2153         /* can't go backwards */
2154         if (iter->idx > i)
2155                 return NULL;
2156
2157         if (iter->idx < 0)
2158                 ent = trace_find_next_entry_inc(iter);
2159         else
2160                 ent = iter;
2161
2162         while (ent && iter->idx < i)
2163                 ent = trace_find_next_entry_inc(iter);
2164
2165         iter->pos = *pos;
2166
2167         return ent;
2168 }
2169
2170 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2171 {
2172         struct ring_buffer_event *event;
2173         struct ring_buffer_iter *buf_iter;
2174         unsigned long entries = 0;
2175         u64 ts;
2176
2177         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2178
2179         buf_iter = trace_buffer_iter(iter, cpu);
2180         if (!buf_iter)
2181                 return;
2182
2183         ring_buffer_iter_reset(buf_iter);
2184
2185         /*
2186          * We could have the case with the max latency tracers
2187          * that a reset never took place on a cpu. This is evident
2188          * by the timestamp being before the start of the buffer.
2189          */
2190         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2191                 if (ts >= iter->trace_buffer->time_start)
2192                         break;
2193                 entries++;
2194                 ring_buffer_read(buf_iter, NULL);
2195         }
2196
2197         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2198 }
2199
2200 /*
2201  * The current tracer is copied to avoid a global locking
2202  * all around.
2203  */
2204 static void *s_start(struct seq_file *m, loff_t *pos)
2205 {
2206         struct trace_iterator *iter = m->private;
2207         struct trace_array *tr = iter->tr;
2208         int cpu_file = iter->cpu_file;
2209         void *p = NULL;
2210         loff_t l = 0;
2211         int cpu;
2212
2213         /*
2214          * copy the tracer to avoid using a global lock all around.
2215          * iter->trace is a copy of current_trace, the pointer to the
2216          * name may be used instead of a strcmp(), as iter->trace->name
2217          * will point to the same string as current_trace->name.
2218          */
2219         mutex_lock(&trace_types_lock);
2220         if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2221                 *iter->trace = *tr->current_trace;
2222         mutex_unlock(&trace_types_lock);
2223
2224 #ifdef CONFIG_TRACER_MAX_TRACE
2225         if (iter->snapshot && iter->trace->use_max_tr)
2226                 return ERR_PTR(-EBUSY);
2227 #endif
2228
2229         if (!iter->snapshot)
2230                 atomic_inc(&trace_record_cmdline_disabled);
2231
2232         if (*pos != iter->pos) {
2233                 iter->ent = NULL;
2234                 iter->cpu = 0;
2235                 iter->idx = -1;
2236
2237                 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2238                         for_each_tracing_cpu(cpu)
2239                                 tracing_iter_reset(iter, cpu);
2240                 } else
2241                         tracing_iter_reset(iter, cpu_file);
2242
2243                 iter->leftover = 0;
2244                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2245                         ;
2246
2247         } else {
2248                 /*
2249                  * If we overflowed the seq_file before, then we want
2250                  * to just reuse the trace_seq buffer again.
2251                  */
2252                 if (iter->leftover)
2253                         p = iter;
2254                 else {
2255                         l = *pos - 1;
2256                         p = s_next(m, p, &l);
2257                 }
2258         }
2259
2260         trace_event_read_lock();
2261         trace_access_lock(cpu_file);
2262         return p;
2263 }
2264
2265 static void s_stop(struct seq_file *m, void *p)
2266 {
2267         struct trace_iterator *iter = m->private;
2268
2269 #ifdef CONFIG_TRACER_MAX_TRACE
2270         if (iter->snapshot && iter->trace->use_max_tr)
2271                 return;
2272 #endif
2273
2274         if (!iter->snapshot)
2275                 atomic_dec(&trace_record_cmdline_disabled);
2276
2277         trace_access_unlock(iter->cpu_file);
2278         trace_event_read_unlock();
2279 }
2280
2281 static void
2282 get_total_entries(struct trace_buffer *buf,
2283                   unsigned long *total, unsigned long *entries)
2284 {
2285         unsigned long count;
2286         int cpu;
2287
2288         *total = 0;
2289         *entries = 0;
2290
2291         for_each_tracing_cpu(cpu) {
2292                 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2293                 /*
2294                  * If this buffer has skipped entries, then we hold all
2295                  * entries for the trace and we need to ignore the
2296                  * ones before the time stamp.
2297                  */
2298                 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2299                         count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2300                         /* total is the same as the entries */
2301                         *total += count;
2302                 } else
2303                         *total += count +
2304                                 ring_buffer_overrun_cpu(buf->buffer, cpu);
2305                 *entries += count;
2306         }
2307 }
2308
2309 static void print_lat_help_header(struct seq_file *m)
2310 {
2311         seq_puts(m, "#                  _------=> CPU#            \n");
2312         seq_puts(m, "#                 / _-----=> irqs-off        \n");
2313         seq_puts(m, "#                | / _----=> need-resched    \n");
2314         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
2315         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
2316         seq_puts(m, "#                |||| /     delay             \n");
2317         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
2318         seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
2319 }
2320
2321 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2322 {
2323         unsigned long total;
2324         unsigned long entries;
2325
2326         get_total_entries(buf, &total, &entries);
2327         seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
2328                    entries, total, num_online_cpus());
2329         seq_puts(m, "#\n");
2330 }
2331
2332 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2333 {
2334         print_event_info(buf, m);
2335         seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2336         seq_puts(m, "#              | |       |          |         |\n");
2337 }
2338
2339 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2340 {
2341         print_event_info(buf, m);
2342         seq_puts(m, "#                              _-----=> irqs-off\n");
2343         seq_puts(m, "#                             / _----=> need-resched\n");
2344         seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
2345         seq_puts(m, "#                            || / _--=> preempt-depth\n");
2346         seq_puts(m, "#                            ||| /     delay\n");
2347         seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
2348         seq_puts(m, "#              | |       |   ||||       |         |\n");
2349 }
2350
2351 void
2352 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2353 {
2354         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2355         struct trace_buffer *buf = iter->trace_buffer;
2356         struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2357         struct tracer *type = iter->trace;
2358         unsigned long entries;
2359         unsigned long total;
2360         const char *name = "preemption";
2361
2362         name = type->name;
2363
2364         get_total_entries(buf, &total, &entries);
2365
2366         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2367                    name, UTS_RELEASE);
2368         seq_puts(m, "# -----------------------------------"
2369                  "---------------------------------\n");
2370         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2371                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2372                    nsecs_to_usecs(data->saved_latency),
2373                    entries,
2374                    total,
2375                    buf->cpu,
2376 #if defined(CONFIG_PREEMPT_NONE)
2377                    "server",
2378 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2379                    "desktop",
2380 #elif defined(CONFIG_PREEMPT)
2381                    "preempt",
2382 #else
2383                    "unknown",
2384 #endif
2385                    /* These are reserved for later use */
2386                    0, 0, 0, 0);
2387 #ifdef CONFIG_SMP
2388         seq_printf(m, " #P:%d)\n", num_online_cpus());
2389 #else
2390         seq_puts(m, ")\n");
2391 #endif
2392         seq_puts(m, "#    -----------------\n");
2393         seq_printf(m, "#    | task: %.16s-%d "
2394                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2395                    data->comm, data->pid,
2396                    from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2397                    data->policy, data->rt_priority);
2398         seq_puts(m, "#    -----------------\n");
2399
2400         if (data->critical_start) {
2401                 seq_puts(m, "#  => started at: ");
2402                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2403                 trace_print_seq(m, &iter->seq);
2404                 seq_puts(m, "\n#  => ended at:   ");
2405                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2406                 trace_print_seq(m, &iter->seq);
2407                 seq_puts(m, "\n#\n");
2408         }
2409
2410         seq_puts(m, "#\n");
2411 }
2412
2413 static void test_cpu_buff_start(struct trace_iterator *iter)
2414 {
2415         struct trace_seq *s = &iter->seq;
2416
2417         if (!(trace_flags & TRACE_ITER_ANNOTATE))
2418                 return;
2419
2420         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2421                 return;
2422
2423         if (cpumask_test_cpu(iter->cpu, iter->started))
2424                 return;
2425
2426         if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2427                 return;
2428
2429         cpumask_set_cpu(iter->cpu, iter->started);
2430
2431         /* Don't print started cpu buffer for the first entry of the trace */
2432         if (iter->idx > 1)
2433                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2434                                 iter->cpu);
2435 }
2436
2437 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2438 {
2439         struct trace_seq *s = &iter->seq;
2440         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2441         struct trace_entry *entry;
2442         struct trace_event *event;
2443
2444         entry = iter->ent;
2445
2446         test_cpu_buff_start(iter);
2447
2448         event = ftrace_find_event(entry->type);
2449
2450         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2451                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2452                         if (!trace_print_lat_context(iter))
2453                                 goto partial;
2454                 } else {
2455                         if (!trace_print_context(iter))
2456                                 goto partial;
2457                 }
2458         }
2459
2460         if (event)
2461                 return event->funcs->trace(iter, sym_flags, event);
2462
2463         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2464                 goto partial;
2465
2466         return TRACE_TYPE_HANDLED;
2467 partial:
2468         return TRACE_TYPE_PARTIAL_LINE;
2469 }
2470
2471 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2472 {
2473         struct trace_seq *s = &iter->seq;
2474         struct trace_entry *entry;
2475         struct trace_event *event;
2476
2477         entry = iter->ent;
2478
2479         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2480                 if (!trace_seq_printf(s, "%d %d %llu ",
2481                                       entry->pid, iter->cpu, iter->ts))
2482                         goto partial;
2483         }
2484
2485         event = ftrace_find_event(entry->type);
2486         if (event)
2487                 return event->funcs->raw(iter, 0, event);
2488
2489         if (!trace_seq_printf(s, "%d ?\n", entry->type))
2490                 goto partial;
2491
2492         return TRACE_TYPE_HANDLED;
2493 partial:
2494         return TRACE_TYPE_PARTIAL_LINE;
2495 }
2496
2497 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2498 {
2499         struct trace_seq *s = &iter->seq;
2500         unsigned char newline = '\n';
2501         struct trace_entry *entry;
2502         struct trace_event *event;
2503
2504         entry = iter->ent;
2505
2506         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2507                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2508                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2509                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2510         }
2511
2512         event = ftrace_find_event(entry->type);
2513         if (event) {
2514                 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2515                 if (ret != TRACE_TYPE_HANDLED)
2516                         return ret;
2517         }
2518
2519         SEQ_PUT_FIELD_RET(s, newline);
2520
2521         return TRACE_TYPE_HANDLED;
2522 }
2523
2524 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2525 {
2526         struct trace_seq *s = &iter->seq;
2527         struct trace_entry *entry;
2528         struct trace_event *event;
2529
2530         entry = iter->ent;
2531
2532         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2533                 SEQ_PUT_FIELD_RET(s, entry->pid);
2534                 SEQ_PUT_FIELD_RET(s, iter->cpu);
2535                 SEQ_PUT_FIELD_RET(s, iter->ts);
2536         }
2537
2538         event = ftrace_find_event(entry->type);
2539         return event ? event->funcs->binary(iter, 0, event) :
2540                 TRACE_TYPE_HANDLED;
2541 }
2542
2543 int trace_empty(struct trace_iterator *iter)
2544 {
2545         struct ring_buffer_iter *buf_iter;
2546         int cpu;
2547
2548         /* If we are looking at one CPU buffer, only check that one */
2549         if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2550                 cpu = iter->cpu_file;
2551                 buf_iter = trace_buffer_iter(iter, cpu);
2552                 if (buf_iter) {
2553                         if (!ring_buffer_iter_empty(buf_iter))
2554                                 return 0;
2555                 } else {
2556                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2557                                 return 0;
2558                 }
2559                 return 1;
2560         }
2561
2562         for_each_tracing_cpu(cpu) {
2563                 buf_iter = trace_buffer_iter(iter, cpu);
2564                 if (buf_iter) {
2565                         if (!ring_buffer_iter_empty(buf_iter))
2566                                 return 0;
2567                 } else {
2568                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2569                                 return 0;
2570                 }
2571         }
2572
2573         return 1;
2574 }
2575
2576 /*  Called with trace_event_read_lock() held. */
2577 enum print_line_t print_trace_line(struct trace_iterator *iter)
2578 {
2579         enum print_line_t ret;
2580
2581         if (iter->lost_events &&
2582             !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2583                                  iter->cpu, iter->lost_events))
2584                 return TRACE_TYPE_PARTIAL_LINE;
2585
2586         if (iter->trace && iter->trace->print_line) {
2587                 ret = iter->trace->print_line(iter);
2588                 if (ret != TRACE_TYPE_UNHANDLED)
2589                         return ret;
2590         }
2591
2592         if (iter->ent->type == TRACE_BPUTS &&
2593                         trace_flags & TRACE_ITER_PRINTK &&
2594                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2595                 return trace_print_bputs_msg_only(iter);
2596
2597         if (iter->ent->type == TRACE_BPRINT &&
2598                         trace_flags & TRACE_ITER_PRINTK &&
2599                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2600                 return trace_print_bprintk_msg_only(iter);
2601
2602         if (iter->ent->type == TRACE_PRINT &&
2603                         trace_flags & TRACE_ITER_PRINTK &&
2604                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2605                 return trace_print_printk_msg_only(iter);
2606
2607         if (trace_flags & TRACE_ITER_BIN)
2608                 return print_bin_fmt(iter);
2609
2610         if (trace_flags & TRACE_ITER_HEX)
2611                 return print_hex_fmt(iter);
2612
2613         if (trace_flags & TRACE_ITER_RAW)
2614                 return print_raw_fmt(iter);
2615
2616         return print_trace_fmt(iter);
2617 }
2618
2619 void trace_latency_header(struct seq_file *m)
2620 {
2621         struct trace_iterator *iter = m->private;
2622
2623         /* print nothing if the buffers are empty */
2624         if (trace_empty(iter))
2625                 return;
2626
2627         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2628                 print_trace_header(m, iter);
2629
2630         if (!(trace_flags & TRACE_ITER_VERBOSE))
2631                 print_lat_help_header(m);
2632 }
2633
2634 void trace_default_header(struct seq_file *m)
2635 {
2636         struct trace_iterator *iter = m->private;
2637
2638         if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2639                 return;
2640
2641         if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2642                 /* print nothing if the buffers are empty */
2643                 if (trace_empty(iter))
2644                         return;
2645                 print_trace_header(m, iter);
2646                 if (!(trace_flags & TRACE_ITER_VERBOSE))
2647                         print_lat_help_header(m);
2648         } else {
2649                 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2650                         if (trace_flags & TRACE_ITER_IRQ_INFO)
2651                                 print_func_help_header_irq(iter->trace_buffer, m);
2652                         else
2653                                 print_func_help_header(iter->trace_buffer, m);
2654                 }
2655         }
2656 }
2657
2658 static void test_ftrace_alive(struct seq_file *m)
2659 {
2660         if (!ftrace_is_dead())
2661                 return;
2662         seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2663         seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
2664 }
2665
2666 #ifdef CONFIG_TRACER_MAX_TRACE
2667 static void show_snapshot_main_help(struct seq_file *m)
2668 {
2669         seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2670         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2671         seq_printf(m, "#                      Takes a snapshot of the main buffer.\n");
2672         seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2673         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2674         seq_printf(m, "#                       is not a '0' or '1')\n");
2675 }
2676
2677 static void show_snapshot_percpu_help(struct seq_file *m)
2678 {
2679         seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2680 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2681         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2682         seq_printf(m, "#                      Takes a snapshot of the main buffer for this cpu.\n");
2683 #else
2684         seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2685         seq_printf(m, "#                     Must use main snapshot file to allocate.\n");
2686 #endif
2687         seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2688         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2689         seq_printf(m, "#                       is not a '0' or '1')\n");
2690 }
2691
2692 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2693 {
2694         if (iter->tr->allocated_snapshot)
2695                 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2696         else
2697                 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2698
2699         seq_printf(m, "# Snapshot commands:\n");
2700         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2701                 show_snapshot_main_help(m);
2702         else
2703                 show_snapshot_percpu_help(m);
2704 }
2705 #else
2706 /* Should never be called */
2707 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2708 #endif
2709
2710 static int s_show(struct seq_file *m, void *v)
2711 {
2712         struct trace_iterator *iter = v;
2713         int ret;
2714
2715         if (iter->ent == NULL) {
2716                 if (iter->tr) {
2717                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
2718                         seq_puts(m, "#\n");
2719                         test_ftrace_alive(m);
2720                 }
2721                 if (iter->snapshot && trace_empty(iter))
2722                         print_snapshot_help(m, iter);
2723                 else if (iter->trace && iter->trace->print_header)
2724                         iter->trace->print_header(m);
2725                 else
2726                         trace_default_header(m);
2727
2728         } else if (iter->leftover) {
2729                 /*
2730                  * If we filled the seq_file buffer earlier, we
2731                  * want to just show it now.
2732                  */
2733                 ret = trace_print_seq(m, &iter->seq);
2734
2735                 /* ret should this time be zero, but you never know */
2736                 iter->leftover = ret;
2737
2738         } else {
2739                 print_trace_line(iter);
2740                 ret = trace_print_seq(m, &iter->seq);
2741                 /*
2742                  * If we overflow the seq_file buffer, then it will
2743                  * ask us for this data again at start up.
2744                  * Use that instead.
2745                  *  ret is 0 if seq_file write succeeded.
2746                  *        -1 otherwise.
2747                  */
2748                 iter->leftover = ret;
2749         }
2750
2751         return 0;
2752 }
2753
2754 static const struct seq_operations tracer_seq_ops = {
2755         .start          = s_start,
2756         .next           = s_next,
2757         .stop           = s_stop,
2758         .show           = s_show,
2759 };
2760
2761 static struct trace_iterator *
2762 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2763 {
2764         struct trace_cpu *tc = inode->i_private;
2765         struct trace_array *tr = tc->tr;
2766         struct trace_iterator *iter;
2767         int cpu;
2768
2769         if (tracing_disabled)
2770                 return ERR_PTR(-ENODEV);
2771
2772         iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2773         if (!iter)
2774                 return ERR_PTR(-ENOMEM);
2775
2776         iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2777                                     GFP_KERNEL);
2778         if (!iter->buffer_iter)
2779                 goto release;
2780
2781         /*
2782          * We make a copy of the current tracer to avoid concurrent
2783          * changes on it while we are reading.
2784          */
2785         mutex_lock(&trace_types_lock);
2786         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2787         if (!iter->trace)
2788                 goto fail;
2789
2790         *iter->trace = *tr->current_trace;
2791
2792         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2793                 goto fail;
2794
2795         iter->tr = tr;
2796
2797 #ifdef CONFIG_TRACER_MAX_TRACE
2798         /* Currently only the top directory has a snapshot */
2799         if (tr->current_trace->print_max || snapshot)
2800                 iter->trace_buffer = &tr->max_buffer;
2801         else
2802 #endif
2803                 iter->trace_buffer = &tr->trace_buffer;
2804         iter->snapshot = snapshot;
2805         iter->pos = -1;
2806         mutex_init(&iter->mutex);
2807         iter->cpu_file = tc->cpu;
2808
2809         /* Notify the tracer early; before we stop tracing. */
2810         if (iter->trace && iter->trace->open)
2811                 iter->trace->open(iter);
2812
2813         /* Annotate start of buffers if we had overruns */
2814         if (ring_buffer_overruns(iter->trace_buffer->buffer))
2815                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2816
2817         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2818         if (trace_clocks[trace_clock_id].in_ns)
2819                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2820
2821         /* stop the trace while dumping if we are not opening "snapshot" */
2822         if (!iter->snapshot)
2823                 tracing_stop_tr(tr);
2824
2825         if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2826                 for_each_tracing_cpu(cpu) {
2827                         iter->buffer_iter[cpu] =
2828                                 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2829                 }
2830                 ring_buffer_read_prepare_sync();
2831                 for_each_tracing_cpu(cpu) {
2832                         ring_buffer_read_start(iter->buffer_iter[cpu]);
2833                         tracing_iter_reset(iter, cpu);
2834                 }
2835         } else {
2836                 cpu = iter->cpu_file;
2837                 iter->buffer_iter[cpu] =
2838                         ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2839                 ring_buffer_read_prepare_sync();
2840                 ring_buffer_read_start(iter->buffer_iter[cpu]);
2841                 tracing_iter_reset(iter, cpu);
2842         }
2843
2844         tr->ref++;
2845
2846         mutex_unlock(&trace_types_lock);
2847
2848         return iter;
2849
2850  fail:
2851         mutex_unlock(&trace_types_lock);
2852         kfree(iter->trace);
2853         kfree(iter->buffer_iter);
2854 release:
2855         seq_release_private(inode, file);
2856         return ERR_PTR(-ENOMEM);
2857 }
2858
2859 int tracing_open_generic(struct inode *inode, struct file *filp)
2860 {
2861         if (tracing_disabled)
2862                 return -ENODEV;
2863
2864         filp->private_data = inode->i_private;
2865         return 0;
2866 }
2867
2868 static int tracing_release(struct inode *inode, struct file *file)
2869 {
2870         struct seq_file *m = file->private_data;
2871         struct trace_iterator *iter;
2872         struct trace_array *tr;
2873         int cpu;
2874
2875         if (!(file->f_mode & FMODE_READ))
2876                 return 0;
2877
2878         iter = m->private;
2879         tr = iter->tr;
2880
2881         mutex_lock(&trace_types_lock);
2882
2883         WARN_ON(!tr->ref);
2884         tr->ref--;
2885
2886         for_each_tracing_cpu(cpu) {
2887                 if (iter->buffer_iter[cpu])
2888                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2889         }
2890
2891         if (iter->trace && iter->trace->close)
2892                 iter->trace->close(iter);
2893
2894         if (!iter->snapshot)
2895                 /* reenable tracing if it was previously enabled */
2896                 tracing_start_tr(tr);
2897         mutex_unlock(&trace_types_lock);
2898
2899         mutex_destroy(&iter->mutex);
2900         free_cpumask_var(iter->started);
2901         kfree(iter->trace);
2902         kfree(iter->buffer_iter);
2903         seq_release_private(inode, file);
2904         return 0;
2905 }
2906
2907 static int tracing_open(struct inode *inode, struct file *file)
2908 {
2909         struct trace_iterator *iter;
2910         int ret = 0;
2911
2912         /* If this file was open for write, then erase contents */
2913         if ((file->f_mode & FMODE_WRITE) &&
2914             (file->f_flags & O_TRUNC)) {
2915                 struct trace_cpu *tc = inode->i_private;
2916                 struct trace_array *tr = tc->tr;
2917
2918                 if (tc->cpu == RING_BUFFER_ALL_CPUS)
2919                         tracing_reset_online_cpus(&tr->trace_buffer);
2920                 else
2921                         tracing_reset(&tr->trace_buffer, tc->cpu);
2922         }
2923
2924         if (file->f_mode & FMODE_READ) {
2925                 iter = __tracing_open(inode, file, false);
2926                 if (IS_ERR(iter))
2927                         ret = PTR_ERR(iter);
2928                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2929                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
2930         }
2931         return ret;
2932 }
2933
2934 static void *
2935 t_next(struct seq_file *m, void *v, loff_t *pos)
2936 {
2937         struct tracer *t = v;
2938
2939         (*pos)++;
2940
2941         if (t)
2942                 t = t->next;
2943
2944         return t;
2945 }
2946
2947 static void *t_start(struct seq_file *m, loff_t *pos)
2948 {
2949         struct tracer *t;
2950         loff_t l = 0;
2951
2952         mutex_lock(&trace_types_lock);
2953         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2954                 ;
2955
2956         return t;
2957 }
2958
2959 static void t_stop(struct seq_file *m, void *p)
2960 {
2961         mutex_unlock(&trace_types_lock);
2962 }
2963
2964 static int t_show(struct seq_file *m, void *v)
2965 {
2966         struct tracer *t = v;
2967
2968         if (!t)
2969                 return 0;
2970
2971         seq_printf(m, "%s", t->name);
2972         if (t->next)
2973                 seq_putc(m, ' ');
2974         else
2975                 seq_putc(m, '\n');
2976
2977         return 0;
2978 }
2979
2980 static const struct seq_operations show_traces_seq_ops = {
2981         .start          = t_start,
2982         .next           = t_next,
2983         .stop           = t_stop,
2984         .show           = t_show,
2985 };
2986
2987 static int show_traces_open(struct inode *inode, struct file *file)
2988 {
2989         if (tracing_disabled)
2990                 return -ENODEV;
2991
2992         return seq_open(file, &show_traces_seq_ops);
2993 }
2994
2995 static ssize_t
2996 tracing_write_stub(struct file *filp, const char __user *ubuf,
2997                    size_t count, loff_t *ppos)
2998 {
2999         return count;
3000 }
3001
3002 static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
3003 {
3004         if (file->f_mode & FMODE_READ)
3005                 return seq_lseek(file, offset, origin);
3006         else
3007                 return 0;
3008 }
3009
3010 static const struct file_operations tracing_fops = {
3011         .open           = tracing_open,
3012         .read           = seq_read,
3013         .write          = tracing_write_stub,
3014         .llseek         = tracing_seek,
3015         .release        = tracing_release,
3016 };
3017
3018 static const struct file_operations show_traces_fops = {
3019         .open           = show_traces_open,
3020         .read           = seq_read,
3021         .release        = seq_release,
3022         .llseek         = seq_lseek,
3023 };
3024
3025 /*
3026  * Only trace on a CPU if the bitmask is set:
3027  */
3028 static cpumask_var_t tracing_cpumask;
3029
3030 /*
3031  * The tracer itself will not take this lock, but still we want
3032  * to provide a consistent cpumask to user-space:
3033  */
3034 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3035
3036 /*
3037  * Temporary storage for the character representation of the
3038  * CPU bitmask (and one more byte for the newline):
3039  */
3040 static char mask_str[NR_CPUS + 1];
3041
3042 static ssize_t
3043 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3044                      size_t count, loff_t *ppos)
3045 {
3046         int len;
3047
3048         mutex_lock(&tracing_cpumask_update_lock);
3049
3050         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
3051         if (count - len < 2) {
3052                 count = -EINVAL;
3053                 goto out_err;
3054         }
3055         len += sprintf(mask_str + len, "\n");
3056         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3057
3058 out_err:
3059         mutex_unlock(&tracing_cpumask_update_lock);
3060
3061         return count;
3062 }
3063
3064 static ssize_t
3065 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3066                       size_t count, loff_t *ppos)
3067 {
3068         struct trace_array *tr = filp->private_data;
3069         cpumask_var_t tracing_cpumask_new;
3070         int err, cpu;
3071
3072         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3073                 return -ENOMEM;
3074
3075         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3076         if (err)
3077                 goto err_unlock;
3078
3079         mutex_lock(&tracing_cpumask_update_lock);
3080
3081         local_irq_disable();
3082         arch_spin_lock(&ftrace_max_lock);
3083         for_each_tracing_cpu(cpu) {
3084                 /*
3085                  * Increase/decrease the disabled counter if we are
3086                  * about to flip a bit in the cpumask:
3087                  */
3088                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
3089                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3090                         atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3091                         ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3092                 }
3093                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
3094                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3095                         atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3096                         ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3097                 }
3098         }
3099         arch_spin_unlock(&ftrace_max_lock);
3100         local_irq_enable();
3101
3102         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
3103
3104         mutex_unlock(&tracing_cpumask_update_lock);
3105         free_cpumask_var(tracing_cpumask_new);
3106
3107         return count;
3108
3109 err_unlock:
3110         free_cpumask_var(tracing_cpumask_new);
3111
3112         return err;
3113 }
3114
3115 static const struct file_operations tracing_cpumask_fops = {
3116         .open           = tracing_open_generic,
3117         .read           = tracing_cpumask_read,
3118         .write          = tracing_cpumask_write,
3119         .llseek         = generic_file_llseek,
3120 };
3121
3122 static int tracing_trace_options_show(struct seq_file *m, void *v)
3123 {
3124         struct tracer_opt *trace_opts;
3125         struct trace_array *tr = m->private;
3126         u32 tracer_flags;
3127         int i;
3128
3129         mutex_lock(&trace_types_lock);
3130         tracer_flags = tr->current_trace->flags->val;
3131         trace_opts = tr->current_trace->flags->opts;
3132
3133         for (i = 0; trace_options[i]; i++) {
3134                 if (trace_flags & (1 << i))
3135                         seq_printf(m, "%s\n", trace_options[i]);
3136                 else
3137                         seq_printf(m, "no%s\n", trace_options[i]);
3138         }
3139
3140         for (i = 0; trace_opts[i].name; i++) {
3141                 if (tracer_flags & trace_opts[i].bit)
3142                         seq_printf(m, "%s\n", trace_opts[i].name);
3143                 else
3144                         seq_printf(m, "no%s\n", trace_opts[i].name);
3145         }
3146         mutex_unlock(&trace_types_lock);
3147
3148         return 0;
3149 }
3150
3151 static int __set_tracer_option(struct tracer *trace,
3152                                struct tracer_flags *tracer_flags,
3153                                struct tracer_opt *opts, int neg)
3154 {
3155         int ret;
3156
3157         ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3158         if (ret)
3159                 return ret;
3160
3161         if (neg)
3162                 tracer_flags->val &= ~opts->bit;
3163         else
3164                 tracer_flags->val |= opts->bit;
3165         return 0;
3166 }
3167
3168 /* Try to assign a tracer specific option */
3169 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3170 {
3171         struct tracer_flags *tracer_flags = trace->flags;
3172         struct tracer_opt *opts = NULL;
3173         int i;
3174
3175         for (i = 0; tracer_flags->opts[i].name; i++) {
3176                 opts = &tracer_flags->opts[i];
3177
3178                 if (strcmp(cmp, opts->name) == 0)
3179                         return __set_tracer_option(trace, trace->flags,
3180                                                    opts, neg);
3181         }
3182
3183         return -EINVAL;
3184 }
3185
3186 /* Some tracers require overwrite to stay enabled */
3187 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3188 {
3189         if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3190                 return -1;
3191
3192         return 0;
3193 }
3194
3195 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3196 {
3197         /* do nothing if flag is already set */
3198         if (!!(trace_flags & mask) == !!enabled)
3199                 return 0;
3200
3201         /* Give the tracer a chance to approve the change */
3202         if (tr->current_trace->flag_changed)
3203                 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
3204                         return -EINVAL;
3205
3206         if (enabled)
3207                 trace_flags |= mask;
3208         else
3209                 trace_flags &= ~mask;
3210
3211         if (mask == TRACE_ITER_RECORD_CMD)
3212                 trace_event_enable_cmd_record(enabled);
3213
3214         if (mask == TRACE_ITER_OVERWRITE) {
3215                 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3216 #ifdef CONFIG_TRACER_MAX_TRACE
3217                 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3218 #endif
3219         }
3220
3221         if (mask == TRACE_ITER_PRINTK)
3222                 trace_printk_start_stop_comm(enabled);
3223
3224         return 0;
3225 }
3226
3227 static int trace_set_options(struct trace_array *tr, char *option)
3228 {
3229         char *cmp;
3230         int neg = 0;
3231         int ret = -ENODEV;
3232         int i;
3233
3234         cmp = strstrip(option);
3235
3236         if (strncmp(cmp, "no", 2) == 0) {
3237                 neg = 1;
3238                 cmp += 2;
3239         }
3240
3241         mutex_lock(&trace_types_lock);
3242
3243         for (i = 0; trace_options[i]; i++) {
3244                 if (strcmp(cmp, trace_options[i]) == 0) {
3245                         ret = set_tracer_flag(tr, 1 << i, !neg);
3246                         break;
3247                 }
3248         }
3249
3250         /* If no option could be set, test the specific tracer options */
3251         if (!trace_options[i])
3252                 ret = set_tracer_option(tr->current_trace, cmp, neg);
3253
3254         mutex_unlock(&trace_types_lock);
3255
3256         return ret;
3257 }
3258
3259 static ssize_t
3260 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3261                         size_t cnt, loff_t *ppos)
3262 {
3263         struct seq_file *m = filp->private_data;
3264         struct trace_array *tr = m->private;
3265         char buf[64];
3266         int ret;
3267
3268         if (cnt >= sizeof(buf))
3269                 return -EINVAL;
3270
3271         if (copy_from_user(&buf, ubuf, cnt))
3272                 return -EFAULT;
3273
3274         buf[cnt] = 0;
3275
3276         ret = trace_set_options(tr, buf);
3277         if (ret < 0)
3278                 return ret;
3279
3280         *ppos += cnt;
3281
3282         return cnt;
3283 }
3284
3285 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3286 {
3287         if (tracing_disabled)
3288                 return -ENODEV;
3289
3290         return single_open(file, tracing_trace_options_show, inode->i_private);
3291 }
3292
3293 static const struct file_operations tracing_iter_fops = {
3294         .open           = tracing_trace_options_open,
3295         .read           = seq_read,
3296         .llseek         = seq_lseek,
3297         .release        = single_release,
3298         .write          = tracing_trace_options_write,
3299 };
3300
3301 static const char readme_msg[] =
3302         "tracing mini-HOWTO:\n\n"
3303         "# mount -t debugfs nodev /sys/kernel/debug\n\n"
3304         "# cat /sys/kernel/debug/tracing/available_tracers\n"
3305         "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
3306         "# cat /sys/kernel/debug/tracing/current_tracer\n"
3307         "nop\n"
3308         "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
3309         "# cat /sys/kernel/debug/tracing/current_tracer\n"
3310         "wakeup\n"
3311         "# cat /sys/kernel/debug/tracing/trace_options\n"
3312         "noprint-parent nosym-offset nosym-addr noverbose\n"
3313         "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
3314         "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
3315         "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
3316         "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
3317 ;
3318
3319 static ssize_t
3320 tracing_readme_read(struct file *filp, char __user *ubuf,
3321                        size_t cnt, loff_t *ppos)
3322 {
3323         return simple_read_from_buffer(ubuf, cnt, ppos,
3324                                         readme_msg, strlen(readme_msg));
3325 }
3326
3327 static const struct file_operations tracing_readme_fops = {
3328         .open           = tracing_open_generic,
3329         .read           = tracing_readme_read,
3330         .llseek         = generic_file_llseek,
3331 };
3332
3333 static ssize_t
3334 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3335                                 size_t cnt, loff_t *ppos)
3336 {
3337         char *buf_comm;
3338         char *file_buf;
3339         char *buf;
3340         int len = 0;
3341         int pid;
3342         int i;
3343
3344         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3345         if (!file_buf)
3346                 return -ENOMEM;
3347
3348         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3349         if (!buf_comm) {
3350                 kfree(file_buf);
3351                 return -ENOMEM;
3352         }
3353
3354         buf = file_buf;
3355
3356         for (i = 0; i < SAVED_CMDLINES; i++) {
3357                 int r;
3358
3359                 pid = map_cmdline_to_pid[i];
3360                 if (pid == -1 || pid == NO_CMDLINE_MAP)
3361                         continue;
3362
3363                 trace_find_cmdline(pid, buf_comm);
3364                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3365                 buf += r;
3366                 len += r;
3367         }
3368
3369         len = simple_read_from_buffer(ubuf, cnt, ppos,
3370                                       file_buf, len);
3371
3372         kfree(file_buf);
3373         kfree(buf_comm);
3374
3375         return len;
3376 }
3377
3378 static const struct file_operations tracing_saved_cmdlines_fops = {
3379     .open       = tracing_open_generic,
3380     .read       = tracing_saved_cmdlines_read,
3381     .llseek     = generic_file_llseek,
3382 };
3383
3384 static ssize_t
3385 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3386                        size_t cnt, loff_t *ppos)
3387 {
3388         struct trace_array *tr = filp->private_data;
3389         char buf[MAX_TRACER_SIZE+2];
3390         int r;
3391
3392         mutex_lock(&trace_types_lock);
3393         r = sprintf(buf, "%s\n", tr->current_trace->name);
3394         mutex_unlock(&trace_types_lock);
3395
3396         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3397 }
3398
3399 int tracer_init(struct tracer *t, struct trace_array *tr)
3400 {
3401         tracing_reset_online_cpus(&tr->trace_buffer);
3402         return t->init(tr);
3403 }
3404
3405 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3406 {
3407         int cpu;
3408
3409         for_each_tracing_cpu(cpu)
3410                 per_cpu_ptr(buf->data, cpu)->entries = val;
3411 }
3412
3413 #ifdef CONFIG_TRACER_MAX_TRACE
3414 /* resize @tr's buffer to the size of @size_tr's entries */
3415 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3416                                         struct trace_buffer *size_buf, int cpu_id)
3417 {
3418         int cpu, ret = 0;
3419
3420         if (cpu_id == RING_BUFFER_ALL_CPUS) {
3421                 for_each_tracing_cpu(cpu) {
3422                         ret = ring_buffer_resize(trace_buf->buffer,
3423                                  per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3424                         if (ret < 0)
3425                                 break;
3426                         per_cpu_ptr(trace_buf->data, cpu)->entries =
3427                                 per_cpu_ptr(size_buf->data, cpu)->entries;
3428                 }
3429         } else {
3430                 ret = ring_buffer_resize(trace_buf->buffer,
3431                                  per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3432                 if (ret == 0)
3433                         per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3434                                 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3435         }
3436
3437         return ret;
3438 }
3439 #endif /* CONFIG_TRACER_MAX_TRACE */
3440
3441 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3442                                         unsigned long size, int cpu)
3443 {
3444         int ret;
3445
3446         /*
3447          * If kernel or user changes the size of the ring buffer
3448          * we use the size that was given, and we can forget about
3449          * expanding it later.
3450          */
3451         ring_buffer_expanded = true;
3452
3453         /* May be called before buffers are initialized */
3454         if (!tr->trace_buffer.buffer)
3455                 return 0;
3456
3457         ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3458         if (ret < 0)
3459                 return ret;
3460
3461 #ifdef CONFIG_TRACER_MAX_TRACE
3462         if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3463             !tr->current_trace->use_max_tr)
3464                 goto out;
3465
3466         ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3467         if (ret < 0) {
3468                 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3469                                                      &tr->trace_buffer, cpu);
3470                 if (r < 0) {
3471                         /*
3472                          * AARGH! We are left with different
3473                          * size max buffer!!!!
3474                          * The max buffer is our "snapshot" buffer.
3475                          * When a tracer needs a snapshot (one of the
3476                          * latency tracers), it swaps the max buffer
3477                          * with the saved snap shot. We succeeded to
3478                          * update the size of the main buffer, but failed to
3479                          * update the size of the max buffer. But when we tried
3480                          * to reset the main buffer to the original size, we
3481                          * failed there too. This is very unlikely to
3482                          * happen, but if it does, warn and kill all
3483                          * tracing.
3484                          */
3485                         WARN_ON(1);
3486                         tracing_disabled = 1;
3487                 }
3488                 return ret;
3489         }
3490
3491         if (cpu == RING_BUFFER_ALL_CPUS)
3492                 set_buffer_entries(&tr->max_buffer, size);
3493         else
3494                 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3495
3496  out:
3497 #endif /* CONFIG_TRACER_MAX_TRACE */
3498
3499         if (cpu == RING_BUFFER_ALL_CPUS)
3500                 set_buffer_entries(&tr->trace_buffer, size);
3501         else
3502                 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
3503
3504         return ret;
3505 }
3506
3507 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3508                                           unsigned long size, int cpu_id)
3509 {
3510         int ret = size;
3511
3512         mutex_lock(&trace_types_lock);
3513
3514         if (cpu_id != RING_BUFFER_ALL_CPUS) {
3515                 /* make sure, this cpu is enabled in the mask */
3516                 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3517                         ret = -EINVAL;
3518                         goto out;
3519                 }
3520         }
3521
3522         ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3523         if (ret < 0)
3524                 ret = -ENOMEM;
3525
3526 out:
3527         mutex_unlock(&trace_types_lock);
3528
3529         return ret;
3530 }
3531
3532
3533 /**
3534  * tracing_update_buffers - used by tracing facility to expand ring buffers
3535  *
3536  * To save on memory when the tracing is never used on a system with it
3537  * configured in. The ring buffers are set to a minimum size. But once
3538  * a user starts to use the tracing facility, then they need to grow
3539  * to their default size.
3540  *
3541  * This function is to be called when a tracer is about to be used.
3542  */
3543 int tracing_update_buffers(void)
3544 {
3545         int ret = 0;
3546
3547         mutex_lock(&trace_types_lock);
3548         if (!ring_buffer_expanded)
3549                 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3550                                                 RING_BUFFER_ALL_CPUS);
3551         mutex_unlock(&trace_types_lock);
3552
3553         return ret;
3554 }
3555
3556 struct trace_option_dentry;
3557
3558 static struct trace_option_dentry *
3559 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
3560
3561 static void
3562 destroy_trace_option_files(struct trace_option_dentry *topts);
3563
3564 static int tracing_set_tracer(const char *buf)
3565 {
3566         static struct trace_option_dentry *topts;
3567         struct trace_array *tr = &global_trace;
3568         struct tracer *t;
3569 #ifdef CONFIG_TRACER_MAX_TRACE
3570         bool had_max_tr;
3571 #endif
3572         int ret = 0;
3573
3574         mutex_lock(&trace_types_lock);
3575
3576         if (!ring_buffer_expanded) {
3577                 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3578                                                 RING_BUFFER_ALL_CPUS);
3579                 if (ret < 0)
3580                         goto out;
3581                 ret = 0;
3582         }
3583
3584         for (t = trace_types; t; t = t->next) {
3585                 if (strcmp(t->name, buf) == 0)
3586                         break;
3587         }
3588         if (!t) {
3589                 ret = -EINVAL;
3590                 goto out;
3591         }
3592         if (t == tr->current_trace)
3593                 goto out;
3594
3595         trace_branch_disable();
3596
3597         tr->current_trace->enabled = false;
3598
3599         if (tr->current_trace->reset)
3600                 tr->current_trace->reset(tr);
3601
3602         /* Current trace needs to be nop_trace before synchronize_sched */
3603         tr->current_trace = &nop_trace;
3604
3605 #ifdef CONFIG_TRACER_MAX_TRACE
3606         had_max_tr = tr->allocated_snapshot;
3607
3608         if (had_max_tr && !t->use_max_tr) {
3609                 /*
3610                  * We need to make sure that the update_max_tr sees that
3611                  * current_trace changed to nop_trace to keep it from
3612                  * swapping the buffers after we resize it.
3613                  * The update_max_tr is called from interrupts disabled
3614                  * so a synchronized_sched() is sufficient.
3615                  */
3616                 synchronize_sched();
3617                 free_snapshot(tr);
3618         }
3619 #endif
3620         destroy_trace_option_files(topts);
3621
3622         topts = create_trace_option_files(tr, t);
3623
3624 #ifdef CONFIG_TRACER_MAX_TRACE
3625         if (t->use_max_tr && !had_max_tr) {
3626                 ret = alloc_snapshot(tr);
3627                 if (ret < 0)
3628                         goto out;
3629         }
3630 #endif
3631
3632         if (t->init) {
3633                 ret = tracer_init(t, tr);
3634                 if (ret)
3635                         goto out;
3636         }
3637
3638         tr->current_trace = t;
3639         tr->current_trace->enabled = true;
3640         trace_branch_enable(tr);
3641  out:
3642         mutex_unlock(&trace_types_lock);
3643
3644         return ret;
3645 }
3646
3647 static ssize_t
3648 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3649                         size_t cnt, loff_t *ppos)
3650 {
3651         char buf[MAX_TRACER_SIZE+1];
3652         int i;
3653         size_t ret;
3654         int err;
3655
3656         ret = cnt;
3657
3658         if (cnt > MAX_TRACER_SIZE)
3659                 cnt = MAX_TRACER_SIZE;
3660
3661         if (copy_from_user(&buf, ubuf, cnt))
3662                 return -EFAULT;
3663
3664         buf[cnt] = 0;
3665
3666         /* strip ending whitespace. */
3667         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3668                 buf[i] = 0;
3669
3670         err = tracing_set_tracer(buf);
3671         if (err)
3672                 return err;
3673
3674         *ppos += ret;
3675
3676         return ret;
3677 }
3678
3679 static ssize_t
3680 tracing_max_lat_read(struct file *filp, char __user *ubuf,
3681                      size_t cnt, loff_t *ppos)
3682 {
3683         unsigned long *ptr = filp->private_data;
3684         char buf[64];
3685         int r;
3686
3687         r = snprintf(buf, sizeof(buf), "%ld\n",
3688                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3689         if (r > sizeof(buf))
3690                 r = sizeof(buf);
3691         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3692 }
3693
3694 static ssize_t
3695 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3696                       size_t cnt, loff_t *ppos)
3697 {
3698         unsigned long *ptr = filp->private_data;
3699         unsigned long val;
3700         int ret;
3701
3702         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3703         if (ret)
3704                 return ret;
3705
3706         *ptr = val * 1000;
3707
3708         return cnt;
3709 }
3710
3711 static int tracing_open_pipe(struct inode *inode, struct file *filp)
3712 {
3713         struct trace_cpu *tc = inode->i_private;
3714         struct trace_array *tr = tc->tr;
3715         struct trace_iterator *iter;
3716         int ret = 0;
3717
3718         if (tracing_disabled)
3719                 return -ENODEV;
3720
3721         mutex_lock(&trace_types_lock);
3722
3723         /* create a buffer to store the information to pass to userspace */
3724         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3725         if (!iter) {
3726                 ret = -ENOMEM;
3727                 goto out;
3728         }
3729
3730         /*
3731          * We make a copy of the current tracer to avoid concurrent
3732          * changes on it while we are reading.
3733          */
3734         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3735         if (!iter->trace) {
3736                 ret = -ENOMEM;
3737                 goto fail;
3738         }
3739         *iter->trace = *tr->current_trace;
3740
3741         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3742                 ret = -ENOMEM;
3743                 goto fail;
3744         }
3745
3746         /* trace pipe does not show start of buffer */
3747         cpumask_setall(iter->started);
3748
3749         if (trace_flags & TRACE_ITER_LATENCY_FMT)
3750                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3751
3752         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3753         if (trace_clocks[trace_clock_id].in_ns)
3754                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3755
3756         iter->cpu_file = tc->cpu;
3757         iter->tr = tc->tr;
3758         iter->trace_buffer = &tc->tr->trace_buffer;
3759         mutex_init(&iter->mutex);
3760         filp->private_data = iter;
3761
3762         if (iter->trace->pipe_open)
3763                 iter->trace->pipe_open(iter);
3764
3765         nonseekable_open(inode, filp);
3766 out:
3767         mutex_unlock(&trace_types_lock);
3768         return ret;
3769
3770 fail:
3771         kfree(iter->trace);
3772         kfree(iter);
3773         mutex_unlock(&trace_types_lock);
3774         return ret;
3775 }
3776
3777 static int tracing_release_pipe(struct inode *inode, struct file *file)
3778 {
3779         struct trace_iterator *iter = file->private_data;
3780
3781         mutex_lock(&trace_types_lock);
3782
3783         if (iter->trace->pipe_close)
3784                 iter->trace->pipe_close(iter);
3785
3786         mutex_unlock(&trace_types_lock);
3787
3788         free_cpumask_var(iter->started);
3789         mutex_destroy(&iter->mutex);
3790         kfree(iter->trace);
3791         kfree(iter);
3792
3793         return 0;
3794 }
3795
3796 static unsigned int
3797 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
3798 {
3799         /* Iterators are static, they should be filled or empty */
3800         if (trace_buffer_iter(iter, iter->cpu_file))
3801                 return POLLIN | POLLRDNORM;
3802
3803         if (trace_flags & TRACE_ITER_BLOCK)
3804                 /*
3805                  * Always select as readable when in blocking mode
3806                  */
3807                 return POLLIN | POLLRDNORM;
3808         else
3809                 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
3810                                              filp, poll_table);
3811 }
3812
3813 static unsigned int
3814 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3815 {
3816         struct trace_iterator *iter = filp->private_data;
3817
3818         return trace_poll(iter, filp, poll_table);
3819 }
3820
3821 /*
3822  * This is a make-shift waitqueue.
3823  * A tracer might use this callback on some rare cases:
3824  *
3825  *  1) the current tracer might hold the runqueue lock when it wakes up
3826  *     a reader, hence a deadlock (sched, function, and function graph tracers)
3827  *  2) the function tracers, trace all functions, we don't want
3828  *     the overhead of calling wake_up and friends
3829  *     (and tracing them too)
3830  *
3831  *     Anyway, this is really very primitive wakeup.
3832  */
3833 void poll_wait_pipe(struct trace_iterator *iter)
3834 {
3835         set_current_state(TASK_INTERRUPTIBLE);
3836         /* sleep for 100 msecs, and try again. */
3837         schedule_timeout(HZ / 10);
3838 }
3839
3840 /* Must be called with trace_types_lock mutex held. */
3841 static int tracing_wait_pipe(struct file *filp)
3842 {
3843         struct trace_iterator *iter = filp->private_data;
3844
3845         while (trace_empty(iter)) {
3846
3847                 if ((filp->f_flags & O_NONBLOCK)) {
3848                         return -EAGAIN;
3849                 }
3850
3851                 mutex_unlock(&iter->mutex);
3852
3853                 iter->trace->wait_pipe(iter);
3854
3855                 mutex_lock(&iter->mutex);
3856
3857                 if (signal_pending(current))
3858                         return -EINTR;
3859
3860                 /*
3861                  * We block until we read something and tracing is disabled.
3862                  * We still block if tracing is disabled, but we have never
3863                  * read anything. This allows a user to cat this file, and
3864                  * then enable tracing. But after we have read something,
3865                  * we give an EOF when tracing is again disabled.
3866                  *
3867                  * iter->pos will be 0 if we haven't read anything.
3868                  */
3869                 if (!tracing_is_enabled() && iter->pos)
3870                         break;
3871         }
3872
3873         return 1;
3874 }
3875
3876 /*
3877  * Consumer reader.
3878  */
3879 static ssize_t
3880 tracing_read_pipe(struct file *filp, char __user *ubuf,
3881                   size_t cnt, loff_t *ppos)
3882 {
3883         struct trace_iterator *iter = filp->private_data;
3884         struct trace_array *tr = iter->tr;
3885         ssize_t sret;
3886
3887         /* return any leftover data */
3888         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3889         if (sret != -EBUSY)
3890                 return sret;
3891
3892         trace_seq_init(&iter->seq);
3893
3894         /* copy the tracer to avoid using a global lock all around */
3895         mutex_lock(&trace_types_lock);
3896         if (unlikely(iter->trace->name != tr->current_trace->name))
3897                 *iter->trace = *tr->current_trace;
3898         mutex_unlock(&trace_types_lock);
3899
3900         /*
3901          * Avoid more than one consumer on a single file descriptor
3902          * This is just a matter of traces coherency, the ring buffer itself
3903          * is protected.
3904          */
3905         mutex_lock(&iter->mutex);
3906         if (iter->trace->read) {
3907                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3908                 if (sret)
3909                         goto out;
3910         }
3911
3912 waitagain:
3913         sret = tracing_wait_pipe(filp);
3914         if (sret <= 0)
3915                 goto out;
3916
3917         /* stop when tracing is finished */
3918         if (trace_empty(iter)) {
3919                 sret = 0;
3920                 goto out;
3921         }
3922
3923         if (cnt >= PAGE_SIZE)
3924                 cnt = PAGE_SIZE - 1;
3925
3926         /* reset all but tr, trace, and overruns */
3927         memset(&iter->seq, 0,
3928                sizeof(struct trace_iterator) -
3929                offsetof(struct trace_iterator, seq));
3930         iter->pos = -1;
3931
3932         trace_event_read_lock();
3933         trace_access_lock(iter->cpu_file);
3934         while (trace_find_next_entry_inc(iter) != NULL) {
3935                 enum print_line_t ret;
3936                 int len = iter->seq.len;
3937
3938                 ret = print_trace_line(iter);
3939                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3940                         /* don't print partial lines */
3941                         iter->seq.len = len;
3942                         break;
3943                 }
3944                 if (ret != TRACE_TYPE_NO_CONSUME)
3945                         trace_consume(iter);
3946
3947                 if (iter->seq.len >= cnt)
3948                         break;
3949
3950                 /*
3951                  * Setting the full flag means we reached the trace_seq buffer
3952                  * size and we should leave by partial output condition above.
3953                  * One of the trace_seq_* functions is not used properly.
3954                  */
3955                 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3956                           iter->ent->type);
3957         }
3958         trace_access_unlock(iter->cpu_file);
3959         trace_event_read_unlock();
3960
3961         /* Now copy what we have to the user */
3962         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3963         if (iter->seq.readpos >= iter->seq.len)
3964                 trace_seq_init(&iter->seq);
3965
3966         /*
3967          * If there was nothing to send to user, in spite of consuming trace
3968          * entries, go back to wait for more entries.
3969          */
3970         if (sret == -EBUSY)
3971                 goto waitagain;
3972
3973 out:
3974         mutex_unlock(&iter->mutex);
3975
3976         return sret;
3977 }
3978
3979 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3980                                      struct pipe_buffer *buf)
3981 {
3982         __free_page(buf->page);
3983 }
3984
3985 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3986                                      unsigned int idx)
3987 {
3988         __free_page(spd->pages[idx]);
3989 }
3990
3991 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
3992         .can_merge              = 0,
3993         .map                    = generic_pipe_buf_map,
3994         .unmap                  = generic_pipe_buf_unmap,
3995         .confirm                = generic_pipe_buf_confirm,
3996         .release                = tracing_pipe_buf_release,
3997         .steal                  = generic_pipe_buf_steal,
3998         .get                    = generic_pipe_buf_get,
3999 };
4000
4001 static size_t
4002 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4003 {
4004         size_t count;
4005         int ret;
4006
4007         /* Seq buffer is page-sized, exactly what we need. */
4008         for (;;) {
4009                 count = iter->seq.len;
4010                 ret = print_trace_line(iter);
4011                 count = iter->seq.len - count;
4012                 if (rem < count) {
4013                         rem = 0;
4014                         iter->seq.len -= count;
4015                         break;
4016                 }
4017                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4018                         iter->seq.len -= count;
4019                         break;
4020                 }
4021
4022                 if (ret != TRACE_TYPE_NO_CONSUME)
4023                         trace_consume(iter);
4024                 rem -= count;
4025                 if (!trace_find_next_entry_inc(iter))   {
4026                         rem = 0;
4027                         iter->ent = NULL;
4028                         break;
4029                 }
4030         }
4031
4032         return rem;
4033 }
4034
4035 static ssize_t tracing_splice_read_pipe(struct file *filp,
4036                                         loff_t *ppos,
4037                                         struct pipe_inode_info *pipe,
4038                                         size_t len,
4039                                         unsigned int flags)
4040 {
4041         struct page *pages_def[PIPE_DEF_BUFFERS];
4042         struct partial_page partial_def[PIPE_DEF_BUFFERS];
4043         struct trace_iterator *iter = filp->private_data;
4044         struct splice_pipe_desc spd = {
4045                 .pages          = pages_def,
4046                 .partial        = partial_def,
4047                 .nr_pages       = 0, /* This gets updated below. */
4048                 .nr_pages_max   = PIPE_DEF_BUFFERS,
4049                 .flags          = flags,
4050                 .ops            = &tracing_pipe_buf_ops,
4051                 .spd_release    = tracing_spd_release_pipe,
4052         };
4053         struct trace_array *tr = iter->tr;
4054         ssize_t ret;
4055         size_t rem;
4056         unsigned int i;
4057
4058         if (splice_grow_spd(pipe, &spd))
4059                 return -ENOMEM;
4060
4061         /* copy the tracer to avoid using a global lock all around */
4062         mutex_lock(&trace_types_lock);
4063         if (unlikely(iter->trace->name != tr->current_trace->name))
4064                 *iter->trace = *tr->current_trace;
4065         mutex_unlock(&trace_types_lock);
4066
4067         mutex_lock(&iter->mutex);
4068
4069         if (iter->trace->splice_read) {
4070                 ret = iter->trace->splice_read(iter, filp,
4071                                                ppos, pipe, len, flags);
4072                 if (ret)
4073                         goto out_err;
4074         }
4075
4076         ret = tracing_wait_pipe(filp);
4077         if (ret <= 0)
4078                 goto out_err;
4079
4080         if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4081                 ret = -EFAULT;
4082                 goto out_err;
4083         }
4084
4085         trace_event_read_lock();
4086         trace_access_lock(iter->cpu_file);
4087
4088         /* Fill as many pages as possible. */
4089         for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4090                 spd.pages[i] = alloc_page(GFP_KERNEL);
4091                 if (!spd.pages[i])
4092                         break;
4093
4094                 rem = tracing_fill_pipe_page(rem, iter);
4095
4096                 /* Copy the data into the page, so we can start over. */
4097                 ret = trace_seq_to_buffer(&iter->seq,
4098                                           page_address(spd.pages[i]),
4099                                           iter->seq.len);
4100                 if (ret < 0) {
4101                         __free_page(spd.pages[i]);
4102                         break;
4103                 }
4104                 spd.partial[i].offset = 0;
4105                 spd.partial[i].len = iter->seq.len;
4106
4107                 trace_seq_init(&iter->seq);
4108         }
4109
4110         trace_access_unlock(iter->cpu_file);
4111         trace_event_read_unlock();
4112         mutex_unlock(&iter->mutex);
4113
4114         spd.nr_pages = i;
4115
4116         ret = splice_to_pipe(pipe, &spd);
4117 out:
4118         splice_shrink_spd(&spd);
4119         return ret;
4120
4121 out_err:
4122         mutex_unlock(&iter->mutex);
4123         goto out;
4124 }
4125
4126 static ssize_t
4127 tracing_entries_read(struct file *filp, char __user *ubuf,
4128                      size_t cnt, loff_t *ppos)
4129 {
4130         struct trace_cpu *tc = filp->private_data;
4131         struct trace_array *tr = tc->tr;
4132         char buf[64];
4133         int r = 0;
4134         ssize_t ret;
4135
4136         mutex_lock(&trace_types_lock);
4137
4138         if (tc->cpu == RING_BUFFER_ALL_CPUS) {
4139                 int cpu, buf_size_same;
4140                 unsigned long size;
4141
4142                 size = 0;
4143                 buf_size_same = 1;
4144                 /* check if all cpu sizes are same */
4145                 for_each_tracing_cpu(cpu) {
4146                         /* fill in the size from first enabled cpu */
4147                         if (size == 0)
4148                                 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4149                         if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4150                                 buf_size_same = 0;
4151                                 break;
4152                         }
4153                 }
4154
4155                 if (buf_size_same) {
4156                         if (!ring_buffer_expanded)
4157                                 r = sprintf(buf, "%lu (expanded: %lu)\n",
4158                                             size >> 10,
4159                                             trace_buf_size >> 10);
4160                         else
4161                                 r = sprintf(buf, "%lu\n", size >> 10);
4162                 } else
4163                         r = sprintf(buf, "X\n");
4164         } else
4165                 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, tc->cpu)->entries >> 10);
4166
4167         mutex_unlock(&trace_types_lock);
4168
4169         ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4170         return ret;
4171 }
4172
4173 static ssize_t
4174 tracing_entries_write(struct file *filp, const char __user *ubuf,
4175                       size_t cnt, loff_t *ppos)
4176 {
4177         struct trace_cpu *tc = filp->private_data;
4178         unsigned long val;
4179         int ret;
4180
4181         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4182         if (ret)
4183                 return ret;
4184
4185         /* must have at least 1 entry */
4186         if (!val)
4187                 return -EINVAL;
4188
4189         /* value is in KB */
4190         val <<= 10;
4191
4192         ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
4193         if (ret < 0)
4194                 return ret;
4195
4196         *ppos += cnt;
4197
4198         return cnt;
4199 }
4200
4201 static ssize_t
4202 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4203                                 size_t cnt, loff_t *ppos)
4204 {
4205         struct trace_array *tr = filp->private_data;
4206         char buf[64];
4207         int r, cpu;
4208         unsigned long size = 0, expanded_size = 0;
4209
4210         mutex_lock(&trace_types_lock);
4211         for_each_tracing_cpu(cpu) {
4212                 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4213                 if (!ring_buffer_expanded)
4214                         expanded_size += trace_buf_size >> 10;
4215         }
4216         if (ring_buffer_expanded)
4217                 r = sprintf(buf, "%lu\n", size);
4218         else
4219                 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4220         mutex_unlock(&trace_types_lock);
4221
4222         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4223 }
4224
4225 static ssize_t
4226 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4227                           size_t cnt, loff_t *ppos)
4228 {
4229         /*
4230          * There is no need to read what the user has written, this function
4231          * is just to make sure that there is no error when "echo" is used
4232          */
4233
4234         *ppos += cnt;
4235
4236         return cnt;
4237 }
4238
4239 static int
4240 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4241 {
4242         struct trace_array *tr = inode->i_private;
4243
4244         /* disable tracing ? */
4245         if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4246                 tracing_off();
4247         /* resize the ring buffer to 0 */
4248         tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4249
4250         return 0;
4251 }
4252
4253 static ssize_t
4254 tracing_mark_write(struct file *filp, const char __user *ubuf,
4255                                         size_t cnt, loff_t *fpos)
4256 {
4257         unsigned long addr = (unsigned long)ubuf;
4258         struct ring_buffer_event *event;
4259         struct ring_buffer *buffer;
4260         struct print_entry *entry;
4261         unsigned long irq_flags;
4262         struct page *pages[2];
4263         void *map_page[2];
4264         int nr_pages = 1;
4265         ssize_t written;
4266         int offset;
4267         int size;
4268         int len;
4269         int ret;
4270         int i;
4271
4272         if (tracing_disabled)
4273                 return -EINVAL;
4274
4275         if (!(trace_flags & TRACE_ITER_MARKERS))
4276                 return -EINVAL;
4277
4278         if (cnt > TRACE_BUF_SIZE)
4279                 cnt = TRACE_BUF_SIZE;
4280
4281         /*
4282          * Userspace is injecting traces into the kernel trace buffer.
4283          * We want to be as non intrusive as possible.
4284          * To do so, we do not want to allocate any special buffers
4285          * or take any locks, but instead write the userspace data
4286          * straight into the ring buffer.
4287          *
4288          * First we need to pin the userspace buffer into memory,
4289          * which, most likely it is, because it just referenced it.
4290          * But there's no guarantee that it is. By using get_user_pages_fast()
4291          * and kmap_atomic/kunmap_atomic() we can get access to the
4292          * pages directly. We then write the data directly into the
4293          * ring buffer.
4294          */
4295         BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4296
4297         /* check if we cross pages */
4298         if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4299                 nr_pages = 2;
4300
4301         offset = addr & (PAGE_SIZE - 1);
4302         addr &= PAGE_MASK;
4303
4304         ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4305         if (ret < nr_pages) {
4306                 while (--ret >= 0)
4307                         put_page(pages[ret]);
4308                 written = -EFAULT;
4309                 goto out;
4310         }
4311
4312         for (i = 0; i < nr_pages; i++)
4313                 map_page[i] = kmap_atomic(pages[i]);
4314
4315         local_save_flags(irq_flags);
4316         size = sizeof(*entry) + cnt + 2; /* possible \n added */
4317         buffer = global_trace.trace_buffer.buffer;
4318         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4319                                           irq_flags, preempt_count());
4320         if (!event) {
4321                 /* Ring buffer disabled, return as if not open for write */
4322                 written = -EBADF;
4323                 goto out_unlock;
4324         }
4325
4326         entry = ring_buffer_event_data(event);
4327         entry->ip = _THIS_IP_;
4328
4329         if (nr_pages == 2) {
4330                 len = PAGE_SIZE - offset;
4331                 memcpy(&entry->buf, map_page[0] + offset, len);
4332                 memcpy(&entry->buf[len], map_page[1], cnt - len);
4333         } else
4334                 memcpy(&entry->buf, map_page[0] + offset, cnt);
4335
4336         if (entry->buf[cnt - 1] != '\n') {
4337                 entry->buf[cnt] = '\n';
4338                 entry->buf[cnt + 1] = '\0';
4339         } else
4340                 entry->buf[cnt] = '\0';
4341
4342         __buffer_unlock_commit(buffer, event);
4343
4344         written = cnt;
4345
4346         *fpos += written;
4347
4348  out_unlock:
4349         for (i = 0; i < nr_pages; i++){
4350                 kunmap_atomic(map_page[i]);
4351                 put_page(pages[i]);
4352         }
4353  out:
4354         return written;
4355 }
4356
4357 static int tracing_clock_show(struct seq_file *m, void *v)
4358 {
4359         struct trace_array *tr = m->private;
4360         int i;
4361
4362         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4363                 seq_printf(m,
4364                         "%s%s%s%s", i ? " " : "",
4365                         i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4366                         i == tr->clock_id ? "]" : "");
4367         seq_putc(m, '\n');
4368
4369         return 0;
4370 }
4371
4372 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4373                                    size_t cnt, loff_t *fpos)
4374 {
4375         struct seq_file *m = filp->private_data;
4376         struct trace_array *tr = m->private;
4377         char buf[64];
4378         const char *clockstr;
4379         int i;
4380
4381         if (cnt >= sizeof(buf))
4382                 return -EINVAL;
4383
4384         if (copy_from_user(&buf, ubuf, cnt))
4385                 return -EFAULT;
4386
4387         buf[cnt] = 0;
4388
4389         clockstr = strstrip(buf);
4390
4391         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4392                 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4393                         break;
4394         }
4395         if (i == ARRAY_SIZE(trace_clocks))
4396                 return -EINVAL;
4397
4398         mutex_lock(&trace_types_lock);
4399
4400         tr->clock_id = i;
4401
4402         ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4403
4404         /*
4405          * New clock may not be consistent with the previous clock.
4406          * Reset the buffer so that it doesn't have incomparable timestamps.
4407          */
4408         tracing_reset_online_cpus(&global_trace.trace_buffer);
4409
4410 #ifdef CONFIG_TRACER_MAX_TRACE
4411         if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4412                 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4413         tracing_reset_online_cpus(&global_trace.max_buffer);
4414 #endif
4415
4416         mutex_unlock(&trace_types_lock);
4417
4418         *fpos += cnt;
4419
4420         return cnt;
4421 }
4422
4423 static int tracing_clock_open(struct inode *inode, struct file *file)
4424 {
4425         if (tracing_disabled)
4426                 return -ENODEV;
4427
4428         return single_open(file, tracing_clock_show, inode->i_private);
4429 }
4430
4431 struct ftrace_buffer_info {
4432         struct trace_iterator   iter;
4433         void                    *spare;
4434         unsigned int            read;
4435 };
4436
4437 #ifdef CONFIG_TRACER_SNAPSHOT
4438 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4439 {
4440         struct trace_cpu *tc = inode->i_private;
4441         struct trace_iterator *iter;
4442         struct seq_file *m;
4443         int ret = 0;
4444
4445         if (file->f_mode & FMODE_READ) {
4446                 iter = __tracing_open(inode, file, true);
4447                 if (IS_ERR(iter))
4448                         ret = PTR_ERR(iter);
4449         } else {
4450                 /* Writes still need the seq_file to hold the private data */
4451                 m = kzalloc(sizeof(*m), GFP_KERNEL);
4452                 if (!m)
4453                         return -ENOMEM;
4454                 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4455                 if (!iter) {
4456                         kfree(m);
4457                         return -ENOMEM;
4458                 }
4459                 iter->tr = tc->tr;
4460                 iter->trace_buffer = &tc->tr->max_buffer;
4461                 iter->cpu_file = tc->cpu;
4462                 m->private = iter;
4463                 file->private_data = m;
4464         }
4465
4466         return ret;
4467 }
4468
4469 static ssize_t
4470 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4471                        loff_t *ppos)
4472 {
4473         struct seq_file *m = filp->private_data;
4474         struct trace_iterator *iter = m->private;
4475         struct trace_array *tr = iter->tr;
4476         unsigned long val;
4477         int ret;
4478
4479         ret = tracing_update_buffers();
4480         if (ret < 0)
4481                 return ret;
4482
4483         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4484         if (ret)
4485                 return ret;
4486
4487         mutex_lock(&trace_types_lock);
4488
4489         if (tr->current_trace->use_max_tr) {
4490                 ret = -EBUSY;
4491                 goto out;
4492         }
4493
4494         switch (val) {
4495         case 0:
4496                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4497                         ret = -EINVAL;
4498                         break;
4499                 }
4500                 if (tr->allocated_snapshot)
4501                         free_snapshot(tr);
4502                 break;
4503         case 1:
4504 /* Only allow per-cpu swap if the ring buffer supports it */
4505 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4506                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4507                         ret = -EINVAL;
4508                         break;
4509                 }
4510 #endif
4511                 if (!tr->allocated_snapshot) {
4512                         ret = alloc_snapshot(tr);
4513                         if (ret < 0)
4514                                 break;
4515                 }
4516                 local_irq_disable();
4517                 /* Now, we're going to swap */
4518                 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4519                         update_max_tr(tr, current, smp_processor_id());
4520                 else
4521                         update_max_tr_single(tr, current, iter->cpu_file);
4522                 local_irq_enable();
4523                 break;
4524         default:
4525                 if (tr->allocated_snapshot) {
4526                         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4527                                 tracing_reset_online_cpus(&tr->max_buffer);
4528                         else
4529                                 tracing_reset(&tr->max_buffer, iter->cpu_file);
4530                 }
4531                 break;
4532         }
4533
4534         if (ret >= 0) {
4535                 *ppos += cnt;
4536                 ret = cnt;
4537         }
4538 out:
4539         mutex_unlock(&trace_types_lock);
4540         return ret;
4541 }
4542
4543 static int tracing_snapshot_release(struct inode *inode, struct file *file)
4544 {
4545         struct seq_file *m = file->private_data;
4546
4547         if (file->f_mode & FMODE_READ)
4548                 return tracing_release(inode, file);
4549
4550         /* If write only, the seq_file is just a stub */
4551         if (m)
4552                 kfree(m->private);
4553         kfree(m);
4554
4555         return 0;
4556 }
4557
4558 static int tracing_buffers_open(struct inode *inode, struct file *filp);
4559 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4560                                     size_t count, loff_t *ppos);
4561 static int tracing_buffers_release(struct inode *inode, struct file *file);
4562 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4563                    struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4564
4565 static int snapshot_raw_open(struct inode *inode, struct file *filp)
4566 {
4567         struct ftrace_buffer_info *info;
4568         int ret;
4569
4570         ret = tracing_buffers_open(inode, filp);
4571         if (ret < 0)
4572                 return ret;
4573
4574         info = filp->private_data;
4575
4576         if (info->iter.trace->use_max_tr) {
4577                 tracing_buffers_release(inode, filp);
4578                 return -EBUSY;
4579         }
4580
4581         info->iter.snapshot = true;
4582         info->iter.trace_buffer = &info->iter.tr->max_buffer;
4583
4584         return ret;
4585 }
4586
4587 #endif /* CONFIG_TRACER_SNAPSHOT */
4588
4589
4590 static const struct file_operations tracing_max_lat_fops = {
4591         .open           = tracing_open_generic,
4592         .read           = tracing_max_lat_read,
4593         .write          = tracing_max_lat_write,
4594         .llseek         = generic_file_llseek,
4595 };
4596
4597 static const struct file_operations set_tracer_fops = {
4598         .open           = tracing_open_generic,
4599         .read           = tracing_set_trace_read,
4600         .write          = tracing_set_trace_write,
4601         .llseek         = generic_file_llseek,
4602 };
4603
4604 static const struct file_operations tracing_pipe_fops = {
4605         .open           = tracing_open_pipe,
4606         .poll           = tracing_poll_pipe,
4607         .read           = tracing_read_pipe,
4608         .splice_read    = tracing_splice_read_pipe,
4609         .release        = tracing_release_pipe,
4610         .llseek         = no_llseek,
4611 };
4612
4613 static const struct file_operations tracing_entries_fops = {
4614         .open           = tracing_open_generic,
4615         .read           = tracing_entries_read,
4616         .write          = tracing_entries_write,
4617         .llseek         = generic_file_llseek,
4618 };
4619
4620 static const struct file_operations tracing_total_entries_fops = {
4621         .open           = tracing_open_generic,
4622         .read           = tracing_total_entries_read,
4623         .llseek         = generic_file_llseek,
4624 };
4625
4626 static const struct file_operations tracing_free_buffer_fops = {
4627         .write          = tracing_free_buffer_write,
4628         .release        = tracing_free_buffer_release,
4629 };
4630
4631 static const struct file_operations tracing_mark_fops = {
4632         .open           = tracing_open_generic,
4633         .write          = tracing_mark_write,
4634         .llseek         = generic_file_llseek,
4635 };
4636
4637 static const struct file_operations trace_clock_fops = {
4638         .open           = tracing_clock_open,
4639         .read           = seq_read,
4640         .llseek         = seq_lseek,
4641         .release        = single_release,
4642         .write          = tracing_clock_write,
4643 };
4644
4645 #ifdef CONFIG_TRACER_SNAPSHOT
4646 static const struct file_operations snapshot_fops = {
4647         .open           = tracing_snapshot_open,
4648         .read           = seq_read,
4649         .write          = tracing_snapshot_write,
4650         .llseek         = tracing_seek,
4651         .release        = tracing_snapshot_release,
4652 };
4653
4654 static const struct file_operations snapshot_raw_fops = {
4655         .open           = snapshot_raw_open,
4656         .read           = tracing_buffers_read,
4657         .release        = tracing_buffers_release,
4658         .splice_read    = tracing_buffers_splice_read,
4659         .llseek         = no_llseek,
4660 };
4661
4662 #endif /* CONFIG_TRACER_SNAPSHOT */
4663
4664 static int tracing_buffers_open(struct inode *inode, struct file *filp)
4665 {
4666         struct trace_cpu *tc = inode->i_private;
4667         struct trace_array *tr = tc->tr;
4668         struct ftrace_buffer_info *info;
4669
4670         if (tracing_disabled)
4671                 return -ENODEV;
4672
4673         info = kzalloc(sizeof(*info), GFP_KERNEL);
4674         if (!info)
4675                 return -ENOMEM;
4676
4677         mutex_lock(&trace_types_lock);
4678
4679         tr->ref++;
4680
4681         info->iter.tr           = tr;
4682         info->iter.cpu_file     = tc->cpu;
4683         info->iter.trace        = tr->current_trace;
4684         info->iter.trace_buffer = &tr->trace_buffer;
4685         info->spare             = NULL;
4686         /* Force reading ring buffer for first read */
4687         info->read              = (unsigned int)-1;
4688
4689         filp->private_data = info;
4690
4691         mutex_unlock(&trace_types_lock);
4692
4693         return nonseekable_open(inode, filp);
4694 }
4695
4696 static unsigned int
4697 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4698 {
4699         struct ftrace_buffer_info *info = filp->private_data;
4700         struct trace_iterator *iter = &info->iter;
4701
4702         return trace_poll(iter, filp, poll_table);
4703 }
4704
4705 static ssize_t
4706 tracing_buffers_read(struct file *filp, char __user *ubuf,
4707                      size_t count, loff_t *ppos)
4708 {
4709         struct ftrace_buffer_info *info = filp->private_data;
4710         struct trace_iterator *iter = &info->iter;
4711         ssize_t ret;
4712         ssize_t size;
4713
4714         if (!count)
4715                 return 0;
4716
4717         mutex_lock(&trace_types_lock);
4718
4719 #ifdef CONFIG_TRACER_MAX_TRACE
4720         if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4721                 size = -EBUSY;
4722                 goto out_unlock;
4723         }
4724 #endif
4725
4726         if (!info->spare)
4727                 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4728                                                           iter->cpu_file);
4729         size = -ENOMEM;
4730         if (!info->spare)
4731                 goto out_unlock;
4732
4733         /* Do we have previous read data to read? */
4734         if (info->read < PAGE_SIZE)
4735                 goto read;
4736
4737  again:
4738         trace_access_lock(iter->cpu_file);
4739         ret = ring_buffer_read_page(iter->trace_buffer->buffer,
4740                                     &info->spare,
4741                                     count,
4742                                     iter->cpu_file, 0);
4743         trace_access_unlock(iter->cpu_file);
4744
4745         if (ret < 0) {
4746                 if (trace_empty(iter)) {
4747                         if ((filp->f_flags & O_NONBLOCK)) {
4748                                 size = -EAGAIN;
4749                                 goto out_unlock;
4750                         }
4751                         mutex_unlock(&trace_types_lock);
4752                         iter->trace->wait_pipe(iter);
4753                         mutex_lock(&trace_types_lock);
4754                         if (signal_pending(current)) {
4755                                 size = -EINTR;
4756                                 goto out_unlock;
4757                         }
4758                         goto again;
4759                 }
4760                 size = 0;
4761                 goto out_unlock;
4762         }
4763
4764         info->read = 0;
4765  read:
4766         size = PAGE_SIZE - info->read;
4767         if (size > count)
4768                 size = count;
4769
4770         ret = copy_to_user(ubuf, info->spare + info->read, size);
4771         if (ret == size) {
4772                 size = -EFAULT;
4773                 goto out_unlock;
4774         }
4775         size -= ret;
4776
4777         *ppos += size;
4778         info->read += size;
4779
4780  out_unlock:
4781         mutex_unlock(&trace_types_lock);
4782
4783         return size;
4784 }
4785
4786 static int tracing_buffers_release(struct inode *inode, struct file *file)
4787 {
4788         struct ftrace_buffer_info *info = file->private_data;
4789         struct trace_iterator *iter = &info->iter;
4790
4791         mutex_lock(&trace_types_lock);
4792
4793         WARN_ON(!iter->tr->ref);
4794         iter->tr->ref--;
4795
4796         if (info->spare)
4797                 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
4798         kfree(info);
4799
4800         mutex_unlock(&trace_types_lock);
4801
4802         return 0;
4803 }
4804
4805 struct buffer_ref {
4806         struct ring_buffer      *buffer;
4807         void                    *page;
4808         int                     ref;
4809 };
4810
4811 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4812                                     struct pipe_buffer *buf)
4813 {
4814         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4815
4816         if (--ref->ref)
4817                 return;
4818
4819         ring_buffer_free_read_page(ref->buffer, ref->page);
4820         kfree(ref);
4821         buf->private = 0;
4822 }
4823
4824 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4825                                 struct pipe_buffer *buf)
4826 {
4827         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4828
4829         ref->ref++;
4830 }
4831
4832 /* Pipe buffer operations for a buffer. */
4833 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4834         .can_merge              = 0,
4835         .map                    = generic_pipe_buf_map,
4836         .unmap                  = generic_pipe_buf_unmap,
4837         .confirm                = generic_pipe_buf_confirm,
4838         .release                = buffer_pipe_buf_release,
4839         .steal                  = generic_pipe_buf_steal,
4840         .get                    = buffer_pipe_buf_get,
4841 };
4842
4843 /*
4844  * Callback from splice_to_pipe(), if we need to release some pages
4845  * at the end of the spd in case we error'ed out in filling the pipe.
4846  */
4847 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4848 {
4849         struct buffer_ref *ref =
4850                 (struct buffer_ref *)spd->partial[i].private;
4851
4852         if (--ref->ref)
4853                 return;
4854
4855         ring_buffer_free_read_page(ref->buffer, ref->page);
4856         kfree(ref);
4857         spd->partial[i].private = 0;
4858 }
4859
4860 static ssize_t
4861 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4862                             struct pipe_inode_info *pipe, size_t len,
4863                             unsigned int flags)
4864 {
4865         struct ftrace_buffer_info *info = file->private_data;
4866         struct trace_iterator *iter = &info->iter;
4867         struct partial_page partial_def[PIPE_DEF_BUFFERS];
4868         struct page *pages_def[PIPE_DEF_BUFFERS];
4869         struct splice_pipe_desc spd = {
4870                 .pages          = pages_def,
4871                 .partial        = partial_def,
4872                 .nr_pages_max   = PIPE_DEF_BUFFERS,
4873                 .flags          = flags,
4874                 .ops            = &buffer_pipe_buf_ops,
4875                 .spd_release    = buffer_spd_release,
4876         };
4877         struct buffer_ref *ref;
4878         int entries, size, i;
4879         ssize_t ret;
4880
4881         mutex_lock(&trace_types_lock);
4882
4883 #ifdef CONFIG_TRACER_MAX_TRACE
4884         if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4885                 ret = -EBUSY;
4886                 goto out;
4887         }
4888 #endif
4889
4890         if (splice_grow_spd(pipe, &spd)) {
4891                 ret = -ENOMEM;
4892                 goto out;
4893         }
4894
4895         if (*ppos & (PAGE_SIZE - 1)) {
4896                 ret = -EINVAL;
4897                 goto out;
4898         }
4899
4900         if (len & (PAGE_SIZE - 1)) {
4901                 if (len < PAGE_SIZE) {
4902                         ret = -EINVAL;
4903                         goto out;
4904                 }
4905                 len &= PAGE_MASK;
4906         }
4907
4908  again:
4909         trace_access_lock(iter->cpu_file);
4910         entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
4911
4912         for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4913                 struct page *page;
4914                 int r;
4915
4916                 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4917                 if (!ref)
4918                         break;
4919
4920                 ref->ref = 1;
4921                 ref->buffer = iter->trace_buffer->buffer;
4922                 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
4923                 if (!ref->page) {
4924                         kfree(ref);
4925                         break;
4926                 }
4927
4928                 r = ring_buffer_read_page(ref->buffer, &ref->page,
4929                                           len, iter->cpu_file, 1);
4930                 if (r < 0) {
4931                         ring_buffer_free_read_page(ref->buffer, ref->page);
4932                         kfree(ref);
4933                         break;
4934                 }
4935
4936                 /*
4937                  * zero out any left over data, this is going to
4938                  * user land.
4939                  */
4940                 size = ring_buffer_page_len(ref->page);
4941                 if (size < PAGE_SIZE)
4942                         memset(ref->page + size, 0, PAGE_SIZE - size);
4943
4944                 page = virt_to_page(ref->page);
4945
4946                 spd.pages[i] = page;
4947                 spd.partial[i].len = PAGE_SIZE;
4948                 spd.partial[i].offset = 0;
4949                 spd.partial[i].private = (unsigned long)ref;
4950                 spd.nr_pages++;
4951                 *ppos += PAGE_SIZE;
4952
4953                 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
4954         }
4955
4956         trace_access_unlock(iter->cpu_file);
4957         spd.nr_pages = i;
4958
4959         /* did we read anything? */
4960         if (!spd.nr_pages) {
4961                 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
4962                         ret = -EAGAIN;
4963                         goto out;
4964                 }
4965                 mutex_unlock(&trace_types_lock);
4966                 iter->trace->wait_pipe(iter);
4967                 mutex_lock(&trace_types_lock);
4968                 if (signal_pending(current)) {
4969                         ret = -EINTR;
4970                         goto out;
4971                 }
4972                 goto again;
4973         }
4974
4975         ret = splice_to_pipe(pipe, &spd);
4976         splice_shrink_spd(&spd);
4977 out:
4978         mutex_unlock(&trace_types_lock);
4979
4980         return ret;
4981 }
4982
4983 static const struct file_operations tracing_buffers_fops = {
4984         .open           = tracing_buffers_open,
4985         .read           = tracing_buffers_read,
4986         .poll           = tracing_buffers_poll,
4987         .release        = tracing_buffers_release,
4988         .splice_read    = tracing_buffers_splice_read,
4989         .llseek         = no_llseek,
4990 };
4991
4992 static ssize_t
4993 tracing_stats_read(struct file *filp, char __user *ubuf,
4994                    size_t count, loff_t *ppos)
4995 {
4996         struct trace_cpu *tc = filp->private_data;
4997         struct trace_array *tr = tc->tr;
4998         struct trace_buffer *trace_buf = &tr->trace_buffer;
4999         struct trace_seq *s;
5000         unsigned long cnt;
5001         unsigned long long t;
5002         unsigned long usec_rem;
5003         int cpu = tc->cpu;
5004
5005         s = kmalloc(sizeof(*s), GFP_KERNEL);
5006         if (!s)
5007                 return -ENOMEM;
5008
5009         trace_seq_init(s);
5010
5011         cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5012         trace_seq_printf(s, "entries: %ld\n", cnt);
5013
5014         cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5015         trace_seq_printf(s, "overrun: %ld\n", cnt);
5016
5017         cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5018         trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5019
5020         cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5021         trace_seq_printf(s, "bytes: %ld\n", cnt);
5022
5023         if (trace_clocks[trace_clock_id].in_ns) {
5024                 /* local or global for trace_clock */
5025                 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5026                 usec_rem = do_div(t, USEC_PER_SEC);
5027                 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5028                                                                 t, usec_rem);
5029
5030                 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5031                 usec_rem = do_div(t, USEC_PER_SEC);
5032                 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5033         } else {
5034                 /* counter or tsc mode for trace_clock */
5035                 trace_seq_printf(s, "oldest event ts: %llu\n",
5036                                 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5037
5038                 trace_seq_printf(s, "now ts: %llu\n",
5039                                 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5040         }
5041
5042         cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5043         trace_seq_printf(s, "dropped events: %ld\n", cnt);
5044
5045         cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5046         trace_seq_printf(s, "read events: %ld\n", cnt);
5047
5048         count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5049
5050         kfree(s);
5051
5052         return count;
5053 }
5054
5055 static const struct file_operations tracing_stats_fops = {
5056         .open           = tracing_open_generic,
5057         .read           = tracing_stats_read,
5058         .llseek         = generic_file_llseek,
5059 };
5060
5061 #ifdef CONFIG_DYNAMIC_FTRACE
5062
5063 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5064 {
5065         return 0;
5066 }
5067
5068 static ssize_t
5069 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5070                   size_t cnt, loff_t *ppos)
5071 {
5072         static char ftrace_dyn_info_buffer[1024];
5073         static DEFINE_MUTEX(dyn_info_mutex);
5074         unsigned long *p = filp->private_data;
5075         char *buf = ftrace_dyn_info_buffer;
5076         int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5077         int r;
5078
5079         mutex_lock(&dyn_info_mutex);
5080         r = sprintf(buf, "%ld ", *p);
5081
5082         r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5083         buf[r++] = '\n';
5084
5085         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5086
5087         mutex_unlock(&dyn_info_mutex);
5088
5089         return r;
5090 }
5091
5092 static const struct file_operations tracing_dyn_info_fops = {
5093         .open           = tracing_open_generic,
5094         .read           = tracing_read_dyn_info,
5095         .llseek         = generic_file_llseek,
5096 };
5097 #endif /* CONFIG_DYNAMIC_FTRACE */
5098
5099 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5100 static void
5101 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5102 {
5103         tracing_snapshot();
5104 }
5105
5106 static void
5107 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5108 {
5109         unsigned long *count = (long *)data;
5110
5111         if (!*count)
5112                 return;
5113
5114         if (*count != -1)
5115                 (*count)--;
5116
5117         tracing_snapshot();
5118 }
5119
5120 static int
5121 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5122                       struct ftrace_probe_ops *ops, void *data)
5123 {
5124         long count = (long)data;
5125
5126         seq_printf(m, "%ps:", (void *)ip);
5127
5128         seq_printf(m, "snapshot");
5129
5130         if (count == -1)
5131                 seq_printf(m, ":unlimited\n");
5132         else
5133                 seq_printf(m, ":count=%ld\n", count);
5134
5135         return 0;
5136 }
5137
5138 static struct ftrace_probe_ops snapshot_probe_ops = {
5139         .func                   = ftrace_snapshot,
5140         .print                  = ftrace_snapshot_print,
5141 };
5142
5143 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5144         .func                   = ftrace_count_snapshot,
5145         .print                  = ftrace_snapshot_print,
5146 };
5147
5148 static int
5149 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5150                                char *glob, char *cmd, char *param, int enable)
5151 {
5152         struct ftrace_probe_ops *ops;
5153         void *count = (void *)-1;
5154         char *number;
5155         int ret;
5156
5157         /* hash funcs only work with set_ftrace_filter */
5158         if (!enable)
5159                 return -EINVAL;
5160
5161         ops = param ? &snapshot_count_probe_ops :  &snapshot_probe_ops;
5162
5163         if (glob[0] == '!') {
5164                 unregister_ftrace_function_probe_func(glob+1, ops);
5165                 return 0;
5166         }
5167
5168         if (!param)
5169                 goto out_reg;
5170
5171         number = strsep(&param, ":");
5172
5173         if (!strlen(number))
5174                 goto out_reg;
5175
5176         /*
5177          * We use the callback data field (which is a pointer)
5178          * as our counter.
5179          */
5180         ret = kstrtoul(number, 0, (unsigned long *)&count);
5181         if (ret)
5182                 return ret;
5183
5184  out_reg:
5185         ret = register_ftrace_function_probe(glob, ops, count);
5186
5187         if (ret >= 0)
5188                 alloc_snapshot(&global_trace);
5189
5190         return ret < 0 ? ret : 0;
5191 }
5192
5193 static struct ftrace_func_command ftrace_snapshot_cmd = {
5194         .name                   = "snapshot",
5195         .func                   = ftrace_trace_snapshot_callback,
5196 };
5197
5198 static int register_snapshot_cmd(void)
5199 {
5200         return register_ftrace_command(&ftrace_snapshot_cmd);
5201 }
5202 #else
5203 static inline int register_snapshot_cmd(void) { return 0; }
5204 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5205
5206 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5207 {
5208         static int once;
5209
5210         if (tr->dir)
5211                 return tr->dir;
5212
5213         if (!debugfs_initialized())
5214                 return NULL;
5215
5216         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5217                 tr->dir = debugfs_create_dir("tracing", NULL);
5218
5219         if (!tr->dir && !once) {
5220                 once = 1;
5221                 pr_warning("Could not create debugfs directory 'tracing'\n");
5222                 return NULL;
5223         }
5224
5225         return tr->dir;
5226 }
5227
5228 struct dentry *tracing_init_dentry(void)
5229 {
5230         return tracing_init_dentry_tr(&global_trace);
5231 }
5232
5233 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5234 {
5235         struct dentry *d_tracer;
5236
5237         if (tr->percpu_dir)
5238                 return tr->percpu_dir;
5239
5240         d_tracer = tracing_init_dentry_tr(tr);
5241         if (!d_tracer)
5242                 return NULL;
5243
5244         tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5245
5246         WARN_ONCE(!tr->percpu_dir,
5247                   "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5248
5249         return tr->percpu_dir;
5250 }
5251
5252 static void
5253 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5254 {
5255         struct trace_array_cpu *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
5256         struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5257         struct dentry *d_cpu;
5258         char cpu_dir[30]; /* 30 characters should be more than enough */
5259
5260         if (!d_percpu)
5261                 return;
5262
5263         snprintf(cpu_dir, 30, "cpu%ld", cpu);
5264         d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5265         if (!d_cpu) {
5266                 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5267                 return;
5268         }
5269
5270         /* per cpu trace_pipe */
5271         trace_create_file("trace_pipe", 0444, d_cpu,
5272                         (void *)&data->trace_cpu, &tracing_pipe_fops);
5273
5274         /* per cpu trace */
5275         trace_create_file("trace", 0644, d_cpu,
5276                         (void *)&data->trace_cpu, &tracing_fops);
5277
5278         trace_create_file("trace_pipe_raw", 0444, d_cpu,
5279                         (void *)&data->trace_cpu, &tracing_buffers_fops);
5280
5281         trace_create_file("stats", 0444, d_cpu,
5282                         (void *)&data->trace_cpu, &tracing_stats_fops);
5283
5284         trace_create_file("buffer_size_kb", 0444, d_cpu,
5285                         (void *)&data->trace_cpu, &tracing_entries_fops);
5286
5287 #ifdef CONFIG_TRACER_SNAPSHOT
5288         trace_create_file("snapshot", 0644, d_cpu,
5289                           (void *)&data->trace_cpu, &snapshot_fops);
5290
5291         trace_create_file("snapshot_raw", 0444, d_cpu,
5292                         (void *)&data->trace_cpu, &snapshot_raw_fops);
5293 #endif
5294 }
5295
5296 #ifdef CONFIG_FTRACE_SELFTEST
5297 /* Let selftest have access to static functions in this file */
5298 #include "trace_selftest.c"
5299 #endif
5300
5301 struct trace_option_dentry {
5302         struct tracer_opt               *opt;
5303         struct tracer_flags             *flags;
5304         struct trace_array              *tr;
5305         struct dentry                   *entry;
5306 };
5307
5308 static ssize_t
5309 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5310                         loff_t *ppos)
5311 {
5312         struct trace_option_dentry *topt = filp->private_data;
5313         char *buf;
5314
5315         if (topt->flags->val & topt->opt->bit)
5316                 buf = "1\n";
5317         else
5318                 buf = "0\n";
5319
5320         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5321 }
5322
5323 static ssize_t
5324 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5325                          loff_t *ppos)
5326 {
5327         struct trace_option_dentry *topt = filp->private_data;
5328         unsigned long val;
5329         int ret;
5330
5331         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5332         if (ret)
5333                 return ret;
5334
5335         if (val != 0 && val != 1)
5336                 return -EINVAL;
5337
5338         if (!!(topt->flags->val & topt->opt->bit) != val) {
5339                 mutex_lock(&trace_types_lock);
5340                 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
5341                                           topt->opt, !val);
5342                 mutex_unlock(&trace_types_lock);
5343                 if (ret)
5344                         return ret;
5345         }
5346
5347         *ppos += cnt;
5348
5349         return cnt;
5350 }
5351
5352
5353 static const struct file_operations trace_options_fops = {
5354         .open = tracing_open_generic,
5355         .read = trace_options_read,
5356         .write = trace_options_write,
5357         .llseek = generic_file_llseek,
5358 };
5359
5360 static ssize_t
5361 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5362                         loff_t *ppos)
5363 {
5364         long index = (long)filp->private_data;
5365         char *buf;
5366
5367         if (trace_flags & (1 << index))
5368                 buf = "1\n";
5369         else
5370                 buf = "0\n";
5371
5372         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5373 }
5374
5375 static ssize_t
5376 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5377                          loff_t *ppos)
5378 {
5379         struct trace_array *tr = &global_trace;
5380         long index = (long)filp->private_data;
5381         unsigned long val;
5382         int ret;
5383
5384         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5385         if (ret)
5386                 return ret;
5387
5388         if (val != 0 && val != 1)
5389                 return -EINVAL;
5390
5391         mutex_lock(&trace_types_lock);
5392         ret = set_tracer_flag(tr, 1 << index, val);
5393         mutex_unlock(&trace_types_lock);
5394
5395         if (ret < 0)
5396                 return ret;
5397
5398         *ppos += cnt;
5399
5400         return cnt;
5401 }
5402
5403 static const struct file_operations trace_options_core_fops = {
5404         .open = tracing_open_generic,
5405         .read = trace_options_core_read,
5406         .write = trace_options_core_write,
5407         .llseek = generic_file_llseek,
5408 };
5409
5410 struct dentry *trace_create_file(const char *name,
5411                                  umode_t mode,
5412                                  struct dentry *parent,
5413                                  void *data,
5414                                  const struct file_operations *fops)
5415 {
5416         struct dentry *ret;
5417
5418         ret = debugfs_create_file(name, mode, parent, data, fops);
5419         if (!ret)
5420                 pr_warning("Could not create debugfs '%s' entry\n", name);
5421
5422         return ret;
5423 }
5424
5425
5426 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
5427 {
5428         struct dentry *d_tracer;
5429
5430         if (tr->options)
5431                 return tr->options;
5432
5433         d_tracer = tracing_init_dentry_tr(tr);
5434         if (!d_tracer)
5435                 return NULL;
5436
5437         tr->options = debugfs_create_dir("options", d_tracer);
5438         if (!tr->options) {
5439                 pr_warning("Could not create debugfs directory 'options'\n");
5440                 return NULL;
5441         }
5442
5443         return tr->options;
5444 }
5445
5446 static void
5447 create_trace_option_file(struct trace_array *tr,
5448                          struct trace_option_dentry *topt,
5449                          struct tracer_flags *flags,
5450                          struct tracer_opt *opt)
5451 {
5452         struct dentry *t_options;
5453
5454         t_options = trace_options_init_dentry(tr);
5455         if (!t_options)
5456                 return;
5457
5458         topt->flags = flags;
5459         topt->opt = opt;
5460         topt->tr = tr;
5461
5462         topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
5463                                     &trace_options_fops);
5464
5465 }
5466
5467 static struct trace_option_dentry *
5468 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
5469 {
5470         struct trace_option_dentry *topts;
5471         struct tracer_flags *flags;
5472         struct tracer_opt *opts;
5473         int cnt;
5474
5475         if (!tracer)
5476                 return NULL;
5477
5478         flags = tracer->flags;
5479
5480         if (!flags || !flags->opts)
5481                 return NULL;
5482
5483         opts = flags->opts;
5484
5485         for (cnt = 0; opts[cnt].name; cnt++)
5486                 ;
5487
5488         topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
5489         if (!topts)
5490                 return NULL;
5491
5492         for (cnt = 0; opts[cnt].name; cnt++)
5493                 create_trace_option_file(tr, &topts[cnt], flags,
5494                                          &opts[cnt]);
5495
5496         return topts;
5497 }
5498
5499 static void
5500 destroy_trace_option_files(struct trace_option_dentry *topts)
5501 {
5502         int cnt;
5503
5504         if (!topts)
5505                 return;
5506
5507         for (cnt = 0; topts[cnt].opt; cnt++) {
5508                 if (topts[cnt].entry)
5509                         debugfs_remove(topts[cnt].entry);
5510         }
5511
5512         kfree(topts);
5513 }
5514
5515 static struct dentry *
5516 create_trace_option_core_file(struct trace_array *tr,
5517                               const char *option, long index)
5518 {
5519         struct dentry *t_options;
5520
5521         t_options = trace_options_init_dentry(tr);
5522         if (!t_options)
5523                 return NULL;
5524
5525         return trace_create_file(option, 0644, t_options, (void *)index,
5526                                     &trace_options_core_fops);
5527 }
5528
5529 static __init void create_trace_options_dir(struct trace_array *tr)
5530 {
5531         struct dentry *t_options;
5532         int i;
5533
5534         t_options = trace_options_init_dentry(tr);
5535         if (!t_options)
5536                 return;
5537
5538         for (i = 0; trace_options[i]; i++)
5539                 create_trace_option_core_file(tr, trace_options[i], i);
5540 }
5541
5542 static ssize_t
5543 rb_simple_read(struct file *filp, char __user *ubuf,
5544                size_t cnt, loff_t *ppos)
5545 {
5546         struct trace_array *tr = filp->private_data;
5547         struct ring_buffer *buffer = tr->trace_buffer.buffer;
5548         char buf[64];
5549         int r;
5550
5551         if (buffer)
5552                 r = ring_buffer_record_is_on(buffer);
5553         else
5554                 r = 0;
5555
5556         r = sprintf(buf, "%d\n", r);
5557
5558         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5559 }
5560
5561 static ssize_t
5562 rb_simple_write(struct file *filp, const char __user *ubuf,
5563                 size_t cnt, loff_t *ppos)
5564 {
5565         struct trace_array *tr = filp->private_data;
5566         struct ring_buffer *buffer = tr->trace_buffer.buffer;
5567         unsigned long val;
5568         int ret;
5569
5570         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5571         if (ret)
5572                 return ret;
5573
5574         if (buffer) {
5575                 mutex_lock(&trace_types_lock);
5576                 if (val) {
5577                         ring_buffer_record_on(buffer);
5578                         if (tr->current_trace->start)
5579                                 tr->current_trace->start(tr);
5580                 } else {
5581                         ring_buffer_record_off(buffer);
5582                         if (tr->current_trace->stop)
5583                                 tr->current_trace->stop(tr);
5584                 }
5585                 mutex_unlock(&trace_types_lock);
5586         }
5587
5588         (*ppos)++;
5589
5590         return cnt;
5591 }
5592
5593 static const struct file_operations rb_simple_fops = {
5594         .open           = tracing_open_generic,
5595         .read           = rb_simple_read,
5596         .write          = rb_simple_write,
5597         .llseek         = default_llseek,
5598 };
5599
5600 struct dentry *trace_instance_dir;
5601
5602 static void
5603 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5604
5605 static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
5606 {
5607         int cpu;
5608
5609         for_each_tracing_cpu(cpu) {
5610                 memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
5611                 per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
5612                 per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
5613         }
5614 }
5615
5616 static int
5617 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
5618 {
5619         enum ring_buffer_flags rb_flags;
5620
5621         rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5622
5623         buf->buffer = ring_buffer_alloc(size, rb_flags);
5624         if (!buf->buffer)
5625                 return -ENOMEM;
5626
5627         buf->data = alloc_percpu(struct trace_array_cpu);
5628         if (!buf->data) {
5629                 ring_buffer_free(buf->buffer);
5630                 return -ENOMEM;
5631         }
5632
5633         init_trace_buffers(tr, buf);
5634
5635         /* Allocate the first page for all buffers */
5636         set_buffer_entries(&tr->trace_buffer,
5637                            ring_buffer_size(tr->trace_buffer.buffer, 0));
5638
5639         return 0;
5640 }
5641
5642 static int allocate_trace_buffers(struct trace_array *tr, int size)
5643 {
5644         int ret;
5645
5646         ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5647         if (ret)
5648                 return ret;
5649
5650 #ifdef CONFIG_TRACER_MAX_TRACE
5651         ret = allocate_trace_buffer(tr, &tr->max_buffer,
5652                                     allocate_snapshot ? size : 1);
5653         if (WARN_ON(ret)) {
5654                 ring_buffer_free(tr->trace_buffer.buffer);
5655                 free_percpu(tr->trace_buffer.data);
5656                 return -ENOMEM;
5657         }
5658         tr->allocated_snapshot = allocate_snapshot;
5659
5660         /*
5661          * Only the top level trace array gets its snapshot allocated
5662          * from the kernel command line.
5663          */
5664         allocate_snapshot = false;
5665 #endif
5666         return 0;
5667 }
5668
5669 static int new_instance_create(const char *name)
5670 {
5671         struct trace_array *tr;
5672         int ret;
5673
5674         mutex_lock(&trace_types_lock);
5675
5676         ret = -EEXIST;
5677         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5678                 if (tr->name && strcmp(tr->name, name) == 0)
5679                         goto out_unlock;
5680         }
5681
5682         ret = -ENOMEM;
5683         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5684         if (!tr)
5685                 goto out_unlock;
5686
5687         tr->name = kstrdup(name, GFP_KERNEL);
5688         if (!tr->name)
5689                 goto out_free_tr;
5690
5691         raw_spin_lock_init(&tr->start_lock);
5692
5693         tr->current_trace = &nop_trace;
5694
5695         INIT_LIST_HEAD(&tr->systems);
5696         INIT_LIST_HEAD(&tr->events);
5697
5698         if (allocate_trace_buffers(tr, trace_buf_size) < 0)
5699                 goto out_free_tr;
5700
5701         /* Holder for file callbacks */
5702         tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5703         tr->trace_cpu.tr = tr;
5704
5705         tr->dir = debugfs_create_dir(name, trace_instance_dir);
5706         if (!tr->dir)
5707                 goto out_free_tr;
5708
5709         ret = event_trace_add_tracer(tr->dir, tr);
5710         if (ret)
5711                 goto out_free_tr;
5712
5713         init_tracer_debugfs(tr, tr->dir);
5714
5715         list_add(&tr->list, &ftrace_trace_arrays);
5716
5717         mutex_unlock(&trace_types_lock);
5718
5719         return 0;
5720
5721  out_free_tr:
5722         if (tr->trace_buffer.buffer)
5723                 ring_buffer_free(tr->trace_buffer.buffer);
5724         kfree(tr->name);
5725         kfree(tr);
5726
5727  out_unlock:
5728         mutex_unlock(&trace_types_lock);
5729
5730         return ret;
5731
5732 }
5733
5734 static int instance_delete(const char *name)
5735 {
5736         struct trace_array *tr;
5737         int found = 0;
5738         int ret;
5739
5740         mutex_lock(&trace_types_lock);
5741
5742         ret = -ENODEV;
5743         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5744                 if (tr->name && strcmp(tr->name, name) == 0) {
5745                         found = 1;
5746                         break;
5747                 }
5748         }
5749         if (!found)
5750                 goto out_unlock;
5751
5752         ret = -EBUSY;
5753         if (tr->ref)
5754                 goto out_unlock;
5755
5756         list_del(&tr->list);
5757
5758         event_trace_del_tracer(tr);
5759         debugfs_remove_recursive(tr->dir);
5760         free_percpu(tr->trace_buffer.data);
5761         ring_buffer_free(tr->trace_buffer.buffer);
5762
5763         kfree(tr->name);
5764         kfree(tr);
5765
5766         ret = 0;
5767
5768  out_unlock:
5769         mutex_unlock(&trace_types_lock);
5770
5771         return ret;
5772 }
5773
5774 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
5775 {
5776         struct dentry *parent;
5777         int ret;
5778
5779         /* Paranoid: Make sure the parent is the "instances" directory */
5780         parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5781         if (WARN_ON_ONCE(parent != trace_instance_dir))
5782                 return -ENOENT;
5783
5784         /*
5785          * The inode mutex is locked, but debugfs_create_dir() will also
5786          * take the mutex. As the instances directory can not be destroyed
5787          * or changed in any other way, it is safe to unlock it, and
5788          * let the dentry try. If two users try to make the same dir at
5789          * the same time, then the new_instance_create() will determine the
5790          * winner.
5791          */
5792         mutex_unlock(&inode->i_mutex);
5793
5794         ret = new_instance_create(dentry->d_iname);
5795
5796         mutex_lock(&inode->i_mutex);
5797
5798         return ret;
5799 }
5800
5801 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
5802 {
5803         struct dentry *parent;
5804         int ret;
5805
5806         /* Paranoid: Make sure the parent is the "instances" directory */
5807         parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5808         if (WARN_ON_ONCE(parent != trace_instance_dir))
5809                 return -ENOENT;
5810
5811         /* The caller did a dget() on dentry */
5812         mutex_unlock(&dentry->d_inode->i_mutex);
5813
5814         /*
5815          * The inode mutex is locked, but debugfs_create_dir() will also
5816          * take the mutex. As the instances directory can not be destroyed
5817          * or changed in any other way, it is safe to unlock it, and
5818          * let the dentry try. If two users try to make the same dir at
5819          * the same time, then the instance_delete() will determine the
5820          * winner.
5821          */
5822         mutex_unlock(&inode->i_mutex);
5823
5824         ret = instance_delete(dentry->d_iname);
5825
5826         mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
5827         mutex_lock(&dentry->d_inode->i_mutex);
5828
5829         return ret;
5830 }
5831
5832 static const struct inode_operations instance_dir_inode_operations = {
5833         .lookup         = simple_lookup,
5834         .mkdir          = instance_mkdir,
5835         .rmdir          = instance_rmdir,
5836 };
5837
5838 static __init void create_trace_instances(struct dentry *d_tracer)
5839 {
5840         trace_instance_dir = debugfs_create_dir("instances", d_tracer);
5841         if (WARN_ON(!trace_instance_dir))
5842                 return;
5843
5844         /* Hijack the dir inode operations, to allow mkdir */
5845         trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
5846 }
5847
5848 static void
5849 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
5850 {
5851         int cpu;
5852
5853         trace_create_file("trace_options", 0644, d_tracer,
5854                           tr, &tracing_iter_fops);
5855
5856         trace_create_file("trace", 0644, d_tracer,
5857                         (void *)&tr->trace_cpu, &tracing_fops);
5858
5859         trace_create_file("trace_pipe", 0444, d_tracer,
5860                         (void *)&tr->trace_cpu, &tracing_pipe_fops);
5861
5862         trace_create_file("buffer_size_kb", 0644, d_tracer,
5863                         (void *)&tr->trace_cpu, &tracing_entries_fops);
5864
5865         trace_create_file("buffer_total_size_kb", 0444, d_tracer,
5866                           tr, &tracing_total_entries_fops);
5867
5868         trace_create_file("free_buffer", 0644, d_tracer,
5869                           tr, &tracing_free_buffer_fops);
5870
5871         trace_create_file("trace_marker", 0220, d_tracer,
5872                           tr, &tracing_mark_fops);
5873
5874         trace_create_file("trace_clock", 0644, d_tracer, tr,
5875                           &trace_clock_fops);
5876
5877         trace_create_file("tracing_on", 0644, d_tracer,
5878                             tr, &rb_simple_fops);
5879
5880 #ifdef CONFIG_TRACER_SNAPSHOT
5881         trace_create_file("snapshot", 0644, d_tracer,
5882                           (void *)&tr->trace_cpu, &snapshot_fops);
5883 #endif
5884
5885         for_each_tracing_cpu(cpu)
5886                 tracing_init_debugfs_percpu(tr, cpu);
5887
5888 }
5889
5890 static __init int tracer_init_debugfs(void)
5891 {
5892         struct dentry *d_tracer;
5893
5894         trace_access_lock_init();
5895
5896         d_tracer = tracing_init_dentry();
5897
5898         init_tracer_debugfs(&global_trace, d_tracer);
5899
5900         trace_create_file("tracing_cpumask", 0644, d_tracer,
5901                         &global_trace, &tracing_cpumask_fops);
5902
5903         trace_create_file("available_tracers", 0444, d_tracer,
5904                         &global_trace, &show_traces_fops);
5905
5906         trace_create_file("current_tracer", 0644, d_tracer,
5907                         &global_trace, &set_tracer_fops);
5908
5909 #ifdef CONFIG_TRACER_MAX_TRACE
5910         trace_create_file("tracing_max_latency", 0644, d_tracer,
5911                         &tracing_max_latency, &tracing_max_lat_fops);
5912 #endif
5913
5914         trace_create_file("tracing_thresh", 0644, d_tracer,
5915                         &tracing_thresh, &tracing_max_lat_fops);
5916
5917         trace_create_file("README", 0444, d_tracer,
5918                         NULL, &tracing_readme_fops);
5919
5920         trace_create_file("saved_cmdlines", 0444, d_tracer,
5921                         NULL, &tracing_saved_cmdlines_fops);
5922
5923 #ifdef CONFIG_DYNAMIC_FTRACE
5924         trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
5925                         &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
5926 #endif
5927
5928         create_trace_instances(d_tracer);
5929
5930         create_trace_options_dir(&global_trace);
5931
5932         return 0;
5933 }
5934
5935 static int trace_panic_handler(struct notifier_block *this,
5936                                unsigned long event, void *unused)
5937 {
5938         if (ftrace_dump_on_oops)
5939                 ftrace_dump(ftrace_dump_on_oops);
5940         return NOTIFY_OK;
5941 }
5942
5943 static struct notifier_block trace_panic_notifier = {
5944         .notifier_call  = trace_panic_handler,
5945         .next           = NULL,
5946         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
5947 };
5948
5949 static int trace_die_handler(struct notifier_block *self,
5950                              unsigned long val,
5951                              void *data)
5952 {
5953         switch (val) {
5954         case DIE_OOPS:
5955                 if (ftrace_dump_on_oops)
5956                         ftrace_dump(ftrace_dump_on_oops);
5957                 break;
5958         default:
5959                 break;
5960         }
5961         return NOTIFY_OK;
5962 }
5963
5964 static struct notifier_block trace_die_notifier = {
5965         .notifier_call = trace_die_handler,
5966         .priority = 200
5967 };
5968
5969 /*
5970  * printk is set to max of 1024, we really don't need it that big.
5971  * Nothing should be printing 1000 characters anyway.
5972  */
5973 #define TRACE_MAX_PRINT         1000
5974
5975 /*
5976  * Define here KERN_TRACE so that we have one place to modify
5977  * it if we decide to change what log level the ftrace dump
5978  * should be at.
5979  */
5980 #define KERN_TRACE              KERN_EMERG
5981
5982 void
5983 trace_printk_seq(struct trace_seq *s)
5984 {
5985         /* Probably should print a warning here. */
5986         if (s->len >= 1000)
5987                 s->len = 1000;
5988
5989         /* should be zero ended, but we are paranoid. */
5990         s->buffer[s->len] = 0;
5991
5992         printk(KERN_TRACE "%s", s->buffer);
5993
5994         trace_seq_init(s);
5995 }
5996
5997 void trace_init_global_iter(struct trace_iterator *iter)
5998 {
5999         iter->tr = &global_trace;
6000         iter->trace = iter->tr->current_trace;
6001         iter->cpu_file = RING_BUFFER_ALL_CPUS;
6002         iter->trace_buffer = &global_trace.trace_buffer;
6003 }
6004
6005 static void
6006 __ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
6007 {
6008         static arch_spinlock_t ftrace_dump_lock =
6009                 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6010         /* use static because iter can be a bit big for the stack */
6011         static struct trace_iterator iter;
6012         unsigned int old_userobj;
6013         static int dump_ran;
6014         unsigned long flags;
6015         int cnt = 0, cpu;
6016
6017         /* only one dump */
6018         local_irq_save(flags);
6019         arch_spin_lock(&ftrace_dump_lock);
6020         if (dump_ran)
6021                 goto out;
6022
6023         dump_ran = 1;
6024
6025         tracing_off();
6026
6027         /* Did function tracer already get disabled? */
6028         if (ftrace_is_dead()) {
6029                 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6030                 printk("#          MAY BE MISSING FUNCTION EVENTS\n");
6031         }
6032
6033         if (disable_tracing)
6034                 ftrace_kill();
6035
6036         /* Simulate the iterator */
6037         trace_init_global_iter(&iter);
6038
6039         for_each_tracing_cpu(cpu) {
6040                 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6041         }
6042
6043         old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6044
6045         /* don't look at user memory in panic mode */
6046         trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6047
6048         switch (oops_dump_mode) {
6049         case DUMP_ALL:
6050                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6051                 break;
6052         case DUMP_ORIG:
6053                 iter.cpu_file = raw_smp_processor_id();
6054                 break;
6055         case DUMP_NONE:
6056                 goto out_enable;
6057         default:
6058                 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6059                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6060         }
6061
6062         printk(KERN_TRACE "Dumping ftrace buffer:\n");
6063
6064         /*
6065          * We need to stop all tracing on all CPUS to read the
6066          * the next buffer. This is a bit expensive, but is
6067          * not done often. We fill all what we can read,
6068          * and then release the locks again.
6069          */
6070
6071         while (!trace_empty(&iter)) {
6072
6073                 if (!cnt)
6074                         printk(KERN_TRACE "---------------------------------\n");
6075
6076                 cnt++;
6077
6078                 /* reset all but tr, trace, and overruns */
6079                 memset(&iter.seq, 0,
6080                        sizeof(struct trace_iterator) -
6081                        offsetof(struct trace_iterator, seq));
6082                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6083                 iter.pos = -1;
6084
6085                 if (trace_find_next_entry_inc(&iter) != NULL) {
6086                         int ret;
6087
6088                         ret = print_trace_line(&iter);
6089                         if (ret != TRACE_TYPE_NO_CONSUME)
6090                                 trace_consume(&iter);
6091                 }
6092                 touch_nmi_watchdog();
6093
6094                 trace_printk_seq(&iter.seq);
6095         }
6096
6097         if (!cnt)
6098                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
6099         else
6100                 printk(KERN_TRACE "---------------------------------\n");
6101
6102  out_enable:
6103         /* Re-enable tracing if requested */
6104         if (!disable_tracing) {
6105                 trace_flags |= old_userobj;
6106
6107                 for_each_tracing_cpu(cpu) {
6108                         atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6109                 }
6110                 tracing_on();
6111         }
6112
6113  out:
6114         arch_spin_unlock(&ftrace_dump_lock);
6115         local_irq_restore(flags);
6116 }
6117
6118 /* By default: disable tracing after the dump */
6119 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6120 {
6121         __ftrace_dump(true, oops_dump_mode);
6122 }
6123 EXPORT_SYMBOL_GPL(ftrace_dump);
6124
6125 __init static int tracer_alloc_buffers(void)
6126 {
6127         int ring_buf_size;
6128         int ret = -ENOMEM;
6129
6130
6131         if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6132                 goto out;
6133
6134         if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
6135                 goto out_free_buffer_mask;
6136
6137         /* Only allocate trace_printk buffers if a trace_printk exists */
6138         if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6139                 /* Must be called before global_trace.buffer is allocated */
6140                 trace_printk_init_buffers();
6141
6142         /* To save memory, keep the ring buffer size to its minimum */
6143         if (ring_buffer_expanded)
6144                 ring_buf_size = trace_buf_size;
6145         else
6146                 ring_buf_size = 1;
6147
6148         cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6149         cpumask_copy(tracing_cpumask, cpu_all_mask);
6150
6151         raw_spin_lock_init(&global_trace.start_lock);
6152
6153         /* TODO: make the number of buffers hot pluggable with CPUS */
6154         if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6155                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6156                 WARN_ON(1);
6157                 goto out_free_cpumask;
6158         }
6159
6160         if (global_trace.buffer_disabled)
6161                 tracing_off();
6162
6163         trace_init_cmdlines();
6164
6165         register_tracer(&nop_trace);
6166
6167         global_trace.current_trace = &nop_trace;
6168
6169         /* All seems OK, enable tracing */
6170         tracing_disabled = 0;
6171
6172         atomic_notifier_chain_register(&panic_notifier_list,
6173                                        &trace_panic_notifier);
6174
6175         register_die_notifier(&trace_die_notifier);
6176
6177         global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6178
6179         /* Holder for file callbacks */
6180         global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6181         global_trace.trace_cpu.tr = &global_trace;
6182
6183         INIT_LIST_HEAD(&global_trace.systems);
6184         INIT_LIST_HEAD(&global_trace.events);
6185         list_add(&global_trace.list, &ftrace_trace_arrays);
6186
6187         while (trace_boot_options) {
6188                 char *option;
6189
6190                 option = strsep(&trace_boot_options, ",");
6191                 trace_set_options(&global_trace, option);
6192         }
6193
6194         register_snapshot_cmd();
6195
6196         return 0;
6197
6198 out_free_cpumask:
6199         free_percpu(global_trace.trace_buffer.data);
6200 #ifdef CONFIG_TRACER_MAX_TRACE
6201         free_percpu(global_trace.max_buffer.data);
6202 #endif
6203         free_cpumask_var(tracing_cpumask);
6204 out_free_buffer_mask:
6205         free_cpumask_var(tracing_buffer_mask);
6206 out:
6207         return ret;
6208 }
6209
6210 __init static int clear_boot_tracer(void)
6211 {
6212         /*
6213          * The default tracer at boot buffer is an init section.
6214          * This function is called in lateinit. If we did not
6215          * find the boot tracer, then clear it out, to prevent
6216          * later registration from accessing the buffer that is
6217          * about to be freed.
6218          */
6219         if (!default_bootup_tracer)
6220                 return 0;
6221
6222         printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6223                default_bootup_tracer);
6224         default_bootup_tracer = NULL;
6225
6226         return 0;
6227 }
6228
6229 early_initcall(tracer_alloc_buffers);
6230 fs_initcall(tracer_init_debugfs);
6231 late_initcall(clear_boot_tracer);