blob: 3f5046a4d81b23cdcb463e09a1c89dc5971ccf2d [file] [log] [blame]
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +010012 * Copyright (C) 2004 Nadia Yvette Chambers
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020013 */
Steven Rostedt2cadf912008-12-01 22:20:19 -050014#include <linux/ring_buffer.h>
Sam Ravnborg273b2812009-10-18 00:52:28 +020015#include <generated/utsrelease.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050016#include <linux/stacktrace.h>
17#include <linux/writeback.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020018#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040020#include <linux/notifier.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050021#include <linux/irqflags.h>
Steven Rostedt0d5c6e12012-11-01 20:54:21 -040022#include <linux/irq_work.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020023#include <linux/debugfs.h>
Steven Rostedt4c11d7a2008-05-12 21:20:43 +020024#include <linux/pagemap.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020025#include <linux/hardirq.h>
26#include <linux/linkage.h>
27#include <linux/uaccess.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050028#include <linux/kprobes.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020029#include <linux/ftrace.h>
30#include <linux/module.h>
31#include <linux/percpu.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050032#include <linux/splice.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040033#include <linux/kdebug.h>
Frederic Weisbecker5f0c6c02009-03-27 14:22:10 +010034#include <linux/string.h>
Lai Jiangshan7e53bd42010-01-06 20:08:50 +080035#include <linux/rwsem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090036#include <linux/slab.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020037#include <linux/ctype.h>
38#include <linux/init.h>
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +020039#include <linux/poll.h>
Steven Rostedtb892e5c2012-03-01 22:06:48 -050040#include <linux/nmi.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020041#include <linux/fs.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060042#include <linux/sched/rt.h>
Ingo Molnar86387f72008-05-12 21:20:51 +020043
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020044#include "trace.h"
Steven Rostedtf0868d12008-12-23 23:24:12 -050045#include "trace_output.h"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020046
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010047/*
Steven Rostedt73c51622009-03-11 13:42:01 -040048 * On boot up, the ring buffer is set to the minimum size, so that
49 * we do not waste memory on systems that are not using tracing.
50 */
Li Zefan020e5f82009-07-01 10:47:05 +080051int ring_buffer_expanded;
Steven Rostedt73c51622009-03-11 13:42:01 -040052
53/*
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010054 * We need to change this state when a selftest is running.
Frederic Weisbeckerff325042008-12-04 23:47:35 +010055 * A selftest will lurk into the ring-buffer to count the
56 * entries inserted during the selftest although some concurrent
Ingo Molnar5e1607a2009-03-05 10:24:48 +010057 * insertions into the ring-buffer such as trace_printk could occurred
Frederic Weisbeckerff325042008-12-04 23:47:35 +010058 * at the same time, giving false positive or negative results.
59 */
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010060static bool __read_mostly tracing_selftest_running;
Frederic Weisbeckerff325042008-12-04 23:47:35 +010061
Steven Rostedtb2821ae2009-02-02 21:38:32 -050062/*
63 * If a tracer is running, we do not want to run SELFTEST.
64 */
Li Zefan020e5f82009-07-01 10:47:05 +080065bool __read_mostly tracing_selftest_disabled;
Steven Rostedtb2821ae2009-02-02 21:38:32 -050066
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010067/* For tracers that don't implement custom flags */
68static struct tracer_opt dummy_tracer_opt[] = {
69 { }
70};
71
72static struct tracer_flags dummy_tracer_flags = {
73 .val = 0,
74 .opts = dummy_tracer_opt
75};
76
77static int dummy_set_flag(u32 old_flags, u32 bit, int set)
78{
79 return 0;
80}
Steven Rostedt0f048702008-11-05 16:05:44 -050081
82/*
Steven Rostedt7ffbd482012-10-11 12:14:25 -040083 * To prevent the comm cache from being overwritten when no
84 * tracing is active, only save the comm when a trace event
85 * occurred.
86 */
87static DEFINE_PER_CPU(bool, trace_cmdline_save);
88
89/*
Steven Rostedt0d5c6e12012-11-01 20:54:21 -040090 * When a reader is waiting for data, then this variable is
91 * set to true.
92 */
93static bool trace_wakeup_needed;
94
95static struct irq_work trace_work_wakeup;
96
97/*
Steven Rostedt0f048702008-11-05 16:05:44 -050098 * Kill all tracing for good (never come back).
99 * It is initialized to 1 but will turn to zero if the initialization
100 * of the tracer is successful. But that is the only place that sets
101 * this back to zero.
102 */
Hannes Eder4fd27352009-02-10 19:44:12 +0100103static int tracing_disabled = 1;
Steven Rostedt0f048702008-11-05 16:05:44 -0500104
Christoph Lameter9288f992009-10-07 19:17:45 -0400105DEFINE_PER_CPU(int, ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -0400106
Jason Wessel955b61e2010-08-05 09:22:23 -0500107cpumask_var_t __read_mostly tracing_buffer_mask;
Steven Rostedtab464282008-05-12 21:21:00 +0200108
Steven Rostedt944ac422008-10-23 19:26:08 -0400109/*
110 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
111 *
112 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
113 * is set, then ftrace_dump is called. This will output the contents
114 * of the ftrace buffers to the console. This is very useful for
115 * capturing traces that lead to crashes and outputing it to a
116 * serial console.
117 *
118 * It is default off, but you can enable it with either specifying
119 * "ftrace_dump_on_oops" in the kernel command line, or setting
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200120 * /proc/sys/kernel/ftrace_dump_on_oops
121 * Set 1 if you want to dump buffers of all CPUs
122 * Set 2 if you want to dump the buffer of the CPU that triggered oops
Steven Rostedt944ac422008-10-23 19:26:08 -0400123 */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200124
125enum ftrace_dump_mode ftrace_dump_on_oops;
Steven Rostedt944ac422008-10-23 19:26:08 -0400126
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500127static int tracing_set_tracer(const char *buf);
128
Li Zefanee6c2c12009-09-18 14:06:47 +0800129#define MAX_TRACER_SIZE 100
130static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500131static char *default_bootup_tracer;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100132
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200133static int __init set_cmdline_ftrace(char *str)
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100134{
Chen Gang67012ab2013-04-08 12:06:44 +0800135 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500136 default_bootup_tracer = bootup_tracer_buf;
Steven Rostedt73c51622009-03-11 13:42:01 -0400137 /* We are using ftrace early, expand it */
138 ring_buffer_expanded = 1;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100139 return 1;
140}
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200141__setup("ftrace=", set_cmdline_ftrace);
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100142
Steven Rostedt944ac422008-10-23 19:26:08 -0400143static int __init set_ftrace_dump_on_oops(char *str)
144{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200145 if (*str++ != '=' || !*str) {
146 ftrace_dump_on_oops = DUMP_ALL;
147 return 1;
148 }
149
150 if (!strcmp("orig_cpu", str)) {
151 ftrace_dump_on_oops = DUMP_ORIG;
152 return 1;
153 }
154
155 return 0;
Steven Rostedt944ac422008-10-23 19:26:08 -0400156}
157__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
Steven Rostedt60a11772008-05-12 21:20:44 +0200158
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400159
160static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
161static char *trace_boot_options __initdata;
162
163static int __init set_trace_boot_options(char *str)
164{
Chen Gang67012ab2013-04-08 12:06:44 +0800165 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400166 trace_boot_options = trace_boot_options_buf;
167 return 0;
168}
169__setup("trace_options=", set_trace_boot_options);
170
Lai Jiangshancf8e3472009-03-30 13:48:00 +0800171unsigned long long ns2usecs(cycle_t nsec)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200172{
173 nsec += 500;
174 do_div(nsec, 1000);
175 return nsec;
176}
177
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200178/*
179 * The global_trace is the descriptor that holds the tracing
180 * buffers for the live tracing. For each CPU, it contains
181 * a link list of pages that will store trace entries. The
182 * page descriptor of the pages in the memory is used to hold
183 * the link list by linking the lru item in the page descriptor
184 * to each of the pages in the buffer per CPU.
185 *
186 * For each active CPU there is a data field that holds the
187 * pages for the buffer for that CPU. Each CPU has the same number
188 * of pages allocated for its buffer.
189 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200190static struct trace_array global_trace;
191
192static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
193
Steven Rostedte77405a2009-09-02 14:17:06 -0400194int filter_current_check_discard(struct ring_buffer *buffer,
195 struct ftrace_event_call *call, void *rec,
Tom Zanussieb02ce02009-04-08 03:15:54 -0500196 struct ring_buffer_event *event)
197{
Steven Rostedte77405a2009-09-02 14:17:06 -0400198 return filter_check_discard(call, rec, buffer, event);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500199}
Steven Rostedt17c873e2009-04-10 18:12:50 -0400200EXPORT_SYMBOL_GPL(filter_current_check_discard);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500201
Steven Rostedt37886f62009-03-17 17:22:06 -0400202cycle_t ftrace_now(int cpu)
203{
204 u64 ts;
205
206 /* Early boot up does not have a buffer yet */
207 if (!global_trace.buffer)
208 return trace_clock_local();
209
210 ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
211 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
212
213 return ts;
214}
215
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200216/*
217 * The max_tr is used to snapshot the global_trace when a maximum
218 * latency is reached. Some tracers will use this to store a maximum
219 * trace while it continues examining live traces.
220 *
221 * The buffers for the max_tr are set up the same as the global_trace.
222 * When a snapshot is taken, the link list of the max_tr is swapped
223 * with the link list of the global_trace and the buffers are reset for
224 * the global_trace so the tracing can continue.
225 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200226static struct trace_array max_tr;
227
Tejun Heo9705f692009-10-29 22:34:13 +0900228static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200229
Steven Rostedt90369902008-11-05 16:05:44 -0500230int tracing_is_enabled(void)
231{
Steven Rostedt0fb96562012-05-11 14:25:30 -0400232 return tracing_is_on();
Steven Rostedt90369902008-11-05 16:05:44 -0500233}
234
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200235/*
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400236 * trace_buf_size is the size in bytes that is allocated
237 * for a buffer. Note, the number of bytes is always rounded
238 * to page size.
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400239 *
240 * This number is purposely set to a low number of 16384.
241 * If the dump on oops happens, it will be much appreciated
242 * to not have to wait for all that output. Anyway this can be
243 * boot time and run time configurable.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200244 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400245#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400246
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400247static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200248
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200249/* trace_types holds a link list of available tracers. */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200250static struct tracer *trace_types __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200251
252/* current_trace points to the tracer that is currently active */
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -0500253static struct tracer *current_trace __read_mostly = &nop_trace;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200254
255/*
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200256 * trace_types_lock is used to protect the trace_types list.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200257 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200258static DEFINE_MUTEX(trace_types_lock);
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200259
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800260/*
261 * serialize the access of the ring buffer
262 *
263 * ring buffer serializes readers, but it is low level protection.
264 * The validity of the events (which returns by ring_buffer_peek() ..etc)
265 * are not protected by ring buffer.
266 *
267 * The content of events may become garbage if we allow other process consumes
268 * these events concurrently:
269 * A) the page of the consumed events may become a normal page
270 * (not reader page) in ring buffer, and this page will be rewrited
271 * by events producer.
272 * B) The page of the consumed events may become a page for splice_read,
273 * and this page will be returned to system.
274 *
275 * These primitives allow multi process access to different cpu ring buffer
276 * concurrently.
277 *
278 * These primitives don't distinguish read-only and read-consume access.
279 * Multi read-only access are also serialized.
280 */
281
282#ifdef CONFIG_SMP
283static DECLARE_RWSEM(all_cpu_access_lock);
284static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
285
286static inline void trace_access_lock(int cpu)
287{
288 if (cpu == TRACE_PIPE_ALL_CPU) {
289 /* gain it for accessing the whole ring buffer. */
290 down_write(&all_cpu_access_lock);
291 } else {
292 /* gain it for accessing a cpu ring buffer. */
293
294 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
295 down_read(&all_cpu_access_lock);
296
297 /* Secondly block other access to this @cpu ring buffer. */
298 mutex_lock(&per_cpu(cpu_access_lock, cpu));
299 }
300}
301
302static inline void trace_access_unlock(int cpu)
303{
304 if (cpu == TRACE_PIPE_ALL_CPU) {
305 up_write(&all_cpu_access_lock);
306 } else {
307 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
308 up_read(&all_cpu_access_lock);
309 }
310}
311
312static inline void trace_access_lock_init(void)
313{
314 int cpu;
315
316 for_each_possible_cpu(cpu)
317 mutex_init(&per_cpu(cpu_access_lock, cpu));
318}
319
320#else
321
322static DEFINE_MUTEX(access_lock);
323
324static inline void trace_access_lock(int cpu)
325{
326 (void)cpu;
327 mutex_lock(&access_lock);
328}
329
330static inline void trace_access_unlock(int cpu)
331{
332 (void)cpu;
333 mutex_unlock(&access_lock);
334}
335
336static inline void trace_access_lock_init(void)
337{
338}
339
340#endif
341
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200342/* trace_wait is a waitqueue for tasks blocked on trace_poll */
Ingo Molnar4e655512008-05-12 21:20:52 +0200343static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
344
Steven Rostedtee6bce52008-11-12 17:52:37 -0500345/* trace_flags holds trace_options default values */
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500346unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
Steven Rostedta2a16d62009-03-24 23:17:58 -0400347 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
Steven Rostedt77271ce2011-11-17 09:34:33 -0500348 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
Mandeep Singh Baines5224c3a2012-09-07 18:12:19 -0700349 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS;
Ingo Molnar4e655512008-05-12 21:20:52 +0200350
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400351static int trace_stop_count;
Thomas Gleixner5389f6f2009-07-25 17:13:33 +0200352static DEFINE_RAW_SPINLOCK(tracing_start_lock);
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400353
Steven Rostedt0d5c6e12012-11-01 20:54:21 -0400354/**
355 * trace_wake_up - wake up tasks waiting for trace input
356 *
357 * Schedules a delayed work to wake up any task that is blocked on the
358 * trace_wait queue. These is used with trace_poll for tasks polling the
359 * trace.
360 */
361static void trace_wake_up(struct irq_work *work)
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700362{
Steven Rostedt0d5c6e12012-11-01 20:54:21 -0400363 wake_up_all(&trace_wait);
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700364
Steven Rostedt0d5c6e12012-11-01 20:54:21 -0400365}
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700366
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200367/**
Steven Rostedt499e5472012-02-22 15:50:28 -0500368 * tracing_on - enable tracing buffers
369 *
370 * This function enables tracing buffers that may have been
371 * disabled with tracing_off.
372 */
373void tracing_on(void)
374{
375 if (global_trace.buffer)
376 ring_buffer_record_on(global_trace.buffer);
377 /*
378 * This flag is only looked at when buffers haven't been
379 * allocated yet. We don't really care about the race
380 * between setting this flag and actually turning
381 * on the buffer.
382 */
383 global_trace.buffer_disabled = 0;
384}
385EXPORT_SYMBOL_GPL(tracing_on);
386
387/**
388 * tracing_off - turn off tracing buffers
389 *
390 * This function stops the tracing buffers from recording data.
391 * It does not disable any overhead the tracers themselves may
392 * be causing. This function simply causes all recording to
393 * the ring buffers to fail.
394 */
395void tracing_off(void)
396{
397 if (global_trace.buffer)
Steven Rostedtf2bf1f62012-06-06 19:50:40 -0400398 ring_buffer_record_off(global_trace.buffer);
Steven Rostedt499e5472012-02-22 15:50:28 -0500399 /*
400 * This flag is only looked at when buffers haven't been
401 * allocated yet. We don't really care about the race
402 * between setting this flag and actually turning
403 * on the buffer.
404 */
405 global_trace.buffer_disabled = 1;
406}
407EXPORT_SYMBOL_GPL(tracing_off);
408
409/**
410 * tracing_is_on - show state of ring buffers enabled
411 */
412int tracing_is_on(void)
413{
414 if (global_trace.buffer)
415 return ring_buffer_record_is_on(global_trace.buffer);
416 return !global_trace.buffer_disabled;
417}
418EXPORT_SYMBOL_GPL(tracing_is_on);
419
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400420static int __init set_buf_size(char *str)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200421{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400422 unsigned long buf_size;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200423
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200424 if (!str)
425 return 0;
Li Zefan9d612be2009-06-24 17:33:15 +0800426 buf_size = memparse(str, &str);
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200427 /* nr_entries can not be zero */
Li Zefan9d612be2009-06-24 17:33:15 +0800428 if (buf_size == 0)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200429 return 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400430 trace_buf_size = buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200431 return 1;
432}
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400433__setup("trace_buf_size=", set_buf_size);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200434
Tim Bird0e950172010-02-25 15:36:43 -0800435static int __init set_tracing_thresh(char *str)
436{
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800437 unsigned long threshold;
Tim Bird0e950172010-02-25 15:36:43 -0800438 int ret;
439
440 if (!str)
441 return 0;
Daniel Walterbcd83ea2012-09-26 22:08:38 +0200442 ret = kstrtoul(str, 0, &threshold);
Tim Bird0e950172010-02-25 15:36:43 -0800443 if (ret < 0)
444 return 0;
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800445 tracing_thresh = threshold * 1000;
Tim Bird0e950172010-02-25 15:36:43 -0800446 return 1;
447}
448__setup("tracing_thresh=", set_tracing_thresh);
449
Steven Rostedt57f50be2008-05-12 21:20:44 +0200450unsigned long nsecs_to_usecs(unsigned long nsecs)
451{
452 return nsecs / 1000;
453}
454
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200455/* These must match the bit postions in trace_iterator_flags */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200456static const char *trace_options[] = {
457 "print-parent",
458 "sym-offset",
459 "sym-addr",
460 "verbose",
Ingo Molnarf9896bf2008-05-12 21:20:47 +0200461 "raw",
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +0200462 "hex",
Ingo Molnarcb0f12a2008-05-12 21:20:47 +0200463 "bin",
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +0200464 "block",
Ingo Molnar86387f72008-05-12 21:20:51 +0200465 "stacktrace",
Ingo Molnar5e1607a2009-03-05 10:24:48 +0100466 "trace_printk",
Steven Rostedtb2a866f2008-11-03 23:15:57 -0500467 "ftrace_preempt",
Steven Rostedt9f029e82008-11-12 15:24:24 -0500468 "branch",
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500469 "annotate",
Török Edwin02b67512008-11-22 13:28:47 +0200470 "userstacktrace",
Török Edwinb54d3de2008-11-22 13:28:48 +0200471 "sym-userobj",
Frederic Weisbecker66896a82008-12-13 20:18:13 +0100472 "printk-msg-only",
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -0200473 "context-info",
Steven Rostedtc032ef642009-03-04 20:34:24 -0500474 "latency-format",
Steven Rostedtbe6f1642009-03-24 11:06:24 -0400475 "sleep-time",
Steven Rostedta2a16d62009-03-24 23:17:58 -0400476 "graph-time",
Li Zefane870e9a2010-07-02 11:07:32 +0800477 "record-cmd",
David Sharp750912f2010-12-08 13:46:47 -0800478 "overwrite",
Steven Rostedtcf30cf62011-06-14 22:44:07 -0400479 "disable_on_free",
Steven Rostedt77271ce2011-11-17 09:34:33 -0500480 "irq-info",
Mandeep Singh Baines5224c3a2012-09-07 18:12:19 -0700481 "markers",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200482 NULL
483};
484
Zhaolei5079f322009-08-25 16:12:56 +0800485static struct {
486 u64 (*func)(void);
487 const char *name;
David Sharp8be07092012-11-13 12:18:22 -0800488 int in_ns; /* is this clock in nanoseconds? */
Zhaolei5079f322009-08-25 16:12:56 +0800489} trace_clocks[] = {
David Sharp8be07092012-11-13 12:18:22 -0800490 { trace_clock_local, "local", 1 },
491 { trace_clock_global, "global", 1 },
492 { trace_clock_counter, "counter", 0 },
David Sharp8cbd9cc2012-11-13 12:18:21 -0800493 ARCH_TRACE_CLOCKS
Zhaolei5079f322009-08-25 16:12:56 +0800494};
495
496int trace_clock_id;
497
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200498/*
499 * trace_parser_get_init - gets the buffer for trace parser
500 */
501int trace_parser_get_init(struct trace_parser *parser, int size)
502{
503 memset(parser, 0, sizeof(*parser));
504
505 parser->buffer = kmalloc(size, GFP_KERNEL);
506 if (!parser->buffer)
507 return 1;
508
509 parser->size = size;
510 return 0;
511}
512
513/*
514 * trace_parser_put - frees the buffer for trace parser
515 */
516void trace_parser_put(struct trace_parser *parser)
517{
518 kfree(parser->buffer);
519}
520
521/*
522 * trace_get_user - reads the user input string separated by space
523 * (matched by isspace(ch))
524 *
525 * For each string found the 'struct trace_parser' is updated,
526 * and the function returns.
527 *
528 * Returns number of bytes read.
529 *
530 * See kernel/trace/trace.h for 'struct trace_parser' details.
531 */
532int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
533 size_t cnt, loff_t *ppos)
534{
535 char ch;
536 size_t read = 0;
537 ssize_t ret;
538
539 if (!*ppos)
540 trace_parser_clear(parser);
541
542 ret = get_user(ch, ubuf++);
543 if (ret)
544 goto out;
545
546 read++;
547 cnt--;
548
549 /*
550 * The parser is not finished with the last write,
551 * continue reading the user input without skipping spaces.
552 */
553 if (!parser->cont) {
554 /* skip white space */
555 while (cnt && isspace(ch)) {
556 ret = get_user(ch, ubuf++);
557 if (ret)
558 goto out;
559 read++;
560 cnt--;
561 }
562
563 /* only spaces were written */
564 if (isspace(ch)) {
565 *ppos += read;
566 ret = read;
567 goto out;
568 }
569
570 parser->idx = 0;
571 }
572
573 /* read the non-space input */
574 while (cnt && !isspace(ch)) {
Li Zefan3c235a32009-09-22 13:51:54 +0800575 if (parser->idx < parser->size - 1)
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200576 parser->buffer[parser->idx++] = ch;
577 else {
578 ret = -EINVAL;
579 goto out;
580 }
581 ret = get_user(ch, ubuf++);
582 if (ret)
583 goto out;
584 read++;
585 cnt--;
586 }
587
588 /* We either got finished input or we have to wait for another call. */
589 if (isspace(ch)) {
590 parser->buffer[parser->idx] = 0;
591 parser->cont = false;
592 } else {
593 parser->cont = true;
594 parser->buffer[parser->idx++] = ch;
595 }
596
597 *ppos += read;
598 ret = read;
599
600out:
601 return ret;
602}
603
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200604ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
605{
606 int len;
607 int ret;
608
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500609 if (!cnt)
610 return 0;
611
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200612 if (s->len <= s->readpos)
613 return -EBUSY;
614
615 len = s->len - s->readpos;
616 if (cnt > len)
617 cnt = len;
618 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500619 if (ret == cnt)
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200620 return -EFAULT;
621
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500622 cnt -= ret;
623
Steven Rostedte74da522009-03-04 20:31:11 -0500624 s->readpos += cnt;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200625 return cnt;
Steven Rostedt214023c2008-05-12 21:20:46 +0200626}
627
Dmitri Vorobievb8b94262009-03-22 19:11:11 +0200628static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200629{
630 int len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200631
632 if (s->len <= s->readpos)
633 return -EBUSY;
634
635 len = s->len - s->readpos;
636 if (cnt > len)
637 cnt = len;
Dan Carpenter5a26c8f2012-04-20 09:31:45 +0300638 memcpy(buf, s->buffer + s->readpos, cnt);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200639
Steven Rostedte74da522009-03-04 20:31:11 -0500640 s->readpos += cnt;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200641 return cnt;
642}
643
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400644/*
645 * ftrace_max_lock is used to protect the swapping of buffers
646 * when taking a max snapshot. The buffers themselves are
647 * protected by per_cpu spinlocks. But the action of the swap
648 * needs its own lock.
649 *
Thomas Gleixner445c8952009-12-02 19:49:50 +0100650 * This is defined as a arch_spinlock_t in order to help
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400651 * with performance when lockdep debugging is enabled.
652 *
653 * It is also used in other places outside the update_max_tr
654 * so it needs to be defined outside of the
655 * CONFIG_TRACER_MAX_TRACE.
656 */
Thomas Gleixner445c8952009-12-02 19:49:50 +0100657static arch_spinlock_t ftrace_max_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100658 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400659
Tim Bird0e950172010-02-25 15:36:43 -0800660unsigned long __read_mostly tracing_thresh;
661
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400662#ifdef CONFIG_TRACER_MAX_TRACE
663unsigned long __read_mostly tracing_max_latency;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400664
665/*
666 * Copy the new maximum trace into the separate maximum-trace
667 * structure. (this way the maximum trace is permanently saved,
668 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
669 */
670static void
671__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
672{
673 struct trace_array_cpu *data = tr->data[cpu];
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300674 struct trace_array_cpu *max_data;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400675
676 max_tr.cpu = cpu;
677 max_tr.time_start = data->preempt_timestamp;
678
Steven Rostedt8248ac02009-09-02 12:27:41 -0400679 max_data = max_tr.data[cpu];
680 max_data->saved_latency = tracing_max_latency;
681 max_data->critical_start = data->critical_start;
682 max_data->critical_end = data->critical_end;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400683
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300684 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
Steven Rostedt8248ac02009-09-02 12:27:41 -0400685 max_data->pid = tsk->pid;
686 max_data->uid = task_uid(tsk);
687 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
688 max_data->policy = tsk->policy;
689 max_data->rt_priority = tsk->rt_priority;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400690
691 /* record this tasks comm */
692 tracing_record_cmdline(tsk);
693}
694
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200695/**
696 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
697 * @tr: tracer
698 * @tsk: the task with the latency
699 * @cpu: The cpu that initiated the trace.
700 *
701 * Flip the buffers between the @tr and the max_tr and record information
702 * about which task was the cause of this latency.
703 */
Ingo Molnare309b412008-05-12 21:20:51 +0200704void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200705update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
706{
Steven Rostedt (Red Hat)2721e722013-03-12 11:32:32 -0400707 struct ring_buffer *buf;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200708
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400709 if (trace_stop_count)
710 return;
711
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200712 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt34600f02013-01-22 13:35:11 -0500713
Hiraku Toyookadebdd572012-12-26 11:53:00 +0900714 if (!current_trace->allocated_snapshot) {
715 /* Only the nop tracer should hit this when disabling */
716 WARN_ON_ONCE(current_trace != &nop_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900717 return;
Hiraku Toyookadebdd572012-12-26 11:53:00 +0900718 }
Steven Rostedt34600f02013-01-22 13:35:11 -0500719
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100720 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400721
Steven Rostedt (Red Hat)2721e722013-03-12 11:32:32 -0400722 buf = tr->buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400723 tr->buffer = max_tr.buffer;
724 max_tr.buffer = buf;
725
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200726 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100727 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200728}
729
730/**
731 * update_max_tr_single - only copy one trace over, and reset the rest
732 * @tr - tracer
733 * @tsk - task with the latency
734 * @cpu - the cpu of the buffer to copy.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200735 *
736 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200737 */
Ingo Molnare309b412008-05-12 21:20:51 +0200738void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200739update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
740{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400741 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200742
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400743 if (trace_stop_count)
744 return;
745
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200746 WARN_ON_ONCE(!irqs_disabled());
Hiraku Toyookadebdd572012-12-26 11:53:00 +0900747 if (WARN_ON_ONCE(!current_trace->allocated_snapshot))
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900748 return;
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900749
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100750 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200751
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400752 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
753
Steven Rostedte8165db2009-09-03 19:13:05 -0400754 if (ret == -EBUSY) {
755 /*
756 * We failed to swap the buffer due to a commit taking
757 * place on this CPU. We fail to record, but we reset
758 * the max trace buffer (no one writes directly to it)
759 * and flag that it failed.
760 */
761 trace_array_printk(&max_tr, _THIS_IP_,
762 "Failed to swap buffers due to commit in progress\n");
763 }
764
Steven Rostedte8165db2009-09-03 19:13:05 -0400765 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200766
767 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100768 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200769}
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400770#endif /* CONFIG_TRACER_MAX_TRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200771
Steven Rostedt0d5c6e12012-11-01 20:54:21 -0400772static void default_wait_pipe(struct trace_iterator *iter)
773{
774 DEFINE_WAIT(wait);
775
776 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
777
778 /*
779 * The events can happen in critical sections where
780 * checking a work queue can cause deadlocks.
781 * After adding a task to the queue, this flag is set
782 * only to notify events to try to wake up the queue
783 * using irq_work.
784 *
785 * We don't clear it even if the buffer is no longer
786 * empty. The flag only causes the next event to run
787 * irq_work to do the work queue wake up. The worse
788 * that can happen if we race with !trace_empty() is that
789 * an event will cause an irq_work to try to wake up
790 * an empty queue.
791 *
792 * There's no reason to protect this flag either, as
793 * the work queue and irq_work logic will do the necessary
794 * synchronization for the wake ups. The only thing
795 * that is necessary is that the wake up happens after
796 * a task has been queued. It's OK for spurious wake ups.
797 */
798 trace_wakeup_needed = true;
799
800 if (trace_empty(iter))
801 schedule();
802
803 finish_wait(&trace_wait, &wait);
804}
805
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200806/**
807 * register_tracer - register a tracer with the ftrace system.
808 * @type - the plugin for the tracer
809 *
810 * Register a new plugin tracer.
811 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200812int register_tracer(struct tracer *type)
813{
814 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200815 int ret = 0;
816
817 if (!type->name) {
818 pr_info("Tracer must have a name\n");
819 return -1;
820 }
821
Dan Carpenter24a461d2010-07-10 12:06:44 +0200822 if (strlen(type->name) >= MAX_TRACER_SIZE) {
Li Zefanee6c2c12009-09-18 14:06:47 +0800823 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
824 return -1;
825 }
826
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200827 mutex_lock(&trace_types_lock);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100828
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +0100829 tracing_selftest_running = true;
830
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200831 for (t = trace_types; t; t = t->next) {
832 if (strcmp(type->name, t->name) == 0) {
833 /* already found */
Li Zefanee6c2c12009-09-18 14:06:47 +0800834 pr_info("Tracer %s already registered\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200835 type->name);
836 ret = -1;
837 goto out;
838 }
839 }
840
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +0100841 if (!type->set_flag)
842 type->set_flag = &dummy_set_flag;
843 if (!type->flags)
844 type->flags = &dummy_tracer_flags;
845 else
846 if (!type->flags->opts)
847 type->flags->opts = dummy_tracer_opt;
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +0100848 if (!type->wait_pipe)
849 type->wait_pipe = default_wait_pipe;
850
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +0100851
Steven Rostedt60a11772008-05-12 21:20:44 +0200852#ifdef CONFIG_FTRACE_STARTUP_TEST
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500853 if (type->selftest && !tracing_selftest_disabled) {
Steven Rostedt60a11772008-05-12 21:20:44 +0200854 struct tracer *saved_tracer = current_trace;
Steven Rostedt60a11772008-05-12 21:20:44 +0200855 struct trace_array *tr = &global_trace;
Frederic Weisbeckerff325042008-12-04 23:47:35 +0100856
Steven Rostedt60a11772008-05-12 21:20:44 +0200857 /*
858 * Run a selftest on this tracer.
859 * Here we reset the trace buffer, and set the current
860 * tracer to be this tracer. The tracer can then run some
861 * internal tracing to verify that everything is in order.
862 * If we fail, we do not register this tracer.
863 */
Steven Rostedt76f0d072009-09-04 12:12:39 -0400864 tracing_reset_online_cpus(tr);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100865
Steven Rostedt60a11772008-05-12 21:20:44 +0200866 current_trace = type;
Steven Rostedt4a0b1662011-03-09 20:09:26 -0500867
Hiraku Toyookadebdd572012-12-26 11:53:00 +0900868 if (type->use_max_tr) {
869 /* If we expanded the buffers, make sure the max is expanded too */
870 if (ring_buffer_expanded)
871 ring_buffer_resize(max_tr.buffer, trace_buf_size,
872 RING_BUFFER_ALL_CPUS);
873 type->allocated_snapshot = true;
874 }
Steven Rostedt4a0b1662011-03-09 20:09:26 -0500875
Steven Rostedt60a11772008-05-12 21:20:44 +0200876 /* the test is responsible for initializing and enabling */
877 pr_info("Testing tracer %s: ", type->name);
878 ret = type->selftest(type, tr);
879 /* the test is responsible for resetting too */
880 current_trace = saved_tracer;
Steven Rostedt60a11772008-05-12 21:20:44 +0200881 if (ret) {
882 printk(KERN_CONT "FAILED!\n");
Steven Rostedt0be61eb2012-06-18 09:28:16 -0400883 /* Add the warning after printing 'FAILED' */
884 WARN_ON(1);
Steven Rostedt60a11772008-05-12 21:20:44 +0200885 goto out;
886 }
Steven Rostedt1d4db002008-05-12 21:20:45 +0200887 /* Only reset on passing, to avoid touching corrupted buffers */
Steven Rostedt76f0d072009-09-04 12:12:39 -0400888 tracing_reset_online_cpus(tr);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100889
Hiraku Toyookadebdd572012-12-26 11:53:00 +0900890 if (type->use_max_tr) {
891 type->allocated_snapshot = false;
892
893 /* Shrink the max buffer again */
894 if (ring_buffer_expanded)
895 ring_buffer_resize(max_tr.buffer, 1,
896 RING_BUFFER_ALL_CPUS);
897 }
Steven Rostedt4a0b1662011-03-09 20:09:26 -0500898
Steven Rostedt60a11772008-05-12 21:20:44 +0200899 printk(KERN_CONT "PASSED\n");
900 }
901#endif
902
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200903 type->next = trace_types;
904 trace_types = type;
Steven Rostedt60a11772008-05-12 21:20:44 +0200905
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200906 out:
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +0100907 tracing_selftest_running = false;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200908 mutex_unlock(&trace_types_lock);
909
Steven Rostedtdac74942009-02-05 01:13:38 -0500910 if (ret || !default_bootup_tracer)
911 goto out_unlock;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500912
Li Zefanee6c2c12009-09-18 14:06:47 +0800913 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
Steven Rostedtdac74942009-02-05 01:13:38 -0500914 goto out_unlock;
915
916 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
917 /* Do we want this tracer to start on bootup? */
918 tracing_set_tracer(type->name);
919 default_bootup_tracer = NULL;
920 /* disable other selftests, since this will break it. */
921 tracing_selftest_disabled = 1;
922#ifdef CONFIG_FTRACE_STARTUP_TEST
923 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
924 type->name);
925#endif
926
927 out_unlock:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200928 return ret;
929}
930
Steven Rostedtf6339032009-09-04 12:35:16 -0400931void tracing_reset(struct trace_array *tr, int cpu)
932{
933 struct ring_buffer *buffer = tr->buffer;
934
Hiraku Toyookaa5416412012-12-19 16:02:34 +0900935 if (!buffer)
936 return;
937
Steven Rostedtf6339032009-09-04 12:35:16 -0400938 ring_buffer_record_disable(buffer);
939
940 /* Make sure all commits have finished */
941 synchronize_sched();
Steven Rostedt68179682012-05-08 20:57:53 -0400942 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedtf6339032009-09-04 12:35:16 -0400943
944 ring_buffer_record_enable(buffer);
945}
946
Pekka J Enberg213cc062008-12-19 12:08:39 +0200947void tracing_reset_online_cpus(struct trace_array *tr)
948{
Steven Rostedt621968c2009-09-04 12:02:35 -0400949 struct ring_buffer *buffer = tr->buffer;
Pekka J Enberg213cc062008-12-19 12:08:39 +0200950 int cpu;
951
Hiraku Toyookaa5416412012-12-19 16:02:34 +0900952 if (!buffer)
953 return;
954
Steven Rostedt621968c2009-09-04 12:02:35 -0400955 ring_buffer_record_disable(buffer);
956
957 /* Make sure all commits have finished */
958 synchronize_sched();
959
Pekka J Enberg213cc062008-12-19 12:08:39 +0200960 tr->time_start = ftrace_now(tr->cpu);
961
962 for_each_online_cpu(cpu)
Steven Rostedt68179682012-05-08 20:57:53 -0400963 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedt621968c2009-09-04 12:02:35 -0400964
965 ring_buffer_record_enable(buffer);
Pekka J Enberg213cc062008-12-19 12:08:39 +0200966}
967
Steven Rostedt9456f0f2009-05-06 21:54:09 -0400968void tracing_reset_current(int cpu)
969{
970 tracing_reset(&global_trace, cpu);
971}
972
973void tracing_reset_current_online_cpus(void)
974{
975 tracing_reset_online_cpus(&global_trace);
976}
977
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200978#define SAVED_CMDLINES 128
Thomas Gleixner2c7eea42009-03-18 09:03:19 +0100979#define NO_CMDLINE_MAP UINT_MAX
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200980static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
981static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
982static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
983static int cmdline_idx;
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100984static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt25b0b442008-05-12 21:21:00 +0200985
Steven Rostedt25b0b442008-05-12 21:21:00 +0200986/* temporary disable recording */
Hannes Eder4fd27352009-02-10 19:44:12 +0100987static atomic_t trace_record_cmdline_disabled __read_mostly;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200988
989static void trace_init_cmdlines(void)
990{
Thomas Gleixner2c7eea42009-03-18 09:03:19 +0100991 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
992 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200993 cmdline_idx = 0;
994}
995
Carsten Emdeb5130b12009-09-13 01:43:07 +0200996int is_tracing_stopped(void)
997{
998 return trace_stop_count;
999}
1000
Steven Rostedt0f048702008-11-05 16:05:44 -05001001/**
Steven Rostedt69bb54e2008-11-21 12:59:38 -05001002 * ftrace_off_permanent - disable all ftrace code permanently
1003 *
1004 * This should only be called when a serious anomally has
1005 * been detected. This will turn off the function tracing,
1006 * ring buffers, and other tracing utilites. It takes no
1007 * locks and can be called from any context.
1008 */
1009void ftrace_off_permanent(void)
1010{
1011 tracing_disabled = 1;
1012 ftrace_stop();
1013 tracing_off_permanent();
1014}
1015
1016/**
Steven Rostedt0f048702008-11-05 16:05:44 -05001017 * tracing_start - quick start of the tracer
1018 *
1019 * If tracing is enabled but was stopped by tracing_stop,
1020 * this will start the tracer back up.
1021 */
1022void tracing_start(void)
1023{
1024 struct ring_buffer *buffer;
1025 unsigned long flags;
1026
1027 if (tracing_disabled)
1028 return;
1029
Thomas Gleixner5389f6f2009-07-25 17:13:33 +02001030 raw_spin_lock_irqsave(&tracing_start_lock, flags);
Steven Rostedtb06a8302009-01-22 14:26:15 -05001031 if (--trace_stop_count) {
1032 if (trace_stop_count < 0) {
1033 /* Someone screwed up their debugging */
1034 WARN_ON_ONCE(1);
1035 trace_stop_count = 0;
1036 }
Steven Rostedt0f048702008-11-05 16:05:44 -05001037 goto out;
1038 }
1039
Steven Rostedta2f80712010-03-12 19:56:00 -05001040 /* Prevent the buffers from switching */
1041 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt0f048702008-11-05 16:05:44 -05001042
1043 buffer = global_trace.buffer;
1044 if (buffer)
1045 ring_buffer_record_enable(buffer);
1046
1047 buffer = max_tr.buffer;
1048 if (buffer)
1049 ring_buffer_record_enable(buffer);
1050
Steven Rostedta2f80712010-03-12 19:56:00 -05001051 arch_spin_unlock(&ftrace_max_lock);
1052
Steven Rostedt0f048702008-11-05 16:05:44 -05001053 ftrace_start();
1054 out:
Thomas Gleixner5389f6f2009-07-25 17:13:33 +02001055 raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001056}
1057
1058/**
1059 * tracing_stop - quick stop of the tracer
1060 *
1061 * Light weight way to stop tracing. Use in conjunction with
1062 * tracing_start.
1063 */
1064void tracing_stop(void)
1065{
1066 struct ring_buffer *buffer;
1067 unsigned long flags;
1068
1069 ftrace_stop();
Thomas Gleixner5389f6f2009-07-25 17:13:33 +02001070 raw_spin_lock_irqsave(&tracing_start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001071 if (trace_stop_count++)
1072 goto out;
1073
Steven Rostedta2f80712010-03-12 19:56:00 -05001074 /* Prevent the buffers from switching */
1075 arch_spin_lock(&ftrace_max_lock);
1076
Steven Rostedt0f048702008-11-05 16:05:44 -05001077 buffer = global_trace.buffer;
1078 if (buffer)
1079 ring_buffer_record_disable(buffer);
1080
1081 buffer = max_tr.buffer;
1082 if (buffer)
1083 ring_buffer_record_disable(buffer);
1084
Steven Rostedta2f80712010-03-12 19:56:00 -05001085 arch_spin_unlock(&ftrace_max_lock);
1086
Steven Rostedt0f048702008-11-05 16:05:44 -05001087 out:
Thomas Gleixner5389f6f2009-07-25 17:13:33 +02001088 raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001089}
1090
Ingo Molnare309b412008-05-12 21:20:51 +02001091void trace_stop_cmdline_recording(void);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001092
Ingo Molnare309b412008-05-12 21:20:51 +02001093static void trace_save_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001094{
Carsten Emdea635cf02009-03-18 09:00:41 +01001095 unsigned pid, idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001096
1097 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1098 return;
1099
1100 /*
1101 * It's not the end of the world if we don't get
1102 * the lock, but we also don't want to spin
1103 * nor do we want to disable interrupts,
1104 * so if we miss here, then better luck next time.
1105 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001106 if (!arch_spin_trylock(&trace_cmdline_lock))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001107 return;
1108
1109 idx = map_pid_to_cmdline[tsk->pid];
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001110 if (idx == NO_CMDLINE_MAP) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001111 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1112
Carsten Emdea635cf02009-03-18 09:00:41 +01001113 /*
1114 * Check whether the cmdline buffer at idx has a pid
1115 * mapped. We are going to overwrite that entry so we
1116 * need to clear the map_pid_to_cmdline. Otherwise we
1117 * would read the new comm for the old pid.
1118 */
1119 pid = map_cmdline_to_pid[idx];
1120 if (pid != NO_CMDLINE_MAP)
1121 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001122
Carsten Emdea635cf02009-03-18 09:00:41 +01001123 map_cmdline_to_pid[idx] = tsk->pid;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001124 map_pid_to_cmdline[tsk->pid] = idx;
1125
1126 cmdline_idx = idx;
1127 }
1128
1129 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1130
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001131 arch_spin_unlock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001132}
1133
Steven Rostedt4ca53082009-03-16 19:20:15 -04001134void trace_find_cmdline(int pid, char comm[])
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001135{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001136 unsigned map;
1137
Steven Rostedt4ca53082009-03-16 19:20:15 -04001138 if (!pid) {
1139 strcpy(comm, "<idle>");
1140 return;
1141 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001142
Steven Rostedt74bf4072010-01-25 15:11:53 -05001143 if (WARN_ON_ONCE(pid < 0)) {
1144 strcpy(comm, "<XXX>");
1145 return;
1146 }
1147
Steven Rostedt4ca53082009-03-16 19:20:15 -04001148 if (pid > PID_MAX_DEFAULT) {
1149 strcpy(comm, "<...>");
1150 return;
1151 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001152
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001153 preempt_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001154 arch_spin_lock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001155 map = map_pid_to_cmdline[pid];
Thomas Gleixner50d88752009-03-18 08:58:44 +01001156 if (map != NO_CMDLINE_MAP)
1157 strcpy(comm, saved_cmdlines[map]);
1158 else
1159 strcpy(comm, "<...>");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001160
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001161 arch_spin_unlock(&trace_cmdline_lock);
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001162 preempt_enable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001163}
1164
Ingo Molnare309b412008-05-12 21:20:51 +02001165void tracing_record_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001166{
Steven Rostedt0fb96562012-05-11 14:25:30 -04001167 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001168 return;
1169
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001170 if (!__this_cpu_read(trace_cmdline_save))
1171 return;
1172
1173 __this_cpu_write(trace_cmdline_save, false);
1174
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001175 trace_save_cmdline(tsk);
1176}
1177
Pekka Paalanen45dcd8b2008-09-16 21:56:41 +03001178void
Steven Rostedt38697052008-10-01 13:14:09 -04001179tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1180 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001181{
1182 struct task_struct *tsk = current;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001183
Steven Rostedt777e2082008-09-29 23:02:42 -04001184 entry->preempt_count = pc & 0xff;
1185 entry->pid = (tsk) ? tsk->pid : 0;
1186 entry->flags =
Steven Rostedt92444892008-10-24 09:42:59 -04001187#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
Steven Rostedt2e2ca152008-08-01 12:26:40 -04001188 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
Steven Rostedt92444892008-10-24 09:42:59 -04001189#else
1190 TRACE_FLAG_IRQS_NOSUPPORT |
1191#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001192 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1193 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1194 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1195}
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +02001196EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001197
Steven Rostedte77405a2009-09-02 14:17:06 -04001198struct ring_buffer_event *
1199trace_buffer_lock_reserve(struct ring_buffer *buffer,
1200 int type,
1201 unsigned long len,
1202 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001203{
1204 struct ring_buffer_event *event;
1205
Steven Rostedte77405a2009-09-02 14:17:06 -04001206 event = ring_buffer_lock_reserve(buffer, len);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001207 if (event != NULL) {
1208 struct trace_entry *ent = ring_buffer_event_data(event);
1209
1210 tracing_generic_entry_update(ent, flags, pc);
1211 ent->type = type;
1212 }
1213
1214 return event;
1215}
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001216
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001217void
1218__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1219{
1220 __this_cpu_write(trace_cmdline_save, true);
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001221 if (trace_wakeup_needed) {
1222 trace_wakeup_needed = false;
1223 /* irq_work_queue() supplies it's own memory barriers */
1224 irq_work_queue(&trace_work_wakeup);
1225 }
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001226 ring_buffer_unlock_commit(buffer, event);
1227}
1228
Steven Rostedte77405a2009-09-02 14:17:06 -04001229static inline void
1230__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1231 struct ring_buffer_event *event,
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001232 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001233{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001234 __buffer_unlock_commit(buffer, event);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001235
Steven Rostedte77405a2009-09-02 14:17:06 -04001236 ftrace_trace_stack(buffer, flags, 6, pc);
1237 ftrace_trace_userstack(buffer, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001238}
1239
Steven Rostedte77405a2009-09-02 14:17:06 -04001240void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1241 struct ring_buffer_event *event,
1242 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001243{
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001244 __trace_buffer_unlock_commit(buffer, event, flags, pc);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001245}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001246EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001247
Steven Rostedtef5580d2009-02-27 19:38:04 -05001248struct ring_buffer_event *
Steven Rostedte77405a2009-09-02 14:17:06 -04001249trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1250 int type, unsigned long len,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001251 unsigned long flags, int pc)
1252{
Steven Rostedte77405a2009-09-02 14:17:06 -04001253 *current_rb = global_trace.buffer;
1254 return trace_buffer_lock_reserve(*current_rb,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001255 type, len, flags, pc);
1256}
Steven Rostedt94487d62009-05-05 19:22:53 -04001257EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001258
Steven Rostedte77405a2009-09-02 14:17:06 -04001259void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1260 struct ring_buffer_event *event,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001261 unsigned long flags, int pc)
1262{
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001263 __trace_buffer_unlock_commit(buffer, event, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001264}
Steven Rostedt94487d62009-05-05 19:22:53 -04001265EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001266
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001267void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1268 struct ring_buffer_event *event,
1269 unsigned long flags, int pc,
1270 struct pt_regs *regs)
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001271{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001272 __buffer_unlock_commit(buffer, event);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001273
1274 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1275 ftrace_trace_userstack(buffer, flags, pc);
1276}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001277EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001278
Steven Rostedte77405a2009-09-02 14:17:06 -04001279void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1280 struct ring_buffer_event *event)
Steven Rostedt77d9f462009-04-02 01:16:59 -04001281{
Steven Rostedte77405a2009-09-02 14:17:06 -04001282 ring_buffer_discard_commit(buffer, event);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001283}
Steven Rostedt12acd472009-04-17 16:01:56 -04001284EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001285
Ingo Molnare309b412008-05-12 21:20:51 +02001286void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001287trace_function(struct trace_array *tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001288 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1289 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001290{
Tom Zanussie1112b42009-03-31 00:48:49 -05001291 struct ftrace_event_call *call = &event_function;
Steven Rostedte77405a2009-09-02 14:17:06 -04001292 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001293 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001294 struct ftrace_entry *entry;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001295
Steven Rostedtd7690412008-10-01 00:29:53 -04001296 /* If we are reading the ring buffer, don't trace */
Rusty Russelldd17c8f2009-10-29 22:34:15 +09001297 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
Steven Rostedtd7690412008-10-01 00:29:53 -04001298 return;
1299
Steven Rostedte77405a2009-09-02 14:17:06 -04001300 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001301 flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001302 if (!event)
1303 return;
1304 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001305 entry->ip = ip;
1306 entry->parent_ip = parent_ip;
Tom Zanussie1112b42009-03-31 00:48:49 -05001307
Steven Rostedte77405a2009-09-02 14:17:06 -04001308 if (!filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001309 __buffer_unlock_commit(buffer, event);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001310}
1311
Ingo Molnare309b412008-05-12 21:20:51 +02001312void
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001313ftrace(struct trace_array *tr, struct trace_array_cpu *data,
Steven Rostedt38697052008-10-01 13:14:09 -04001314 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1315 int pc)
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001316{
1317 if (likely(!atomic_read(&data->disabled)))
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001318 trace_function(tr, ip, parent_ip, flags, pc);
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001319}
1320
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001321#ifdef CONFIG_STACKTRACE
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001322
1323#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1324struct ftrace_stack {
1325 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1326};
1327
1328static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1329static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1330
Steven Rostedte77405a2009-09-02 14:17:06 -04001331static void __ftrace_trace_stack(struct ring_buffer *buffer,
Steven Rostedt53614992009-01-15 19:12:40 -05001332 unsigned long flags,
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001333 int skip, int pc, struct pt_regs *regs)
Ingo Molnar86387f72008-05-12 21:20:51 +02001334{
Tom Zanussie1112b42009-03-31 00:48:49 -05001335 struct ftrace_event_call *call = &event_kernel_stack;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001336 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001337 struct stack_entry *entry;
Ingo Molnar86387f72008-05-12 21:20:51 +02001338 struct stack_trace trace;
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001339 int use_stack;
1340 int size = FTRACE_STACK_ENTRIES;
Ingo Molnar86387f72008-05-12 21:20:51 +02001341
1342 trace.nr_entries = 0;
Ingo Molnar86387f72008-05-12 21:20:51 +02001343 trace.skip = skip;
Ingo Molnar86387f72008-05-12 21:20:51 +02001344
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001345 /*
1346 * Since events can happen in NMIs there's no safe way to
1347 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1348 * or NMI comes in, it will just have to use the default
1349 * FTRACE_STACK_SIZE.
1350 */
1351 preempt_disable_notrace();
1352
Shan Wei82146522012-11-19 13:21:01 +08001353 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001354 /*
1355 * We don't need any atomic variables, just a barrier.
1356 * If an interrupt comes in, we don't care, because it would
1357 * have exited and put the counter back to what we want.
1358 * We just need a barrier to keep gcc from moving things
1359 * around.
1360 */
1361 barrier();
1362 if (use_stack == 1) {
1363 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1364 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1365
1366 if (regs)
1367 save_stack_trace_regs(regs, &trace);
1368 else
1369 save_stack_trace(&trace);
1370
1371 if (trace.nr_entries > size)
1372 size = trace.nr_entries;
1373 } else
1374 /* From now on, use_stack is a boolean */
1375 use_stack = 0;
1376
1377 size *= sizeof(unsigned long);
1378
1379 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1380 sizeof(*entry) + size, flags, pc);
1381 if (!event)
1382 goto out;
1383 entry = ring_buffer_event_data(event);
1384
1385 memset(&entry->caller, 0, size);
1386
1387 if (use_stack)
1388 memcpy(&entry->caller, trace.entries,
1389 trace.nr_entries * sizeof(unsigned long));
1390 else {
1391 trace.max_entries = FTRACE_STACK_ENTRIES;
1392 trace.entries = entry->caller;
1393 if (regs)
1394 save_stack_trace_regs(regs, &trace);
1395 else
1396 save_stack_trace(&trace);
1397 }
1398
1399 entry->size = trace.nr_entries;
1400
Steven Rostedte77405a2009-09-02 14:17:06 -04001401 if (!filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001402 __buffer_unlock_commit(buffer, event);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001403
1404 out:
1405 /* Again, don't let gcc optimize things here */
1406 barrier();
Shan Wei82146522012-11-19 13:21:01 +08001407 __this_cpu_dec(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001408 preempt_enable_notrace();
1409
Ingo Molnarf0a920d2008-05-12 21:20:47 +02001410}
1411
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001412void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1413 int skip, int pc, struct pt_regs *regs)
1414{
1415 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1416 return;
1417
1418 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1419}
1420
Steven Rostedte77405a2009-09-02 14:17:06 -04001421void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1422 int skip, int pc)
Steven Rostedt53614992009-01-15 19:12:40 -05001423{
1424 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1425 return;
1426
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001427 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
Steven Rostedt53614992009-01-15 19:12:40 -05001428}
1429
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001430void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1431 int pc)
Steven Rostedt38697052008-10-01 13:14:09 -04001432{
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001433 __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
Steven Rostedt38697052008-10-01 13:14:09 -04001434}
1435
Steven Rostedt03889382009-12-11 09:48:22 -05001436/**
1437 * trace_dump_stack - record a stack back trace in the trace buffer
1438 */
1439void trace_dump_stack(void)
1440{
1441 unsigned long flags;
1442
1443 if (tracing_disabled || tracing_selftest_running)
Steven Rostedte36c5452009-12-14 15:58:33 -05001444 return;
Steven Rostedt03889382009-12-11 09:48:22 -05001445
1446 local_save_flags(flags);
1447
1448 /* skipping 3 traces, seems to get us at the caller of this function */
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001449 __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
Steven Rostedt03889382009-12-11 09:48:22 -05001450}
1451
Steven Rostedt91e86e52010-11-10 12:56:12 +01001452static DEFINE_PER_CPU(int, user_stack_count);
1453
Steven Rostedte77405a2009-09-02 14:17:06 -04001454void
1455ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
Török Edwin02b67512008-11-22 13:28:47 +02001456{
Tom Zanussie1112b42009-03-31 00:48:49 -05001457 struct ftrace_event_call *call = &event_user_stack;
Török Edwin8d7c6a92008-11-23 12:39:06 +02001458 struct ring_buffer_event *event;
Török Edwin02b67512008-11-22 13:28:47 +02001459 struct userstack_entry *entry;
1460 struct stack_trace trace;
Török Edwin02b67512008-11-22 13:28:47 +02001461
1462 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1463 return;
1464
Steven Rostedtb6345872010-03-12 20:03:30 -05001465 /*
1466 * NMIs can not handle page faults, even with fix ups.
1467 * The save user stack can (and often does) fault.
1468 */
1469 if (unlikely(in_nmi()))
1470 return;
1471
Steven Rostedt91e86e52010-11-10 12:56:12 +01001472 /*
1473 * prevent recursion, since the user stack tracing may
1474 * trigger other kernel events.
1475 */
1476 preempt_disable();
1477 if (__this_cpu_read(user_stack_count))
1478 goto out;
1479
1480 __this_cpu_inc(user_stack_count);
1481
Steven Rostedte77405a2009-09-02 14:17:06 -04001482 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001483 sizeof(*entry), flags, pc);
Török Edwin02b67512008-11-22 13:28:47 +02001484 if (!event)
Li Zefan1dbd1952010-12-09 15:47:56 +08001485 goto out_drop_count;
Török Edwin02b67512008-11-22 13:28:47 +02001486 entry = ring_buffer_event_data(event);
Török Edwin02b67512008-11-22 13:28:47 +02001487
Steven Rostedt48659d32009-09-11 11:36:23 -04001488 entry->tgid = current->tgid;
Török Edwin02b67512008-11-22 13:28:47 +02001489 memset(&entry->caller, 0, sizeof(entry->caller));
1490
1491 trace.nr_entries = 0;
1492 trace.max_entries = FTRACE_STACK_ENTRIES;
1493 trace.skip = 0;
1494 trace.entries = entry->caller;
1495
1496 save_stack_trace_user(&trace);
Steven Rostedte77405a2009-09-02 14:17:06 -04001497 if (!filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001498 __buffer_unlock_commit(buffer, event);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001499
Li Zefan1dbd1952010-12-09 15:47:56 +08001500 out_drop_count:
Steven Rostedt91e86e52010-11-10 12:56:12 +01001501 __this_cpu_dec(user_stack_count);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001502 out:
1503 preempt_enable();
Török Edwin02b67512008-11-22 13:28:47 +02001504}
1505
Hannes Eder4fd27352009-02-10 19:44:12 +01001506#ifdef UNUSED
1507static void __trace_userstack(struct trace_array *tr, unsigned long flags)
Török Edwin02b67512008-11-22 13:28:47 +02001508{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001509 ftrace_trace_userstack(tr, flags, preempt_count());
Török Edwin02b67512008-11-22 13:28:47 +02001510}
Hannes Eder4fd27352009-02-10 19:44:12 +01001511#endif /* UNUSED */
Török Edwin02b67512008-11-22 13:28:47 +02001512
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001513#endif /* CONFIG_STACKTRACE */
1514
Steven Rostedt07d777f2011-09-22 14:01:55 -04001515/* created for use with alloc_percpu */
1516struct trace_buffer_struct {
1517 char buffer[TRACE_BUF_SIZE];
1518};
1519
1520static struct trace_buffer_struct *trace_percpu_buffer;
1521static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1522static struct trace_buffer_struct *trace_percpu_irq_buffer;
1523static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1524
1525/*
1526 * The buffer used is dependent on the context. There is a per cpu
1527 * buffer for normal context, softirq contex, hard irq context and
1528 * for NMI context. Thise allows for lockless recording.
1529 *
1530 * Note, if the buffers failed to be allocated, then this returns NULL
1531 */
1532static char *get_trace_buf(void)
1533{
1534 struct trace_buffer_struct *percpu_buffer;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001535
1536 /*
1537 * If we have allocated per cpu buffers, then we do not
1538 * need to do any locking.
1539 */
1540 if (in_nmi())
1541 percpu_buffer = trace_percpu_nmi_buffer;
1542 else if (in_irq())
1543 percpu_buffer = trace_percpu_irq_buffer;
1544 else if (in_softirq())
1545 percpu_buffer = trace_percpu_sirq_buffer;
1546 else
1547 percpu_buffer = trace_percpu_buffer;
1548
1549 if (!percpu_buffer)
1550 return NULL;
1551
Shan Weid8a03492012-11-13 09:53:04 +08001552 return this_cpu_ptr(&percpu_buffer->buffer[0]);
Steven Rostedt07d777f2011-09-22 14:01:55 -04001553}
1554
1555static int alloc_percpu_trace_buffer(void)
1556{
1557 struct trace_buffer_struct *buffers;
1558 struct trace_buffer_struct *sirq_buffers;
1559 struct trace_buffer_struct *irq_buffers;
1560 struct trace_buffer_struct *nmi_buffers;
1561
1562 buffers = alloc_percpu(struct trace_buffer_struct);
1563 if (!buffers)
1564 goto err_warn;
1565
1566 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1567 if (!sirq_buffers)
1568 goto err_sirq;
1569
1570 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1571 if (!irq_buffers)
1572 goto err_irq;
1573
1574 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1575 if (!nmi_buffers)
1576 goto err_nmi;
1577
1578 trace_percpu_buffer = buffers;
1579 trace_percpu_sirq_buffer = sirq_buffers;
1580 trace_percpu_irq_buffer = irq_buffers;
1581 trace_percpu_nmi_buffer = nmi_buffers;
1582
1583 return 0;
1584
1585 err_nmi:
1586 free_percpu(irq_buffers);
1587 err_irq:
1588 free_percpu(sirq_buffers);
1589 err_sirq:
1590 free_percpu(buffers);
1591 err_warn:
1592 WARN(1, "Could not allocate percpu trace_printk buffer");
1593 return -ENOMEM;
1594}
1595
Steven Rostedt81698832012-10-11 10:15:05 -04001596static int buffers_allocated;
1597
Steven Rostedt07d777f2011-09-22 14:01:55 -04001598void trace_printk_init_buffers(void)
1599{
Steven Rostedt07d777f2011-09-22 14:01:55 -04001600 if (buffers_allocated)
1601 return;
1602
1603 if (alloc_percpu_trace_buffer())
1604 return;
1605
1606 pr_info("ftrace: Allocated trace_printk buffers\n");
1607
Steven Rostedtb382ede62012-10-10 21:44:34 -04001608 /* Expand the buffers to set size */
1609 tracing_update_buffers();
1610
Steven Rostedt07d777f2011-09-22 14:01:55 -04001611 buffers_allocated = 1;
Steven Rostedt81698832012-10-11 10:15:05 -04001612
1613 /*
1614 * trace_printk_init_buffers() can be called by modules.
1615 * If that happens, then we need to start cmdline recording
1616 * directly here. If the global_trace.buffer is already
1617 * allocated here, then this was called by module code.
1618 */
1619 if (global_trace.buffer)
1620 tracing_start_cmdline_record();
1621}
1622
1623void trace_printk_start_comm(void)
1624{
1625 /* Start tracing comms if trace printk is set */
1626 if (!buffers_allocated)
1627 return;
1628 tracing_start_cmdline_record();
1629}
1630
1631static void trace_printk_start_stop_comm(int enabled)
1632{
1633 if (!buffers_allocated)
1634 return;
1635
1636 if (enabled)
1637 tracing_start_cmdline_record();
1638 else
1639 tracing_stop_cmdline_record();
Steven Rostedt07d777f2011-09-22 14:01:55 -04001640}
1641
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001642/**
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001643 * trace_vbprintk - write binary msg to tracing buffer
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001644 *
1645 */
Steven Rostedt40ce74f2009-03-19 14:03:53 -04001646int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001647{
Tom Zanussie1112b42009-03-31 00:48:49 -05001648 struct ftrace_event_call *call = &event_bprint;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001649 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001650 struct ring_buffer *buffer;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001651 struct trace_array *tr = &global_trace;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001652 struct bprint_entry *entry;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001653 unsigned long flags;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001654 char *tbuffer;
1655 int len = 0, size, pc;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001656
1657 if (unlikely(tracing_selftest_running || tracing_disabled))
1658 return 0;
1659
1660 /* Don't pollute graph traces with trace_vprintk internals */
1661 pause_graph_tracing();
1662
1663 pc = preempt_count();
Steven Rostedt5168ae52010-06-03 09:36:50 -04001664 preempt_disable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001665
Steven Rostedt07d777f2011-09-22 14:01:55 -04001666 tbuffer = get_trace_buf();
1667 if (!tbuffer) {
1668 len = 0;
1669 goto out;
1670 }
1671
1672 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1673
1674 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001675 goto out;
1676
Steven Rostedt07d777f2011-09-22 14:01:55 -04001677 local_save_flags(flags);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001678 size = sizeof(*entry) + sizeof(u32) * len;
Steven Rostedte77405a2009-09-02 14:17:06 -04001679 buffer = tr->buffer;
1680 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1681 flags, pc);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001682 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04001683 goto out;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001684 entry = ring_buffer_event_data(event);
1685 entry->ip = ip;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001686 entry->fmt = fmt;
1687
Steven Rostedt07d777f2011-09-22 14:01:55 -04001688 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
Steven Rostedtd9313692010-01-06 17:27:11 -05001689 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001690 __buffer_unlock_commit(buffer, event);
Steven Rostedtd9313692010-01-06 17:27:11 -05001691 ftrace_trace_stack(buffer, flags, 6, pc);
1692 }
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001693
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001694out:
Steven Rostedt5168ae52010-06-03 09:36:50 -04001695 preempt_enable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001696 unpause_graph_tracing();
1697
1698 return len;
1699}
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001700EXPORT_SYMBOL_GPL(trace_vbprintk);
1701
Steven Rostedt659372d2009-09-03 19:11:07 -04001702int trace_array_printk(struct trace_array *tr,
1703 unsigned long ip, const char *fmt, ...)
1704{
1705 int ret;
1706 va_list ap;
1707
1708 if (!(trace_flags & TRACE_ITER_PRINTK))
1709 return 0;
1710
1711 va_start(ap, fmt);
1712 ret = trace_array_vprintk(tr, ip, fmt, ap);
1713 va_end(ap);
1714 return ret;
1715}
1716
1717int trace_array_vprintk(struct trace_array *tr,
1718 unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001719{
Tom Zanussie1112b42009-03-31 00:48:49 -05001720 struct ftrace_event_call *call = &event_print;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001721 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001722 struct ring_buffer *buffer;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001723 int len = 0, size, pc;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001724 struct print_entry *entry;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001725 unsigned long flags;
1726 char *tbuffer;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001727
1728 if (tracing_disabled || tracing_selftest_running)
1729 return 0;
1730
Steven Rostedt07d777f2011-09-22 14:01:55 -04001731 /* Don't pollute graph traces with trace_vprintk internals */
1732 pause_graph_tracing();
1733
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001734 pc = preempt_count();
1735 preempt_disable_notrace();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001736
Steven Rostedt07d777f2011-09-22 14:01:55 -04001737
1738 tbuffer = get_trace_buf();
1739 if (!tbuffer) {
1740 len = 0;
1741 goto out;
1742 }
1743
1744 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1745 if (len > TRACE_BUF_SIZE)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001746 goto out;
1747
Steven Rostedt07d777f2011-09-22 14:01:55 -04001748 local_save_flags(flags);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001749 size = sizeof(*entry) + len + 1;
Steven Rostedte77405a2009-09-02 14:17:06 -04001750 buffer = tr->buffer;
1751 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
Steven Rostedt07d777f2011-09-22 14:01:55 -04001752 flags, pc);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001753 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04001754 goto out;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001755 entry = ring_buffer_event_data(event);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001756 entry->ip = ip;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001757
Steven Rostedt07d777f2011-09-22 14:01:55 -04001758 memcpy(&entry->buf, tbuffer, len);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001759 entry->buf[len] = '\0';
Steven Rostedtd9313692010-01-06 17:27:11 -05001760 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001761 __buffer_unlock_commit(buffer, event);
Steven Rostedt07d777f2011-09-22 14:01:55 -04001762 ftrace_trace_stack(buffer, flags, 6, pc);
Steven Rostedtd9313692010-01-06 17:27:11 -05001763 }
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001764 out:
1765 preempt_enable_notrace();
Steven Rostedt07d777f2011-09-22 14:01:55 -04001766 unpause_graph_tracing();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001767
1768 return len;
1769}
Steven Rostedt659372d2009-09-03 19:11:07 -04001770
1771int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1772{
Steven Rostedta813a152009-10-09 01:41:35 -04001773 return trace_array_vprintk(&global_trace, ip, fmt, args);
Steven Rostedt659372d2009-09-03 19:11:07 -04001774}
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001775EXPORT_SYMBOL_GPL(trace_vprintk);
1776
Robert Richtere2ac8ef2008-11-12 12:59:32 +01001777static void trace_iterator_increment(struct trace_iterator *iter)
Steven Rostedt5a90f572008-09-03 17:42:51 -04001778{
Steven Rostedt6d158a82012-06-27 20:46:14 -04001779 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
1780
Steven Rostedt5a90f572008-09-03 17:42:51 -04001781 iter->idx++;
Steven Rostedt6d158a82012-06-27 20:46:14 -04001782 if (buf_iter)
1783 ring_buffer_read(buf_iter, NULL);
Steven Rostedt5a90f572008-09-03 17:42:51 -04001784}
1785
Ingo Molnare309b412008-05-12 21:20:51 +02001786static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04001787peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1788 unsigned long *lost_events)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001789{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001790 struct ring_buffer_event *event;
Steven Rostedt6d158a82012-06-27 20:46:14 -04001791 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001792
Steven Rostedtd7690412008-10-01 00:29:53 -04001793 if (buf_iter)
1794 event = ring_buffer_iter_peek(buf_iter, ts);
1795 else
Steven Rostedtbc21b472010-03-31 19:49:26 -04001796 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1797 lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04001798
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001799 if (event) {
1800 iter->ent_size = ring_buffer_event_length(event);
1801 return ring_buffer_event_data(event);
1802 }
1803 iter->ent_size = 0;
1804 return NULL;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001805}
Steven Rostedtd7690412008-10-01 00:29:53 -04001806
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001807static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04001808__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1809 unsigned long *missing_events, u64 *ent_ts)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001810{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001811 struct ring_buffer *buffer = iter->tr->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001812 struct trace_entry *ent, *next = NULL;
Lai Jiangshanaa274972010-04-05 17:11:05 +08001813 unsigned long lost_events = 0, next_lost = 0;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001814 int cpu_file = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001815 u64 next_ts = 0, ts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001816 int next_cpu = -1;
Steven Rostedt12b5da32012-03-27 10:43:28 -04001817 int next_size = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001818 int cpu;
1819
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001820 /*
1821 * If we are in a per_cpu trace file, don't bother by iterating over
1822 * all cpu and peek directly.
1823 */
1824 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1825 if (ring_buffer_empty_cpu(buffer, cpu_file))
1826 return NULL;
Steven Rostedtbc21b472010-03-31 19:49:26 -04001827 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001828 if (ent_cpu)
1829 *ent_cpu = cpu_file;
1830
1831 return ent;
1832 }
1833
Steven Rostedtab464282008-05-12 21:21:00 +02001834 for_each_tracing_cpu(cpu) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001835
1836 if (ring_buffer_empty_cpu(buffer, cpu))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001837 continue;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001838
Steven Rostedtbc21b472010-03-31 19:49:26 -04001839 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001840
Ingo Molnarcdd31cd2008-05-12 21:20:46 +02001841 /*
1842 * Pick the entry with the smallest timestamp:
1843 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001844 if (ent && (!next || ts < next_ts)) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001845 next = ent;
1846 next_cpu = cpu;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001847 next_ts = ts;
Steven Rostedtbc21b472010-03-31 19:49:26 -04001848 next_lost = lost_events;
Steven Rostedt12b5da32012-03-27 10:43:28 -04001849 next_size = iter->ent_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001850 }
1851 }
1852
Steven Rostedt12b5da32012-03-27 10:43:28 -04001853 iter->ent_size = next_size;
1854
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001855 if (ent_cpu)
1856 *ent_cpu = next_cpu;
1857
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001858 if (ent_ts)
1859 *ent_ts = next_ts;
1860
Steven Rostedtbc21b472010-03-31 19:49:26 -04001861 if (missing_events)
1862 *missing_events = next_lost;
1863
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001864 return next;
1865}
1866
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001867/* Find the next real entry, without updating the iterator itself */
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001868struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1869 int *ent_cpu, u64 *ent_ts)
Steven Rostedtb3806b42008-05-12 21:20:46 +02001870{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001871 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001872}
Ingo Molnar8c523a92008-05-12 21:20:46 +02001873
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001874/* Find the next real entry, and increment the iterator to the next entry */
Jason Wessel955b61e2010-08-05 09:22:23 -05001875void *trace_find_next_entry_inc(struct trace_iterator *iter)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001876{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001877 iter->ent = __find_next_entry(iter, &iter->cpu,
1878 &iter->lost_events, &iter->ts);
Steven Rostedtb3806b42008-05-12 21:20:46 +02001879
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001880 if (iter->ent)
Robert Richtere2ac8ef2008-11-12 12:59:32 +01001881 trace_iterator_increment(iter);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001882
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001883 return iter->ent ? iter : NULL;
Steven Rostedtb3806b42008-05-12 21:20:46 +02001884}
1885
Ingo Molnare309b412008-05-12 21:20:51 +02001886static void trace_consume(struct trace_iterator *iter)
Steven Rostedtb3806b42008-05-12 21:20:46 +02001887{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001888 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1889 &iter->lost_events);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001890}
1891
Ingo Molnare309b412008-05-12 21:20:51 +02001892static void *s_next(struct seq_file *m, void *v, loff_t *pos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001893{
1894 struct trace_iterator *iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001895 int i = (int)*pos;
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001896 void *ent;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001897
Steven Rostedta63ce5b2009-12-07 09:11:39 -05001898 WARN_ON_ONCE(iter->leftover);
1899
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001900 (*pos)++;
1901
1902 /* can't go backwards */
1903 if (iter->idx > i)
1904 return NULL;
1905
1906 if (iter->idx < 0)
Jason Wessel955b61e2010-08-05 09:22:23 -05001907 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001908 else
1909 ent = iter;
1910
1911 while (ent && iter->idx < i)
Jason Wessel955b61e2010-08-05 09:22:23 -05001912 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001913
1914 iter->pos = *pos;
1915
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001916 return ent;
1917}
1918
Jason Wessel955b61e2010-08-05 09:22:23 -05001919void tracing_iter_reset(struct trace_iterator *iter, int cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001920{
1921 struct trace_array *tr = iter->tr;
1922 struct ring_buffer_event *event;
1923 struct ring_buffer_iter *buf_iter;
1924 unsigned long entries = 0;
1925 u64 ts;
1926
1927 tr->data[cpu]->skipped_entries = 0;
1928
Steven Rostedt6d158a82012-06-27 20:46:14 -04001929 buf_iter = trace_buffer_iter(iter, cpu);
1930 if (!buf_iter)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001931 return;
1932
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001933 ring_buffer_iter_reset(buf_iter);
1934
1935 /*
1936 * We could have the case with the max latency tracers
1937 * that a reset never took place on a cpu. This is evident
1938 * by the timestamp being before the start of the buffer.
1939 */
1940 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1941 if (ts >= iter->tr->time_start)
1942 break;
1943 entries++;
1944 ring_buffer_read(buf_iter, NULL);
1945 }
1946
1947 tr->data[cpu]->skipped_entries = entries;
1948}
1949
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001950/*
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001951 * The current tracer is copied to avoid a global locking
1952 * all around.
1953 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001954static void *s_start(struct seq_file *m, loff_t *pos)
1955{
1956 struct trace_iterator *iter = m->private;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001957 int cpu_file = iter->cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001958 void *p = NULL;
1959 loff_t l = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001960 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001961
Hiraku Toyooka2fd196e2012-12-26 11:52:52 +09001962 /*
1963 * copy the tracer to avoid using a global lock all around.
1964 * iter->trace is a copy of current_trace, the pointer to the
1965 * name may be used instead of a strcmp(), as iter->trace->name
1966 * will point to the same string as current_trace->name.
1967 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001968 mutex_lock(&trace_types_lock);
Hiraku Toyooka2fd196e2012-12-26 11:52:52 +09001969 if (unlikely(current_trace && iter->trace->name != current_trace->name))
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001970 *iter->trace = *current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001971 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001972
Hiraku Toyookadebdd572012-12-26 11:53:00 +09001973 if (iter->snapshot && iter->trace->use_max_tr)
1974 return ERR_PTR(-EBUSY);
1975
1976 if (!iter->snapshot)
1977 atomic_inc(&trace_record_cmdline_disabled);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001978
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001979 if (*pos != iter->pos) {
1980 iter->ent = NULL;
1981 iter->cpu = 0;
1982 iter->idx = -1;
1983
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001984 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1985 for_each_tracing_cpu(cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001986 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001987 } else
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001988 tracing_iter_reset(iter, cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001989
Lai Jiangshanac91d852010-03-02 17:54:50 +08001990 iter->leftover = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001991 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1992 ;
1993
1994 } else {
Steven Rostedta63ce5b2009-12-07 09:11:39 -05001995 /*
1996 * If we overflowed the seq_file before, then we want
1997 * to just reuse the trace_seq buffer again.
1998 */
1999 if (iter->leftover)
2000 p = iter;
2001 else {
2002 l = *pos - 1;
2003 p = s_next(m, p, &l);
2004 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002005 }
2006
Lai Jiangshan4f535962009-05-18 19:35:34 +08002007 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002008 trace_access_lock(cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002009 return p;
2010}
2011
2012static void s_stop(struct seq_file *m, void *p)
2013{
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002014 struct trace_iterator *iter = m->private;
2015
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002016 if (iter->snapshot && iter->trace->use_max_tr)
2017 return;
2018
2019 if (!iter->snapshot)
2020 atomic_dec(&trace_record_cmdline_disabled);
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002021 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08002022 trace_event_read_unlock();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002023}
2024
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002025static void
2026get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries)
2027{
2028 unsigned long count;
2029 int cpu;
2030
2031 *total = 0;
2032 *entries = 0;
2033
2034 for_each_tracing_cpu(cpu) {
2035 count = ring_buffer_entries_cpu(tr->buffer, cpu);
2036 /*
2037 * If this buffer has skipped entries, then we hold all
2038 * entries for the trace and we need to ignore the
2039 * ones before the time stamp.
2040 */
2041 if (tr->data[cpu]->skipped_entries) {
2042 count -= tr->data[cpu]->skipped_entries;
2043 /* total is the same as the entries */
2044 *total += count;
2045 } else
2046 *total += count +
2047 ring_buffer_overrun_cpu(tr->buffer, cpu);
2048 *entries += count;
2049 }
2050}
2051
Ingo Molnare309b412008-05-12 21:20:51 +02002052static void print_lat_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002053{
Michael Ellermana6168352008-08-20 16:36:11 -07002054 seq_puts(m, "# _------=> CPU# \n");
2055 seq_puts(m, "# / _-----=> irqs-off \n");
2056 seq_puts(m, "# | / _----=> need-resched \n");
2057 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2058 seq_puts(m, "# ||| / _--=> preempt-depth \n");
Steven Rostedte6e1e252011-03-09 10:41:56 -05002059 seq_puts(m, "# |||| / delay \n");
2060 seq_puts(m, "# cmd pid ||||| time | caller \n");
2061 seq_puts(m, "# \\ / ||||| \\ | / \n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002062}
2063
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002064static void print_event_info(struct trace_array *tr, struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002065{
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002066 unsigned long total;
2067 unsigned long entries;
2068
2069 get_total_entries(tr, &total, &entries);
2070 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2071 entries, total, num_online_cpus());
2072 seq_puts(m, "#\n");
2073}
2074
2075static void print_func_help_header(struct trace_array *tr, struct seq_file *m)
2076{
2077 print_event_info(tr, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002078 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
Michael Ellermana6168352008-08-20 16:36:11 -07002079 seq_puts(m, "# | | | | |\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002080}
2081
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002082static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
Steven Rostedt77271ce2011-11-17 09:34:33 -05002083{
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002084 print_event_info(tr, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002085 seq_puts(m, "# _-----=> irqs-off\n");
2086 seq_puts(m, "# / _----=> need-resched\n");
2087 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2088 seq_puts(m, "# || / _--=> preempt-depth\n");
2089 seq_puts(m, "# ||| / delay\n");
2090 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2091 seq_puts(m, "# | | | |||| | |\n");
2092}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002093
Jiri Olsa62b915f2010-04-02 19:01:22 +02002094void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002095print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2096{
2097 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2098 struct trace_array *tr = iter->tr;
2099 struct trace_array_cpu *data = tr->data[tr->cpu];
2100 struct tracer *type = current_trace;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002101 unsigned long entries;
2102 unsigned long total;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002103 const char *name = "preemption";
2104
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05002105 name = type->name;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002106
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002107 get_total_entries(tr, &total, &entries);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002108
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002109 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002110 name, UTS_RELEASE);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002111 seq_puts(m, "# -----------------------------------"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002112 "---------------------------------\n");
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002113 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002114 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
Steven Rostedt57f50be2008-05-12 21:20:44 +02002115 nsecs_to_usecs(data->saved_latency),
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002116 entries,
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02002117 total,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002118 tr->cpu,
2119#if defined(CONFIG_PREEMPT_NONE)
2120 "server",
2121#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2122 "desktop",
Steven Rostedtb5c21b42008-07-10 20:58:12 -04002123#elif defined(CONFIG_PREEMPT)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002124 "preempt",
2125#else
2126 "unknown",
2127#endif
2128 /* These are reserved for later use */
2129 0, 0, 0, 0);
2130#ifdef CONFIG_SMP
2131 seq_printf(m, " #P:%d)\n", num_online_cpus());
2132#else
2133 seq_puts(m, ")\n");
2134#endif
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002135 seq_puts(m, "# -----------------\n");
2136 seq_printf(m, "# | task: %.16s-%d "
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002137 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
Eric W. Biedermand20b92a2012-03-13 16:02:19 -07002138 data->comm, data->pid,
2139 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002140 data->policy, data->rt_priority);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002141 seq_puts(m, "# -----------------\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002142
2143 if (data->critical_start) {
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002144 seq_puts(m, "# => started at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002145 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2146 trace_print_seq(m, &iter->seq);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002147 seq_puts(m, "\n# => ended at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002148 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2149 trace_print_seq(m, &iter->seq);
Steven Rostedt8248ac02009-09-02 12:27:41 -04002150 seq_puts(m, "\n#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002151 }
2152
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002153 seq_puts(m, "#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002154}
2155
Steven Rostedta3097202008-11-07 22:36:02 -05002156static void test_cpu_buff_start(struct trace_iterator *iter)
2157{
2158 struct trace_seq *s = &iter->seq;
2159
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002160 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2161 return;
2162
2163 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2164 return;
2165
Rusty Russell44623442009-01-01 10:12:23 +10302166 if (cpumask_test_cpu(iter->cpu, iter->started))
Steven Rostedta3097202008-11-07 22:36:02 -05002167 return;
2168
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002169 if (iter->tr->data[iter->cpu]->skipped_entries)
2170 return;
2171
Rusty Russell44623442009-01-01 10:12:23 +10302172 cpumask_set_cpu(iter->cpu, iter->started);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002173
2174 /* Don't print started cpu buffer for the first entry of the trace */
2175 if (iter->idx > 1)
2176 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2177 iter->cpu);
Steven Rostedta3097202008-11-07 22:36:02 -05002178}
2179
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002180static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002181{
Steven Rostedt214023c2008-05-12 21:20:46 +02002182 struct trace_seq *s = &iter->seq;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002183 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002184 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002185 struct trace_event *event;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002186
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002187 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002188
Steven Rostedta3097202008-11-07 22:36:02 -05002189 test_cpu_buff_start(iter);
2190
Steven Rostedtf633cef2008-12-23 23:24:13 -05002191 event = ftrace_find_event(entry->type);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002192
2193 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt27d48be2009-03-04 21:57:29 -05002194 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2195 if (!trace_print_lat_context(iter))
2196 goto partial;
2197 } else {
2198 if (!trace_print_context(iter))
2199 goto partial;
2200 }
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002201 }
2202
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002203 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002204 return event->funcs->trace(iter, sym_flags, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002205
2206 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2207 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04002208
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002209 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002210partial:
2211 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002212}
2213
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002214static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002215{
2216 struct trace_seq *s = &iter->seq;
2217 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002218 struct trace_event *event;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002219
2220 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002221
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002222 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002223 if (!trace_seq_printf(s, "%d %d %llu ",
2224 entry->pid, iter->cpu, iter->ts))
2225 goto partial;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002226 }
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002227
Steven Rostedtf633cef2008-12-23 23:24:13 -05002228 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002229 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002230 return event->funcs->raw(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002231
2232 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2233 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04002234
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002235 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002236partial:
2237 return TRACE_TYPE_PARTIAL_LINE;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002238}
2239
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002240static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002241{
2242 struct trace_seq *s = &iter->seq;
2243 unsigned char newline = '\n';
2244 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002245 struct trace_event *event;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002246
2247 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002248
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002249 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2250 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2251 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2252 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2253 }
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002254
Steven Rostedtf633cef2008-12-23 23:24:13 -05002255 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002256 if (event) {
Steven Rostedta9a57762010-04-22 18:46:14 -04002257 enum print_line_t ret = event->funcs->hex(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002258 if (ret != TRACE_TYPE_HANDLED)
2259 return ret;
2260 }
Steven Rostedt7104f302008-10-01 10:52:51 -04002261
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002262 SEQ_PUT_FIELD_RET(s, newline);
2263
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002264 return TRACE_TYPE_HANDLED;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002265}
2266
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002267static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002268{
2269 struct trace_seq *s = &iter->seq;
2270 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002271 struct trace_event *event;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002272
2273 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002274
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002275 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2276 SEQ_PUT_FIELD_RET(s, entry->pid);
Steven Rostedt1830b52d2009-02-07 19:38:43 -05002277 SEQ_PUT_FIELD_RET(s, iter->cpu);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002278 SEQ_PUT_FIELD_RET(s, iter->ts);
2279 }
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002280
Steven Rostedtf633cef2008-12-23 23:24:13 -05002281 event = ftrace_find_event(entry->type);
Steven Rostedta9a57762010-04-22 18:46:14 -04002282 return event ? event->funcs->binary(iter, 0, event) :
2283 TRACE_TYPE_HANDLED;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002284}
2285
Jiri Olsa62b915f2010-04-02 19:01:22 +02002286int trace_empty(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002287{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002288 struct ring_buffer_iter *buf_iter;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002289 int cpu;
2290
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002291 /* If we are looking at one CPU buffer, only check that one */
2292 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2293 cpu = iter->cpu_file;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002294 buf_iter = trace_buffer_iter(iter, cpu);
2295 if (buf_iter) {
2296 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002297 return 0;
2298 } else {
2299 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2300 return 0;
2301 }
2302 return 1;
2303 }
2304
Steven Rostedtab464282008-05-12 21:21:00 +02002305 for_each_tracing_cpu(cpu) {
Steven Rostedt6d158a82012-06-27 20:46:14 -04002306 buf_iter = trace_buffer_iter(iter, cpu);
2307 if (buf_iter) {
2308 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedtd7690412008-10-01 00:29:53 -04002309 return 0;
2310 } else {
2311 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2312 return 0;
2313 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002314 }
Steven Rostedtd7690412008-10-01 00:29:53 -04002315
Frederic Weisbecker797d3712008-09-30 18:13:45 +02002316 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002317}
2318
Lai Jiangshan4f535962009-05-18 19:35:34 +08002319/* Called with trace_event_read_lock() held. */
Jason Wessel955b61e2010-08-05 09:22:23 -05002320enum print_line_t print_trace_line(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002321{
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002322 enum print_line_t ret;
2323
Jiri Olsaee5e51f2011-03-25 12:05:18 +01002324 if (iter->lost_events &&
2325 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2326 iter->cpu, iter->lost_events))
2327 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002328
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002329 if (iter->trace && iter->trace->print_line) {
2330 ret = iter->trace->print_line(iter);
2331 if (ret != TRACE_TYPE_UNHANDLED)
2332 return ret;
2333 }
Thomas Gleixner72829bc2008-05-23 21:37:28 +02002334
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002335 if (iter->ent->type == TRACE_BPRINT &&
2336 trace_flags & TRACE_ITER_PRINTK &&
2337 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002338 return trace_print_bprintk_msg_only(iter);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002339
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002340 if (iter->ent->type == TRACE_PRINT &&
2341 trace_flags & TRACE_ITER_PRINTK &&
2342 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002343 return trace_print_printk_msg_only(iter);
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002344
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002345 if (trace_flags & TRACE_ITER_BIN)
2346 return print_bin_fmt(iter);
2347
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002348 if (trace_flags & TRACE_ITER_HEX)
2349 return print_hex_fmt(iter);
2350
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002351 if (trace_flags & TRACE_ITER_RAW)
2352 return print_raw_fmt(iter);
2353
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002354 return print_trace_fmt(iter);
2355}
2356
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002357void trace_latency_header(struct seq_file *m)
2358{
2359 struct trace_iterator *iter = m->private;
2360
2361 /* print nothing if the buffers are empty */
2362 if (trace_empty(iter))
2363 return;
2364
2365 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2366 print_trace_header(m, iter);
2367
2368 if (!(trace_flags & TRACE_ITER_VERBOSE))
2369 print_lat_help_header(m);
2370}
2371
Jiri Olsa62b915f2010-04-02 19:01:22 +02002372void trace_default_header(struct seq_file *m)
2373{
2374 struct trace_iterator *iter = m->private;
2375
Jiri Olsaf56e7f82011-06-03 16:58:49 +02002376 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2377 return;
2378
Jiri Olsa62b915f2010-04-02 19:01:22 +02002379 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2380 /* print nothing if the buffers are empty */
2381 if (trace_empty(iter))
2382 return;
2383 print_trace_header(m, iter);
2384 if (!(trace_flags & TRACE_ITER_VERBOSE))
2385 print_lat_help_header(m);
2386 } else {
Steven Rostedt77271ce2011-11-17 09:34:33 -05002387 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2388 if (trace_flags & TRACE_ITER_IRQ_INFO)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002389 print_func_help_header_irq(iter->tr, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002390 else
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002391 print_func_help_header(iter->tr, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002392 }
Jiri Olsa62b915f2010-04-02 19:01:22 +02002393 }
2394}
2395
Steven Rostedte0a413f2011-09-29 21:26:16 -04002396static void test_ftrace_alive(struct seq_file *m)
2397{
2398 if (!ftrace_is_dead())
2399 return;
2400 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2401 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2402}
2403
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002404#ifdef CONFIG_TRACER_MAX_TRACE
2405static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2406{
2407 if (iter->trace->allocated_snapshot)
2408 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2409 else
2410 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2411
2412 seq_printf(m, "# Snapshot commands:\n");
2413 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2414 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2415 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2416 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2417 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2418 seq_printf(m, "# is not a '0' or '1')\n");
2419}
2420#else
2421/* Should never be called */
2422static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2423#endif
2424
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002425static int s_show(struct seq_file *m, void *v)
2426{
2427 struct trace_iterator *iter = v;
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002428 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002429
2430 if (iter->ent == NULL) {
2431 if (iter->tr) {
2432 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2433 seq_puts(m, "#\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002434 test_ftrace_alive(m);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002435 }
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002436 if (iter->snapshot && trace_empty(iter))
2437 print_snapshot_help(m, iter);
2438 else if (iter->trace && iter->trace->print_header)
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002439 iter->trace->print_header(m);
Jiri Olsa62b915f2010-04-02 19:01:22 +02002440 else
2441 trace_default_header(m);
2442
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002443 } else if (iter->leftover) {
2444 /*
2445 * If we filled the seq_file buffer earlier, we
2446 * want to just show it now.
2447 */
2448 ret = trace_print_seq(m, &iter->seq);
2449
2450 /* ret should this time be zero, but you never know */
2451 iter->leftover = ret;
2452
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002453 } else {
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002454 print_trace_line(iter);
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002455 ret = trace_print_seq(m, &iter->seq);
2456 /*
2457 * If we overflow the seq_file buffer, then it will
2458 * ask us for this data again at start up.
2459 * Use that instead.
2460 * ret is 0 if seq_file write succeeded.
2461 * -1 otherwise.
2462 */
2463 iter->leftover = ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002464 }
2465
2466 return 0;
2467}
2468
James Morris88e9d342009-09-22 16:43:43 -07002469static const struct seq_operations tracer_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002470 .start = s_start,
2471 .next = s_next,
2472 .stop = s_stop,
2473 .show = s_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002474};
2475
Ingo Molnare309b412008-05-12 21:20:51 +02002476static struct trace_iterator *
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002477__tracing_open(struct inode *inode, struct file *file, bool snapshot)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002478{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002479 long cpu_file = (long) inode->i_private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002480 struct trace_iterator *iter;
Jiri Olsa50e18b92012-04-25 10:23:39 +02002481 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002482
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002483 if (tracing_disabled)
2484 return ERR_PTR(-ENODEV);
Steven Rostedt60a11772008-05-12 21:20:44 +02002485
Jiri Olsa50e18b92012-04-25 10:23:39 +02002486 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002487 if (!iter)
2488 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002489
Steven Rostedt6d158a82012-06-27 20:46:14 -04002490 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2491 GFP_KERNEL);
Dan Carpenter93574fc2012-07-11 09:35:08 +03002492 if (!iter->buffer_iter)
2493 goto release;
2494
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002495 /*
2496 * We make a copy of the current tracer to avoid concurrent
2497 * changes on it while we are reading.
2498 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002499 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002500 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002501 if (!iter->trace)
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002502 goto fail;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002503
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05002504 *iter->trace = *current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002505
Li Zefan79f55992009-06-15 14:58:26 +08002506 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002507 goto fail;
2508
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05002509 if (current_trace->print_max || snapshot)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002510 iter->tr = &max_tr;
2511 else
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002512 iter->tr = &global_trace;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002513 iter->snapshot = snapshot;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002514 iter->pos = -1;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002515 mutex_init(&iter->mutex);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002516 iter->cpu_file = cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002517
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002518 /* Notify the tracer early; before we stop tracing. */
2519 if (iter->trace && iter->trace->open)
Markus Metzgera93751c2008-12-11 13:53:26 +01002520 iter->trace->open(iter);
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002521
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002522 /* Annotate start of buffers if we had overruns */
2523 if (ring_buffer_overruns(iter->tr->buffer))
2524 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2525
David Sharp8be07092012-11-13 12:18:22 -08002526 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2527 if (trace_clocks[trace_clock_id].in_ns)
2528 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2529
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002530 /* stop the trace while dumping if we are not opening "snapshot" */
2531 if (!iter->snapshot)
2532 tracing_stop();
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002533
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002534 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2535 for_each_tracing_cpu(cpu) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002536 iter->buffer_iter[cpu] =
David Miller72c9ddf2010-04-20 15:47:11 -07002537 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2538 }
2539 ring_buffer_read_prepare_sync();
2540 for_each_tracing_cpu(cpu) {
2541 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002542 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002543 }
2544 } else {
2545 cpu = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002546 iter->buffer_iter[cpu] =
David Miller72c9ddf2010-04-20 15:47:11 -07002547 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2548 ring_buffer_read_prepare_sync();
2549 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002550 tracing_iter_reset(iter, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002551 }
2552
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002553 mutex_unlock(&trace_types_lock);
2554
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002555 return iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002556
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002557 fail:
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002558 mutex_unlock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002559 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04002560 kfree(iter->buffer_iter);
Dan Carpenter93574fc2012-07-11 09:35:08 +03002561release:
Jiri Olsa50e18b92012-04-25 10:23:39 +02002562 seq_release_private(inode, file);
2563 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002564}
2565
2566int tracing_open_generic(struct inode *inode, struct file *filp)
2567{
Steven Rostedt60a11772008-05-12 21:20:44 +02002568 if (tracing_disabled)
2569 return -ENODEV;
2570
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002571 filp->private_data = inode->i_private;
2572 return 0;
2573}
2574
Hannes Eder4fd27352009-02-10 19:44:12 +01002575static int tracing_release(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002576{
matt mooney907f2782010-09-27 19:04:53 -07002577 struct seq_file *m = file->private_data;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002578 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002579 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002580
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002581 if (!(file->f_mode & FMODE_READ))
2582 return 0;
2583
2584 iter = m->private;
2585
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002586 mutex_lock(&trace_types_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002587 for_each_tracing_cpu(cpu) {
2588 if (iter->buffer_iter[cpu])
2589 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2590 }
2591
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002592 if (iter->trace && iter->trace->close)
2593 iter->trace->close(iter);
2594
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002595 if (!iter->snapshot)
2596 /* reenable tracing if it was previously enabled */
2597 tracing_start();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002598 mutex_unlock(&trace_types_lock);
2599
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002600 mutex_destroy(&iter->mutex);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002601 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002602 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04002603 kfree(iter->buffer_iter);
Jiri Olsa50e18b92012-04-25 10:23:39 +02002604 seq_release_private(inode, file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002605 return 0;
2606}
2607
2608static int tracing_open(struct inode *inode, struct file *file)
2609{
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002610 struct trace_iterator *iter;
2611 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002612
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002613 /* If this file was open for write, then erase contents */
2614 if ((file->f_mode & FMODE_WRITE) &&
Steven Rostedt8650ae32009-07-22 23:29:30 -04002615 (file->f_flags & O_TRUNC)) {
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002616 long cpu = (long) inode->i_private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002617
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002618 if (cpu == TRACE_PIPE_ALL_CPU)
2619 tracing_reset_online_cpus(&global_trace);
2620 else
2621 tracing_reset(&global_trace, cpu);
2622 }
2623
2624 if (file->f_mode & FMODE_READ) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002625 iter = __tracing_open(inode, file, false);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002626 if (IS_ERR(iter))
2627 ret = PTR_ERR(iter);
2628 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2629 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2630 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002631 return ret;
2632}
2633
Ingo Molnare309b412008-05-12 21:20:51 +02002634static void *
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002635t_next(struct seq_file *m, void *v, loff_t *pos)
2636{
Li Zefanf129e962009-06-24 09:53:44 +08002637 struct tracer *t = v;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002638
2639 (*pos)++;
2640
2641 if (t)
2642 t = t->next;
2643
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002644 return t;
2645}
2646
2647static void *t_start(struct seq_file *m, loff_t *pos)
2648{
Li Zefanf129e962009-06-24 09:53:44 +08002649 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002650 loff_t l = 0;
2651
2652 mutex_lock(&trace_types_lock);
Li Zefanf129e962009-06-24 09:53:44 +08002653 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002654 ;
2655
2656 return t;
2657}
2658
2659static void t_stop(struct seq_file *m, void *p)
2660{
2661 mutex_unlock(&trace_types_lock);
2662}
2663
2664static int t_show(struct seq_file *m, void *v)
2665{
2666 struct tracer *t = v;
2667
2668 if (!t)
2669 return 0;
2670
2671 seq_printf(m, "%s", t->name);
2672 if (t->next)
2673 seq_putc(m, ' ');
2674 else
2675 seq_putc(m, '\n');
2676
2677 return 0;
2678}
2679
James Morris88e9d342009-09-22 16:43:43 -07002680static const struct seq_operations show_traces_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002681 .start = t_start,
2682 .next = t_next,
2683 .stop = t_stop,
2684 .show = t_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002685};
2686
2687static int show_traces_open(struct inode *inode, struct file *file)
2688{
Steven Rostedt60a11772008-05-12 21:20:44 +02002689 if (tracing_disabled)
2690 return -ENODEV;
2691
Li Zefanf129e962009-06-24 09:53:44 +08002692 return seq_open(file, &show_traces_seq_ops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002693}
2694
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002695static ssize_t
2696tracing_write_stub(struct file *filp, const char __user *ubuf,
2697 size_t count, loff_t *ppos)
2698{
2699 return count;
2700}
2701
Slava Pestov364829b2010-11-24 15:13:16 -08002702static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2703{
2704 if (file->f_mode & FMODE_READ)
2705 return seq_lseek(file, offset, origin);
2706 else
2707 return 0;
2708}
2709
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002710static const struct file_operations tracing_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002711 .open = tracing_open,
2712 .read = seq_read,
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002713 .write = tracing_write_stub,
Slava Pestov364829b2010-11-24 15:13:16 -08002714 .llseek = tracing_seek,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002715 .release = tracing_release,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002716};
2717
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002718static const struct file_operations show_traces_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002719 .open = show_traces_open,
2720 .read = seq_read,
2721 .release = seq_release,
Arnd Bergmannb4447862010-07-07 23:40:11 +02002722 .llseek = seq_lseek,
Ingo Molnarc7078de2008-05-12 21:20:52 +02002723};
2724
Ingo Molnar36dfe922008-05-12 21:20:52 +02002725/*
2726 * Only trace on a CPU if the bitmask is set:
2727 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302728static cpumask_var_t tracing_cpumask;
Ingo Molnar36dfe922008-05-12 21:20:52 +02002729
2730/*
2731 * The tracer itself will not take this lock, but still we want
2732 * to provide a consistent cpumask to user-space:
2733 */
2734static DEFINE_MUTEX(tracing_cpumask_update_lock);
2735
2736/*
2737 * Temporary storage for the character representation of the
2738 * CPU bitmask (and one more byte for the newline):
2739 */
2740static char mask_str[NR_CPUS + 1];
2741
Ingo Molnarc7078de2008-05-12 21:20:52 +02002742static ssize_t
2743tracing_cpumask_read(struct file *filp, char __user *ubuf,
2744 size_t count, loff_t *ppos)
2745{
Ingo Molnar36dfe922008-05-12 21:20:52 +02002746 int len;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002747
2748 mutex_lock(&tracing_cpumask_update_lock);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002749
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302750 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002751 if (count - len < 2) {
2752 count = -EINVAL;
2753 goto out_err;
2754 }
2755 len += sprintf(mask_str + len, "\n");
2756 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2757
2758out_err:
Ingo Molnarc7078de2008-05-12 21:20:52 +02002759 mutex_unlock(&tracing_cpumask_update_lock);
2760
2761 return count;
2762}
2763
2764static ssize_t
2765tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2766 size_t count, loff_t *ppos)
2767{
Ingo Molnar36dfe922008-05-12 21:20:52 +02002768 int err, cpu;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302769 cpumask_var_t tracing_cpumask_new;
2770
2771 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2772 return -ENOMEM;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002773
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302774 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002775 if (err)
2776 goto err_unlock;
2777
Li Zefan215368e2009-06-15 10:56:42 +08002778 mutex_lock(&tracing_cpumask_update_lock);
2779
Steven Rostedta5e25882008-12-02 15:34:05 -05002780 local_irq_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01002781 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtab464282008-05-12 21:21:00 +02002782 for_each_tracing_cpu(cpu) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002783 /*
2784 * Increase/decrease the disabled counter if we are
2785 * about to flip a bit in the cpumask:
2786 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302787 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2788 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002789 atomic_inc(&global_trace.data[cpu]->disabled);
Vaibhav Nagarnaik71babb22012-05-03 18:59:52 -07002790 ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002791 }
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302792 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2793 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002794 atomic_dec(&global_trace.data[cpu]->disabled);
Vaibhav Nagarnaik71babb22012-05-03 18:59:52 -07002795 ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002796 }
2797 }
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01002798 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedta5e25882008-12-02 15:34:05 -05002799 local_irq_enable();
Ingo Molnar36dfe922008-05-12 21:20:52 +02002800
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302801 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002802
Ingo Molnarc7078de2008-05-12 21:20:52 +02002803 mutex_unlock(&tracing_cpumask_update_lock);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302804 free_cpumask_var(tracing_cpumask_new);
Ingo Molnarc7078de2008-05-12 21:20:52 +02002805
Ingo Molnarc7078de2008-05-12 21:20:52 +02002806 return count;
Ingo Molnar36dfe922008-05-12 21:20:52 +02002807
2808err_unlock:
Li Zefan215368e2009-06-15 10:56:42 +08002809 free_cpumask_var(tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002810
2811 return err;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002812}
2813
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002814static const struct file_operations tracing_cpumask_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002815 .open = tracing_open_generic,
2816 .read = tracing_cpumask_read,
2817 .write = tracing_cpumask_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02002818 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002819};
2820
Li Zefanfdb372e2009-12-08 11:15:59 +08002821static int tracing_trace_options_show(struct seq_file *m, void *v)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002822{
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002823 struct tracer_opt *trace_opts;
2824 u32 tracer_flags;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002825 int i;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002826
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002827 mutex_lock(&trace_types_lock);
2828 tracer_flags = current_trace->flags->val;
2829 trace_opts = current_trace->flags->opts;
2830
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002831 for (i = 0; trace_options[i]; i++) {
2832 if (trace_flags & (1 << i))
Li Zefanfdb372e2009-12-08 11:15:59 +08002833 seq_printf(m, "%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002834 else
Li Zefanfdb372e2009-12-08 11:15:59 +08002835 seq_printf(m, "no%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002836 }
2837
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002838 for (i = 0; trace_opts[i].name; i++) {
2839 if (tracer_flags & trace_opts[i].bit)
Li Zefanfdb372e2009-12-08 11:15:59 +08002840 seq_printf(m, "%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002841 else
Li Zefanfdb372e2009-12-08 11:15:59 +08002842 seq_printf(m, "no%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002843 }
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002844 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002845
Li Zefanfdb372e2009-12-08 11:15:59 +08002846 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002847}
2848
Li Zefan8d18eaa2009-12-08 11:17:06 +08002849static int __set_tracer_option(struct tracer *trace,
2850 struct tracer_flags *tracer_flags,
2851 struct tracer_opt *opts, int neg)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002852{
Li Zefan8d18eaa2009-12-08 11:17:06 +08002853 int ret;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002854
Li Zefan8d18eaa2009-12-08 11:17:06 +08002855 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002856 if (ret)
2857 return ret;
2858
2859 if (neg)
Zhaolei77708412009-08-07 18:53:21 +08002860 tracer_flags->val &= ~opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002861 else
Zhaolei77708412009-08-07 18:53:21 +08002862 tracer_flags->val |= opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002863 return 0;
2864}
2865
Li Zefan8d18eaa2009-12-08 11:17:06 +08002866/* Try to assign a tracer specific option */
2867static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2868{
2869 struct tracer_flags *tracer_flags = trace->flags;
2870 struct tracer_opt *opts = NULL;
2871 int i;
2872
2873 for (i = 0; tracer_flags->opts[i].name; i++) {
2874 opts = &tracer_flags->opts[i];
2875
2876 if (strcmp(cmp, opts->name) == 0)
2877 return __set_tracer_option(trace, trace->flags,
2878 opts, neg);
2879 }
2880
2881 return -EINVAL;
2882}
2883
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04002884/* Some tracers require overwrite to stay enabled */
2885int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
2886{
2887 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
2888 return -1;
2889
2890 return 0;
2891}
2892
2893int set_tracer_flag(unsigned int mask, int enabled)
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002894{
2895 /* do nothing if flag is already set */
2896 if (!!(trace_flags & mask) == !!enabled)
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04002897 return 0;
2898
2899 /* Give the tracer a chance to approve the change */
2900 if (current_trace->flag_changed)
2901 if (current_trace->flag_changed(current_trace, mask, !!enabled))
2902 return -EINVAL;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002903
2904 if (enabled)
2905 trace_flags |= mask;
2906 else
2907 trace_flags &= ~mask;
Li Zefane870e9a2010-07-02 11:07:32 +08002908
2909 if (mask == TRACE_ITER_RECORD_CMD)
2910 trace_event_enable_cmd_record(enabled);
David Sharp750912f2010-12-08 13:46:47 -08002911
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04002912 if (mask == TRACE_ITER_OVERWRITE) {
David Sharp750912f2010-12-08 13:46:47 -08002913 ring_buffer_change_overwrite(global_trace.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04002914#ifdef CONFIG_TRACER_MAX_TRACE
2915 ring_buffer_change_overwrite(max_tr.buffer, enabled);
2916#endif
2917 }
Steven Rostedt81698832012-10-11 10:15:05 -04002918
2919 if (mask == TRACE_ITER_PRINTK)
2920 trace_printk_start_stop_comm(enabled);
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04002921
2922 return 0;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002923}
2924
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04002925static int trace_set_options(char *option)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002926{
Li Zefan8d18eaa2009-12-08 11:17:06 +08002927 char *cmp;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002928 int neg = 0;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04002929 int ret = -ENODEV;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002930 int i;
2931
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04002932 cmp = strstrip(option);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002933
Li Zefan8d18eaa2009-12-08 11:17:06 +08002934 if (strncmp(cmp, "no", 2) == 0) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002935 neg = 1;
2936 cmp += 2;
2937 }
2938
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04002939 mutex_lock(&trace_types_lock);
2940
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002941 for (i = 0; trace_options[i]; i++) {
Li Zefan8d18eaa2009-12-08 11:17:06 +08002942 if (strcmp(cmp, trace_options[i]) == 0) {
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04002943 ret = set_tracer_flag(1 << i, !neg);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002944 break;
2945 }
2946 }
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002947
2948 /* If no option could be set, test the specific tracer options */
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04002949 if (!trace_options[i])
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002950 ret = set_tracer_option(current_trace, cmp, neg);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04002951
2952 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002953
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04002954 return ret;
2955}
2956
2957static ssize_t
2958tracing_trace_options_write(struct file *filp, const char __user *ubuf,
2959 size_t cnt, loff_t *ppos)
2960{
2961 char buf[64];
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04002962 int ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04002963
2964 if (cnt >= sizeof(buf))
2965 return -EINVAL;
2966
2967 if (copy_from_user(&buf, ubuf, cnt))
2968 return -EFAULT;
2969
Steven Rostedta8dd2172013-01-09 20:54:17 -05002970 buf[cnt] = 0;
2971
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04002972 ret = trace_set_options(buf);
2973 if (ret < 0)
2974 return ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04002975
Jiri Olsacf8517c2009-10-23 19:36:16 -04002976 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002977
2978 return cnt;
2979}
2980
Li Zefanfdb372e2009-12-08 11:15:59 +08002981static int tracing_trace_options_open(struct inode *inode, struct file *file)
2982{
2983 if (tracing_disabled)
2984 return -ENODEV;
2985 return single_open(file, tracing_trace_options_show, NULL);
2986}
2987
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002988static const struct file_operations tracing_iter_fops = {
Li Zefanfdb372e2009-12-08 11:15:59 +08002989 .open = tracing_trace_options_open,
2990 .read = seq_read,
2991 .llseek = seq_lseek,
2992 .release = single_release,
Steven Rostedtee6bce52008-11-12 17:52:37 -05002993 .write = tracing_trace_options_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002994};
2995
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002996static const char readme_msg[] =
2997 "tracing mini-HOWTO:\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002998 "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2999 "# cat /sys/kernel/debug/tracing/available_tracers\n"
Geunsik Lim1e42e832012-02-08 19:05:36 +09003000 "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003001 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Nikanth Karthikesanbc2b6872009-03-23 11:58:31 +05303002 "nop\n"
Geunsik Lim1e42e832012-02-08 19:05:36 +09003003 "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003004 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Geunsik Lim1e42e832012-02-08 19:05:36 +09003005 "wakeup\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003006 "# cat /sys/kernel/debug/tracing/trace_options\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003007 "noprint-parent nosym-offset nosym-addr noverbose\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003008 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
Geunsik Lim9b5f8b32011-08-12 14:30:22 +09003009 "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003010 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
Geunsik Lim9b5f8b32011-08-12 14:30:22 +09003011 "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003012;
3013
3014static ssize_t
3015tracing_readme_read(struct file *filp, char __user *ubuf,
3016 size_t cnt, loff_t *ppos)
3017{
3018 return simple_read_from_buffer(ubuf, cnt, ppos,
3019 readme_msg, strlen(readme_msg));
3020}
3021
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003022static const struct file_operations tracing_readme_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003023 .open = tracing_open_generic,
3024 .read = tracing_readme_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003025 .llseek = generic_file_llseek,
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003026};
3027
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003028static ssize_t
Avadh Patel69abe6a2009-04-10 16:04:48 -04003029tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3030 size_t cnt, loff_t *ppos)
3031{
3032 char *buf_comm;
3033 char *file_buf;
3034 char *buf;
3035 int len = 0;
3036 int pid;
3037 int i;
3038
3039 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3040 if (!file_buf)
3041 return -ENOMEM;
3042
3043 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3044 if (!buf_comm) {
3045 kfree(file_buf);
3046 return -ENOMEM;
3047 }
3048
3049 buf = file_buf;
3050
3051 for (i = 0; i < SAVED_CMDLINES; i++) {
3052 int r;
3053
3054 pid = map_cmdline_to_pid[i];
3055 if (pid == -1 || pid == NO_CMDLINE_MAP)
3056 continue;
3057
3058 trace_find_cmdline(pid, buf_comm);
3059 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3060 buf += r;
3061 len += r;
3062 }
3063
3064 len = simple_read_from_buffer(ubuf, cnt, ppos,
3065 file_buf, len);
3066
3067 kfree(file_buf);
3068 kfree(buf_comm);
3069
3070 return len;
3071}
3072
3073static const struct file_operations tracing_saved_cmdlines_fops = {
3074 .open = tracing_open_generic,
3075 .read = tracing_saved_cmdlines_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003076 .llseek = generic_file_llseek,
Avadh Patel69abe6a2009-04-10 16:04:48 -04003077};
3078
3079static ssize_t
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003080tracing_set_trace_read(struct file *filp, char __user *ubuf,
3081 size_t cnt, loff_t *ppos)
3082{
Li Zefanee6c2c12009-09-18 14:06:47 +08003083 char buf[MAX_TRACER_SIZE+2];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003084 int r;
3085
3086 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05003087 r = sprintf(buf, "%s\n", current_trace->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003088 mutex_unlock(&trace_types_lock);
3089
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003090 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003091}
3092
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003093int tracer_init(struct tracer *t, struct trace_array *tr)
3094{
3095 tracing_reset_online_cpus(tr);
3096 return t->init(tr);
3097}
3098
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003099static void set_buffer_entries(struct trace_array *tr, unsigned long val)
3100{
3101 int cpu;
3102 for_each_tracing_cpu(cpu)
3103 tr->data[cpu]->entries = val;
3104}
3105
Hiraku Toyookad60da502012-10-17 11:56:16 +09003106/* resize @tr's buffer to the size of @size_tr's entries */
3107static int resize_buffer_duplicate_size(struct trace_array *tr,
3108 struct trace_array *size_tr, int cpu_id)
3109{
3110 int cpu, ret = 0;
3111
3112 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3113 for_each_tracing_cpu(cpu) {
3114 ret = ring_buffer_resize(tr->buffer,
3115 size_tr->data[cpu]->entries, cpu);
3116 if (ret < 0)
3117 break;
3118 tr->data[cpu]->entries = size_tr->data[cpu]->entries;
3119 }
3120 } else {
3121 ret = ring_buffer_resize(tr->buffer,
3122 size_tr->data[cpu_id]->entries, cpu_id);
3123 if (ret == 0)
3124 tr->data[cpu_id]->entries =
3125 size_tr->data[cpu_id]->entries;
3126 }
3127
3128 return ret;
3129}
3130
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003131static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
Steven Rostedt73c51622009-03-11 13:42:01 -04003132{
3133 int ret;
3134
3135 /*
3136 * If kernel or user changes the size of the ring buffer
Steven Rostedta123c522009-03-12 11:21:08 -04003137 * we use the size that was given, and we can forget about
3138 * expanding it later.
Steven Rostedt73c51622009-03-11 13:42:01 -04003139 */
3140 ring_buffer_expanded = 1;
3141
Steven Rostedtb382ede62012-10-10 21:44:34 -04003142 /* May be called before buffers are initialized */
3143 if (!global_trace.buffer)
3144 return 0;
3145
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003146 ret = ring_buffer_resize(global_trace.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003147 if (ret < 0)
3148 return ret;
3149
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003150 if (!current_trace->use_max_tr)
3151 goto out;
3152
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003153 ret = ring_buffer_resize(max_tr.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003154 if (ret < 0) {
Hiraku Toyookad60da502012-10-17 11:56:16 +09003155 int r = resize_buffer_duplicate_size(&global_trace,
3156 &global_trace, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003157 if (r < 0) {
Steven Rostedta123c522009-03-12 11:21:08 -04003158 /*
3159 * AARGH! We are left with different
3160 * size max buffer!!!!
3161 * The max buffer is our "snapshot" buffer.
3162 * When a tracer needs a snapshot (one of the
3163 * latency tracers), it swaps the max buffer
3164 * with the saved snap shot. We succeeded to
3165 * update the size of the main buffer, but failed to
3166 * update the size of the max buffer. But when we tried
3167 * to reset the main buffer to the original size, we
3168 * failed there too. This is very unlikely to
3169 * happen, but if it does, warn and kill all
3170 * tracing.
3171 */
Steven Rostedt73c51622009-03-11 13:42:01 -04003172 WARN_ON(1);
3173 tracing_disabled = 1;
3174 }
3175 return ret;
3176 }
3177
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003178 if (cpu == RING_BUFFER_ALL_CPUS)
3179 set_buffer_entries(&max_tr, size);
3180 else
3181 max_tr.data[cpu]->entries = size;
3182
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003183 out:
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003184 if (cpu == RING_BUFFER_ALL_CPUS)
3185 set_buffer_entries(&global_trace, size);
3186 else
3187 global_trace.data[cpu]->entries = size;
Steven Rostedt73c51622009-03-11 13:42:01 -04003188
3189 return ret;
3190}
3191
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003192static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003193{
Vaibhav Nagarnaik83f40312012-05-03 18:59:50 -07003194 int ret = size;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003195
3196 mutex_lock(&trace_types_lock);
3197
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003198 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3199 /* make sure, this cpu is enabled in the mask */
3200 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3201 ret = -EINVAL;
3202 goto out;
3203 }
3204 }
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003205
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003206 ret = __tracing_resize_ring_buffer(size, cpu_id);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003207 if (ret < 0)
3208 ret = -ENOMEM;
3209
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003210out:
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003211 mutex_unlock(&trace_types_lock);
3212
3213 return ret;
3214}
3215
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003216
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003217/**
3218 * tracing_update_buffers - used by tracing facility to expand ring buffers
3219 *
3220 * To save on memory when the tracing is never used on a system with it
3221 * configured in. The ring buffers are set to a minimum size. But once
3222 * a user starts to use the tracing facility, then they need to grow
3223 * to their default size.
3224 *
3225 * This function is to be called when a tracer is about to be used.
3226 */
3227int tracing_update_buffers(void)
3228{
3229 int ret = 0;
3230
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003231 mutex_lock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003232 if (!ring_buffer_expanded)
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003233 ret = __tracing_resize_ring_buffer(trace_buf_size,
3234 RING_BUFFER_ALL_CPUS);
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003235 mutex_unlock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003236
3237 return ret;
3238}
3239
Steven Rostedt577b7852009-02-26 23:43:05 -05003240struct trace_option_dentry;
3241
3242static struct trace_option_dentry *
3243create_trace_option_files(struct tracer *tracer);
3244
3245static void
3246destroy_trace_option_files(struct trace_option_dentry *topts);
3247
Steven Rostedtb2821ae2009-02-02 21:38:32 -05003248static int tracing_set_tracer(const char *buf)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003249{
Steven Rostedt577b7852009-02-26 23:43:05 -05003250 static struct trace_option_dentry *topts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003251 struct trace_array *tr = &global_trace;
3252 struct tracer *t;
Steven Rostedt34600f02013-01-22 13:35:11 -05003253 bool had_max_tr;
Peter Zijlstrad9e54072008-11-01 19:57:37 +01003254 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003255
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003256 mutex_lock(&trace_types_lock);
3257
Steven Rostedt73c51622009-03-11 13:42:01 -04003258 if (!ring_buffer_expanded) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003259 ret = __tracing_resize_ring_buffer(trace_buf_size,
3260 RING_BUFFER_ALL_CPUS);
Steven Rostedt73c51622009-03-11 13:42:01 -04003261 if (ret < 0)
Frederic Weisbecker59f586d2009-03-15 22:10:39 +01003262 goto out;
Steven Rostedt73c51622009-03-11 13:42:01 -04003263 ret = 0;
3264 }
3265
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003266 for (t = trace_types; t; t = t->next) {
3267 if (strcmp(t->name, buf) == 0)
3268 break;
3269 }
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02003270 if (!t) {
3271 ret = -EINVAL;
3272 goto out;
3273 }
3274 if (t == current_trace)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003275 goto out;
3276
Steven Rostedt9f029e82008-11-12 15:24:24 -05003277 trace_branch_disable();
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003278
3279 current_trace->enabled = false;
3280
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05003281 if (current_trace->reset)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003282 current_trace->reset(tr);
Steven Rostedt34600f02013-01-22 13:35:11 -05003283
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05003284 had_max_tr = current_trace->allocated_snapshot;
Steven Rostedt34600f02013-01-22 13:35:11 -05003285 current_trace = &nop_trace;
3286
3287 if (had_max_tr && !t->use_max_tr) {
3288 /*
3289 * We need to make sure that the update_max_tr sees that
3290 * current_trace changed to nop_trace to keep it from
3291 * swapping the buffers after we resize it.
3292 * The update_max_tr is called from interrupts disabled
3293 * so a synchronized_sched() is sufficient.
3294 */
3295 synchronize_sched();
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003296 /*
3297 * We don't free the ring buffer. instead, resize it because
3298 * The max_tr ring buffer has some state (e.g. ring->clock) and
3299 * we want preserve it.
3300 */
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003301 ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3302 set_buffer_entries(&max_tr, 1);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003303 tracing_reset_online_cpus(&max_tr);
3304 current_trace->allocated_snapshot = false;
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003305 }
Steven Rostedt577b7852009-02-26 23:43:05 -05003306 destroy_trace_option_files(topts);
3307
Anton Vorontsovb2ad3682012-07-09 17:10:39 -07003308 topts = create_trace_option_files(t);
Steven Rostedt34600f02013-01-22 13:35:11 -05003309 if (t->use_max_tr && !had_max_tr) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003310 /* we need to make per cpu buffer sizes equivalent */
Hiraku Toyookad60da502012-10-17 11:56:16 +09003311 ret = resize_buffer_duplicate_size(&max_tr, &global_trace,
3312 RING_BUFFER_ALL_CPUS);
3313 if (ret < 0)
3314 goto out;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003315 t->allocated_snapshot = true;
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003316 }
Steven Rostedt577b7852009-02-26 23:43:05 -05003317
Frederic Weisbecker1c800252008-11-16 05:57:26 +01003318 if (t->init) {
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003319 ret = tracer_init(t, tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +01003320 if (ret)
3321 goto out;
3322 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003323
Anton Vorontsovb2ad3682012-07-09 17:10:39 -07003324 current_trace = t;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003325 current_trace->enabled = true;
Steven Rostedt9f029e82008-11-12 15:24:24 -05003326 trace_branch_enable(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003327 out:
3328 mutex_unlock(&trace_types_lock);
3329
Peter Zijlstrad9e54072008-11-01 19:57:37 +01003330 return ret;
3331}
3332
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003333static ssize_t
3334tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3335 size_t cnt, loff_t *ppos)
3336{
Li Zefanee6c2c12009-09-18 14:06:47 +08003337 char buf[MAX_TRACER_SIZE+1];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003338 int i;
3339 size_t ret;
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01003340 int err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003341
Steven Rostedt60063a62008-10-28 10:44:24 -04003342 ret = cnt;
3343
Li Zefanee6c2c12009-09-18 14:06:47 +08003344 if (cnt > MAX_TRACER_SIZE)
3345 cnt = MAX_TRACER_SIZE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003346
3347 if (copy_from_user(&buf, ubuf, cnt))
3348 return -EFAULT;
3349
3350 buf[cnt] = 0;
3351
3352 /* strip ending whitespace. */
3353 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3354 buf[i] = 0;
3355
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01003356 err = tracing_set_tracer(buf);
3357 if (err)
3358 return err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003359
Jiri Olsacf8517c2009-10-23 19:36:16 -04003360 *ppos += ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003361
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02003362 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003363}
3364
3365static ssize_t
3366tracing_max_lat_read(struct file *filp, char __user *ubuf,
3367 size_t cnt, loff_t *ppos)
3368{
3369 unsigned long *ptr = filp->private_data;
3370 char buf[64];
3371 int r;
3372
Steven Rostedtcffae432008-05-12 21:21:00 +02003373 r = snprintf(buf, sizeof(buf), "%ld\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003374 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
Steven Rostedtcffae432008-05-12 21:21:00 +02003375 if (r > sizeof(buf))
3376 r = sizeof(buf);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003377 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003378}
3379
3380static ssize_t
3381tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3382 size_t cnt, loff_t *ppos)
3383{
Hannes Eder5e398412009-02-10 19:44:34 +01003384 unsigned long *ptr = filp->private_data;
Hannes Eder5e398412009-02-10 19:44:34 +01003385 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003386 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003387
Peter Huewe22fe9b52011-06-07 21:58:27 +02003388 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3389 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003390 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003391
3392 *ptr = val * 1000;
3393
3394 return cnt;
3395}
3396
Steven Rostedtb3806b42008-05-12 21:20:46 +02003397static int tracing_open_pipe(struct inode *inode, struct file *filp)
3398{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003399 long cpu_file = (long) inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003400 struct trace_iterator *iter;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003401 int ret = 0;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003402
3403 if (tracing_disabled)
3404 return -ENODEV;
3405
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003406 mutex_lock(&trace_types_lock);
3407
Steven Rostedtb3806b42008-05-12 21:20:46 +02003408 /* create a buffer to store the information to pass to userspace */
3409 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003410 if (!iter) {
3411 ret = -ENOMEM;
3412 goto out;
3413 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003414
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003415 /*
3416 * We make a copy of the current tracer to avoid concurrent
3417 * changes on it while we are reading.
3418 */
3419 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3420 if (!iter->trace) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003421 ret = -ENOMEM;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003422 goto fail;
3423 }
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05003424 *iter->trace = *current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003425
3426 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3427 ret = -ENOMEM;
3428 goto fail;
Rusty Russell44623442009-01-01 10:12:23 +10303429 }
3430
Steven Rostedta3097202008-11-07 22:36:02 -05003431 /* trace pipe does not show start of buffer */
Rusty Russell44623442009-01-01 10:12:23 +10303432 cpumask_setall(iter->started);
Steven Rostedta3097202008-11-07 22:36:02 -05003433
Steven Rostedt112f38a72009-06-01 15:16:05 -04003434 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3435 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3436
David Sharp8be07092012-11-13 12:18:22 -08003437 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3438 if (trace_clocks[trace_clock_id].in_ns)
3439 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3440
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003441 iter->cpu_file = cpu_file;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003442 iter->tr = &global_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003443 mutex_init(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003444 filp->private_data = iter;
3445
Steven Rostedt107bad82008-05-12 21:21:01 +02003446 if (iter->trace->pipe_open)
3447 iter->trace->pipe_open(iter);
Steven Rostedt107bad82008-05-12 21:21:01 +02003448
Arnd Bergmannb4447862010-07-07 23:40:11 +02003449 nonseekable_open(inode, filp);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003450out:
3451 mutex_unlock(&trace_types_lock);
3452 return ret;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003453
3454fail:
3455 kfree(iter->trace);
3456 kfree(iter);
3457 mutex_unlock(&trace_types_lock);
3458 return ret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003459}
3460
3461static int tracing_release_pipe(struct inode *inode, struct file *file)
3462{
3463 struct trace_iterator *iter = file->private_data;
3464
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003465 mutex_lock(&trace_types_lock);
3466
Steven Rostedt29bf4a52009-12-09 12:37:43 -05003467 if (iter->trace->pipe_close)
Steven Rostedtc521efd2009-12-07 09:06:24 -05003468 iter->trace->pipe_close(iter);
3469
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003470 mutex_unlock(&trace_types_lock);
3471
Rusty Russell44623442009-01-01 10:12:23 +10303472 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003473 mutex_destroy(&iter->mutex);
3474 kfree(iter->trace);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003475 kfree(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003476
3477 return 0;
3478}
3479
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003480static unsigned int
3481tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3482{
3483 struct trace_iterator *iter = filp->private_data;
3484
3485 if (trace_flags & TRACE_ITER_BLOCK) {
3486 /*
3487 * Always select as readable when in blocking mode
3488 */
3489 return POLLIN | POLLRDNORM;
Ingo Molnarafc2abc2008-05-12 21:21:00 +02003490 } else {
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003491 if (!trace_empty(iter))
3492 return POLLIN | POLLRDNORM;
3493 poll_wait(filp, &trace_wait, poll_table);
3494 if (!trace_empty(iter))
3495 return POLLIN | POLLRDNORM;
3496
3497 return 0;
3498 }
3499}
3500
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003501/*
3502 * This is a make-shift waitqueue.
3503 * A tracer might use this callback on some rare cases:
3504 *
3505 * 1) the current tracer might hold the runqueue lock when it wakes up
3506 * a reader, hence a deadlock (sched, function, and function graph tracers)
3507 * 2) the function tracers, trace all functions, we don't want
3508 * the overhead of calling wake_up and friends
3509 * (and tracing them too)
3510 *
3511 * Anyway, this is really very primitive wakeup.
3512 */
3513void poll_wait_pipe(struct trace_iterator *iter)
3514{
3515 set_current_state(TASK_INTERRUPTIBLE);
3516 /* sleep for 100 msecs, and try again. */
3517 schedule_timeout(HZ / 10);
3518}
3519
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003520/* Must be called with trace_types_lock mutex held. */
3521static int tracing_wait_pipe(struct file *filp)
3522{
3523 struct trace_iterator *iter = filp->private_data;
3524
3525 while (trace_empty(iter)) {
3526
3527 if ((filp->f_flags & O_NONBLOCK)) {
3528 return -EAGAIN;
3529 }
3530
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003531 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003532
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003533 iter->trace->wait_pipe(iter);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003534
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003535 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003536
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003537 if (signal_pending(current))
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003538 return -EINTR;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003539
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003540 /*
Liu Bo250bfd32013-01-14 10:54:11 +08003541 * We block until we read something and tracing is disabled.
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003542 * We still block if tracing is disabled, but we have never
3543 * read anything. This allows a user to cat this file, and
3544 * then enable tracing. But after we have read something,
3545 * we give an EOF when tracing is again disabled.
3546 *
3547 * iter->pos will be 0 if we haven't read anything.
3548 */
Liu Bo250bfd32013-01-14 10:54:11 +08003549 if (!tracing_is_enabled() && iter->pos)
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003550 break;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003551 }
3552
3553 return 1;
3554}
3555
Steven Rostedtb3806b42008-05-12 21:20:46 +02003556/*
3557 * Consumer reader.
3558 */
3559static ssize_t
3560tracing_read_pipe(struct file *filp, char __user *ubuf,
3561 size_t cnt, loff_t *ppos)
3562{
3563 struct trace_iterator *iter = filp->private_data;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003564 ssize_t sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003565
3566 /* return any leftover data */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003567 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3568 if (sret != -EBUSY)
3569 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003570
Steven Rostedtf9520752009-03-02 14:04:40 -05003571 trace_seq_init(&iter->seq);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003572
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003573 /* copy the tracer to avoid using a global lock all around */
Steven Rostedt107bad82008-05-12 21:21:01 +02003574 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05003575 if (unlikely(iter->trace->name != current_trace->name))
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003576 *iter->trace = *current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003577 mutex_unlock(&trace_types_lock);
3578
3579 /*
3580 * Avoid more than one consumer on a single file descriptor
3581 * This is just a matter of traces coherency, the ring buffer itself
3582 * is protected.
3583 */
3584 mutex_lock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003585 if (iter->trace->read) {
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003586 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3587 if (sret)
Steven Rostedt107bad82008-05-12 21:21:01 +02003588 goto out;
Steven Rostedt107bad82008-05-12 21:21:01 +02003589 }
3590
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003591waitagain:
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003592 sret = tracing_wait_pipe(filp);
3593 if (sret <= 0)
3594 goto out;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003595
3596 /* stop when tracing is finished */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003597 if (trace_empty(iter)) {
3598 sret = 0;
Steven Rostedt107bad82008-05-12 21:21:01 +02003599 goto out;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003600 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003601
3602 if (cnt >= PAGE_SIZE)
3603 cnt = PAGE_SIZE - 1;
3604
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003605 /* reset all but tr, trace, and overruns */
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003606 memset(&iter->seq, 0,
3607 sizeof(struct trace_iterator) -
3608 offsetof(struct trace_iterator, seq));
Steven Rostedt4823ed72008-05-12 21:21:01 +02003609 iter->pos = -1;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003610
Lai Jiangshan4f535962009-05-18 19:35:34 +08003611 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003612 trace_access_lock(iter->cpu_file);
Jason Wessel955b61e2010-08-05 09:22:23 -05003613 while (trace_find_next_entry_inc(iter) != NULL) {
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003614 enum print_line_t ret;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003615 int len = iter->seq.len;
3616
Ingo Molnarf9896bf2008-05-12 21:20:47 +02003617 ret = print_trace_line(iter);
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003618 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt088b1e422008-05-12 21:20:48 +02003619 /* don't print partial lines */
3620 iter->seq.len = len;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003621 break;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003622 }
Frederic Weisbeckerb91facc2009-02-06 18:30:44 +01003623 if (ret != TRACE_TYPE_NO_CONSUME)
3624 trace_consume(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003625
3626 if (iter->seq.len >= cnt)
3627 break;
Jiri Olsaee5e51f2011-03-25 12:05:18 +01003628
3629 /*
3630 * Setting the full flag means we reached the trace_seq buffer
3631 * size and we should leave by partial output condition above.
3632 * One of the trace_seq_* functions is not used properly.
3633 */
3634 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3635 iter->ent->type);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003636 }
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003637 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003638 trace_event_read_unlock();
Steven Rostedtb3806b42008-05-12 21:20:46 +02003639
Steven Rostedtb3806b42008-05-12 21:20:46 +02003640 /* Now copy what we have to the user */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003641 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3642 if (iter->seq.readpos >= iter->seq.len)
Steven Rostedtf9520752009-03-02 14:04:40 -05003643 trace_seq_init(&iter->seq);
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003644
3645 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003646 * If there was nothing to send to user, in spite of consuming trace
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003647 * entries, go back to wait for more entries.
3648 */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003649 if (sret == -EBUSY)
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003650 goto waitagain;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003651
Steven Rostedt107bad82008-05-12 21:21:01 +02003652out:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003653 mutex_unlock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003654
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003655 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003656}
3657
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003658static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3659 struct pipe_buffer *buf)
3660{
3661 __free_page(buf->page);
3662}
3663
3664static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3665 unsigned int idx)
3666{
3667 __free_page(spd->pages[idx]);
3668}
3669
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08003670static const struct pipe_buf_operations tracing_pipe_buf_ops = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003671 .can_merge = 0,
3672 .map = generic_pipe_buf_map,
3673 .unmap = generic_pipe_buf_unmap,
3674 .confirm = generic_pipe_buf_confirm,
3675 .release = tracing_pipe_buf_release,
3676 .steal = generic_pipe_buf_steal,
3677 .get = generic_pipe_buf_get,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003678};
3679
Steven Rostedt34cd4992009-02-09 12:06:29 -05003680static size_t
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01003681tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003682{
3683 size_t count;
3684 int ret;
3685
3686 /* Seq buffer is page-sized, exactly what we need. */
3687 for (;;) {
3688 count = iter->seq.len;
3689 ret = print_trace_line(iter);
3690 count = iter->seq.len - count;
3691 if (rem < count) {
3692 rem = 0;
3693 iter->seq.len -= count;
3694 break;
3695 }
3696 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3697 iter->seq.len -= count;
3698 break;
3699 }
3700
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08003701 if (ret != TRACE_TYPE_NO_CONSUME)
3702 trace_consume(iter);
Steven Rostedt34cd4992009-02-09 12:06:29 -05003703 rem -= count;
Jason Wessel955b61e2010-08-05 09:22:23 -05003704 if (!trace_find_next_entry_inc(iter)) {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003705 rem = 0;
3706 iter->ent = NULL;
3707 break;
3708 }
3709 }
3710
3711 return rem;
3712}
3713
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003714static ssize_t tracing_splice_read_pipe(struct file *filp,
3715 loff_t *ppos,
3716 struct pipe_inode_info *pipe,
3717 size_t len,
3718 unsigned int flags)
3719{
Jens Axboe35f3d142010-05-20 10:43:18 +02003720 struct page *pages_def[PIPE_DEF_BUFFERS];
3721 struct partial_page partial_def[PIPE_DEF_BUFFERS];
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003722 struct trace_iterator *iter = filp->private_data;
3723 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02003724 .pages = pages_def,
3725 .partial = partial_def,
Steven Rostedt34cd4992009-02-09 12:06:29 -05003726 .nr_pages = 0, /* This gets updated below. */
Eric Dumazet047fe362012-06-12 15:24:40 +02003727 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt34cd4992009-02-09 12:06:29 -05003728 .flags = flags,
3729 .ops = &tracing_pipe_buf_ops,
3730 .spd_release = tracing_spd_release_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003731 };
3732 ssize_t ret;
Steven Rostedt34cd4992009-02-09 12:06:29 -05003733 size_t rem;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003734 unsigned int i;
3735
Jens Axboe35f3d142010-05-20 10:43:18 +02003736 if (splice_grow_spd(pipe, &spd))
3737 return -ENOMEM;
3738
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003739 /* copy the tracer to avoid using a global lock all around */
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003740 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05003741 if (unlikely(iter->trace->name != current_trace->name))
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003742 *iter->trace = *current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003743 mutex_unlock(&trace_types_lock);
3744
3745 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003746
3747 if (iter->trace->splice_read) {
3748 ret = iter->trace->splice_read(iter, filp,
3749 ppos, pipe, len, flags);
3750 if (ret)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003751 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003752 }
3753
3754 ret = tracing_wait_pipe(filp);
3755 if (ret <= 0)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003756 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003757
Jason Wessel955b61e2010-08-05 09:22:23 -05003758 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003759 ret = -EFAULT;
Steven Rostedt34cd4992009-02-09 12:06:29 -05003760 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003761 }
3762
Lai Jiangshan4f535962009-05-18 19:35:34 +08003763 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003764 trace_access_lock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003765
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003766 /* Fill as many pages as possible. */
Jens Axboe35f3d142010-05-20 10:43:18 +02003767 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3768 spd.pages[i] = alloc_page(GFP_KERNEL);
3769 if (!spd.pages[i])
Steven Rostedt34cd4992009-02-09 12:06:29 -05003770 break;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003771
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01003772 rem = tracing_fill_pipe_page(rem, iter);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003773
3774 /* Copy the data into the page, so we can start over. */
3775 ret = trace_seq_to_buffer(&iter->seq,
Jens Axboe35f3d142010-05-20 10:43:18 +02003776 page_address(spd.pages[i]),
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003777 iter->seq.len);
3778 if (ret < 0) {
Jens Axboe35f3d142010-05-20 10:43:18 +02003779 __free_page(spd.pages[i]);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003780 break;
3781 }
Jens Axboe35f3d142010-05-20 10:43:18 +02003782 spd.partial[i].offset = 0;
3783 spd.partial[i].len = iter->seq.len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003784
Steven Rostedtf9520752009-03-02 14:04:40 -05003785 trace_seq_init(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003786 }
3787
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003788 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003789 trace_event_read_unlock();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003790 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003791
3792 spd.nr_pages = i;
3793
Jens Axboe35f3d142010-05-20 10:43:18 +02003794 ret = splice_to_pipe(pipe, &spd);
3795out:
Eric Dumazet047fe362012-06-12 15:24:40 +02003796 splice_shrink_spd(&spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02003797 return ret;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003798
Steven Rostedt34cd4992009-02-09 12:06:29 -05003799out_err:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003800 mutex_unlock(&iter->mutex);
Jens Axboe35f3d142010-05-20 10:43:18 +02003801 goto out;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003802}
3803
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003804struct ftrace_entries_info {
3805 struct trace_array *tr;
3806 int cpu;
3807};
3808
3809static int tracing_entries_open(struct inode *inode, struct file *filp)
3810{
3811 struct ftrace_entries_info *info;
3812
3813 if (tracing_disabled)
3814 return -ENODEV;
3815
3816 info = kzalloc(sizeof(*info), GFP_KERNEL);
3817 if (!info)
3818 return -ENOMEM;
3819
3820 info->tr = &global_trace;
3821 info->cpu = (unsigned long)inode->i_private;
3822
3823 filp->private_data = info;
3824
3825 return 0;
3826}
3827
Steven Rostedta98a3c32008-05-12 21:20:59 +02003828static ssize_t
3829tracing_entries_read(struct file *filp, char __user *ubuf,
3830 size_t cnt, loff_t *ppos)
3831{
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003832 struct ftrace_entries_info *info = filp->private_data;
3833 struct trace_array *tr = info->tr;
3834 char buf[64];
3835 int r = 0;
3836 ssize_t ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003837
Steven Rostedtdb526ca2009-03-12 13:53:25 -04003838 mutex_lock(&trace_types_lock);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003839
3840 if (info->cpu == RING_BUFFER_ALL_CPUS) {
3841 int cpu, buf_size_same;
3842 unsigned long size;
3843
3844 size = 0;
3845 buf_size_same = 1;
3846 /* check if all cpu sizes are same */
3847 for_each_tracing_cpu(cpu) {
3848 /* fill in the size from first enabled cpu */
3849 if (size == 0)
3850 size = tr->data[cpu]->entries;
3851 if (size != tr->data[cpu]->entries) {
3852 buf_size_same = 0;
3853 break;
3854 }
3855 }
3856
3857 if (buf_size_same) {
3858 if (!ring_buffer_expanded)
3859 r = sprintf(buf, "%lu (expanded: %lu)\n",
3860 size >> 10,
3861 trace_buf_size >> 10);
3862 else
3863 r = sprintf(buf, "%lu\n", size >> 10);
3864 } else
3865 r = sprintf(buf, "X\n");
3866 } else
3867 r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3868
Steven Rostedtdb526ca2009-03-12 13:53:25 -04003869 mutex_unlock(&trace_types_lock);
3870
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003871 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3872 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003873}
3874
3875static ssize_t
3876tracing_entries_write(struct file *filp, const char __user *ubuf,
3877 size_t cnt, loff_t *ppos)
3878{
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003879 struct ftrace_entries_info *info = filp->private_data;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003880 unsigned long val;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003881 int ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003882
Peter Huewe22fe9b52011-06-07 21:58:27 +02003883 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3884 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003885 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003886
3887 /* must have at least 1 entry */
3888 if (!val)
3889 return -EINVAL;
3890
Steven Rostedt1696b2b2008-11-13 00:09:35 -05003891 /* value is in KB */
3892 val <<= 10;
3893
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003894 ret = tracing_resize_ring_buffer(val, info->cpu);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003895 if (ret < 0)
3896 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003897
Jiri Olsacf8517c2009-10-23 19:36:16 -04003898 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003899
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003900 return cnt;
3901}
Steven Rostedtbf5e6512008-11-10 21:46:00 -05003902
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003903static int
3904tracing_entries_release(struct inode *inode, struct file *filp)
3905{
3906 struct ftrace_entries_info *info = filp->private_data;
3907
3908 kfree(info);
3909
3910 return 0;
3911}
3912
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003913static ssize_t
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07003914tracing_total_entries_read(struct file *filp, char __user *ubuf,
3915 size_t cnt, loff_t *ppos)
3916{
3917 struct trace_array *tr = filp->private_data;
3918 char buf[64];
3919 int r, cpu;
3920 unsigned long size = 0, expanded_size = 0;
3921
3922 mutex_lock(&trace_types_lock);
3923 for_each_tracing_cpu(cpu) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003924 size += tr->data[cpu]->entries >> 10;
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07003925 if (!ring_buffer_expanded)
3926 expanded_size += trace_buf_size >> 10;
3927 }
3928 if (ring_buffer_expanded)
3929 r = sprintf(buf, "%lu\n", size);
3930 else
3931 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3932 mutex_unlock(&trace_types_lock);
3933
3934 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3935}
3936
3937static ssize_t
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003938tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
3939 size_t cnt, loff_t *ppos)
3940{
3941 /*
3942 * There is no need to read what the user has written, this function
3943 * is just to make sure that there is no error when "echo" is used
3944 */
3945
3946 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003947
3948 return cnt;
3949}
3950
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003951static int
3952tracing_free_buffer_release(struct inode *inode, struct file *filp)
3953{
Steven Rostedtcf30cf62011-06-14 22:44:07 -04003954 /* disable tracing ? */
3955 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3956 tracing_off();
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003957 /* resize the ring buffer to 0 */
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003958 tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003959
3960 return 0;
3961}
3962
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003963static ssize_t
3964tracing_mark_write(struct file *filp, const char __user *ubuf,
3965 size_t cnt, loff_t *fpos)
3966{
Steven Rostedtd696b582011-09-22 11:50:27 -04003967 unsigned long addr = (unsigned long)ubuf;
3968 struct ring_buffer_event *event;
3969 struct ring_buffer *buffer;
3970 struct print_entry *entry;
3971 unsigned long irq_flags;
3972 struct page *pages[2];
Steven Rostedt6edb2a82012-05-11 23:28:49 -04003973 void *map_page[2];
Steven Rostedtd696b582011-09-22 11:50:27 -04003974 int nr_pages = 1;
3975 ssize_t written;
Steven Rostedtd696b582011-09-22 11:50:27 -04003976 int offset;
3977 int size;
3978 int len;
3979 int ret;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04003980 int i;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003981
Steven Rostedtc76f0692008-11-07 22:36:02 -05003982 if (tracing_disabled)
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003983 return -EINVAL;
3984
Mandeep Singh Baines5224c3a2012-09-07 18:12:19 -07003985 if (!(trace_flags & TRACE_ITER_MARKERS))
3986 return -EINVAL;
3987
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003988 if (cnt > TRACE_BUF_SIZE)
3989 cnt = TRACE_BUF_SIZE;
3990
Steven Rostedtd696b582011-09-22 11:50:27 -04003991 /*
3992 * Userspace is injecting traces into the kernel trace buffer.
3993 * We want to be as non intrusive as possible.
3994 * To do so, we do not want to allocate any special buffers
3995 * or take any locks, but instead write the userspace data
3996 * straight into the ring buffer.
3997 *
3998 * First we need to pin the userspace buffer into memory,
3999 * which, most likely it is, because it just referenced it.
4000 * But there's no guarantee that it is. By using get_user_pages_fast()
4001 * and kmap_atomic/kunmap_atomic() we can get access to the
4002 * pages directly. We then write the data directly into the
4003 * ring buffer.
4004 */
4005 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004006
Steven Rostedtd696b582011-09-22 11:50:27 -04004007 /* check if we cross pages */
4008 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4009 nr_pages = 2;
4010
4011 offset = addr & (PAGE_SIZE - 1);
4012 addr &= PAGE_MASK;
4013
4014 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4015 if (ret < nr_pages) {
4016 while (--ret >= 0)
4017 put_page(pages[ret]);
4018 written = -EFAULT;
4019 goto out;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004020 }
4021
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004022 for (i = 0; i < nr_pages; i++)
4023 map_page[i] = kmap_atomic(pages[i]);
Steven Rostedtd696b582011-09-22 11:50:27 -04004024
4025 local_save_flags(irq_flags);
4026 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4027 buffer = global_trace.buffer;
4028 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4029 irq_flags, preempt_count());
4030 if (!event) {
4031 /* Ring buffer disabled, return as if not open for write */
4032 written = -EBADF;
4033 goto out_unlock;
4034 }
4035
4036 entry = ring_buffer_event_data(event);
4037 entry->ip = _THIS_IP_;
4038
4039 if (nr_pages == 2) {
4040 len = PAGE_SIZE - offset;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004041 memcpy(&entry->buf, map_page[0] + offset, len);
4042 memcpy(&entry->buf[len], map_page[1], cnt - len);
Steven Rostedtd696b582011-09-22 11:50:27 -04004043 } else
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004044 memcpy(&entry->buf, map_page[0] + offset, cnt);
Steven Rostedtd696b582011-09-22 11:50:27 -04004045
4046 if (entry->buf[cnt - 1] != '\n') {
4047 entry->buf[cnt] = '\n';
4048 entry->buf[cnt + 1] = '\0';
4049 } else
4050 entry->buf[cnt] = '\0';
4051
Steven Rostedt7ffbd482012-10-11 12:14:25 -04004052 __buffer_unlock_commit(buffer, event);
Steven Rostedtd696b582011-09-22 11:50:27 -04004053
4054 written = cnt;
4055
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02004056 *fpos += written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004057
Steven Rostedtd696b582011-09-22 11:50:27 -04004058 out_unlock:
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004059 for (i = 0; i < nr_pages; i++){
4060 kunmap_atomic(map_page[i]);
4061 put_page(pages[i]);
4062 }
Steven Rostedtd696b582011-09-22 11:50:27 -04004063 out:
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02004064 return written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004065}
4066
Li Zefan13f16d22009-12-08 11:16:11 +08004067static int tracing_clock_show(struct seq_file *m, void *v)
Zhaolei5079f322009-08-25 16:12:56 +08004068{
Zhaolei5079f322009-08-25 16:12:56 +08004069 int i;
4070
4071 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
Li Zefan13f16d22009-12-08 11:16:11 +08004072 seq_printf(m,
Zhaolei5079f322009-08-25 16:12:56 +08004073 "%s%s%s%s", i ? " " : "",
4074 i == trace_clock_id ? "[" : "", trace_clocks[i].name,
4075 i == trace_clock_id ? "]" : "");
Li Zefan13f16d22009-12-08 11:16:11 +08004076 seq_putc(m, '\n');
Zhaolei5079f322009-08-25 16:12:56 +08004077
Li Zefan13f16d22009-12-08 11:16:11 +08004078 return 0;
Zhaolei5079f322009-08-25 16:12:56 +08004079}
4080
4081static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4082 size_t cnt, loff_t *fpos)
4083{
4084 char buf[64];
4085 const char *clockstr;
4086 int i;
4087
4088 if (cnt >= sizeof(buf))
4089 return -EINVAL;
4090
4091 if (copy_from_user(&buf, ubuf, cnt))
4092 return -EFAULT;
4093
4094 buf[cnt] = 0;
4095
4096 clockstr = strstrip(buf);
4097
4098 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4099 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4100 break;
4101 }
4102 if (i == ARRAY_SIZE(trace_clocks))
4103 return -EINVAL;
4104
4105 trace_clock_id = i;
4106
4107 mutex_lock(&trace_types_lock);
4108
4109 ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
4110 if (max_tr.buffer)
4111 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
4112
David Sharp60303ed2012-10-11 16:27:52 -07004113 /*
4114 * New clock may not be consistent with the previous clock.
4115 * Reset the buffer so that it doesn't have incomparable timestamps.
4116 */
4117 tracing_reset_online_cpus(&global_trace);
Steven Rostedt84c6cf02012-12-20 21:43:52 -05004118 tracing_reset_online_cpus(&max_tr);
David Sharp60303ed2012-10-11 16:27:52 -07004119
Zhaolei5079f322009-08-25 16:12:56 +08004120 mutex_unlock(&trace_types_lock);
4121
4122 *fpos += cnt;
4123
4124 return cnt;
4125}
4126
Li Zefan13f16d22009-12-08 11:16:11 +08004127static int tracing_clock_open(struct inode *inode, struct file *file)
4128{
4129 if (tracing_disabled)
4130 return -ENODEV;
4131 return single_open(file, tracing_clock_show, NULL);
4132}
4133
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004134#ifdef CONFIG_TRACER_SNAPSHOT
4135static int tracing_snapshot_open(struct inode *inode, struct file *file)
4136{
4137 struct trace_iterator *iter;
4138 int ret = 0;
4139
4140 if (file->f_mode & FMODE_READ) {
4141 iter = __tracing_open(inode, file, true);
4142 if (IS_ERR(iter))
4143 ret = PTR_ERR(iter);
4144 }
4145 return ret;
4146}
4147
4148static ssize_t
4149tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4150 loff_t *ppos)
4151{
4152 unsigned long val;
4153 int ret;
4154
4155 ret = tracing_update_buffers();
4156 if (ret < 0)
4157 return ret;
4158
4159 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4160 if (ret)
4161 return ret;
4162
4163 mutex_lock(&trace_types_lock);
4164
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05004165 if (current_trace->use_max_tr) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004166 ret = -EBUSY;
4167 goto out;
4168 }
4169
4170 switch (val) {
4171 case 0:
4172 if (current_trace->allocated_snapshot) {
4173 /* free spare buffer */
4174 ring_buffer_resize(max_tr.buffer, 1,
4175 RING_BUFFER_ALL_CPUS);
4176 set_buffer_entries(&max_tr, 1);
4177 tracing_reset_online_cpus(&max_tr);
4178 current_trace->allocated_snapshot = false;
4179 }
4180 break;
4181 case 1:
4182 if (!current_trace->allocated_snapshot) {
4183 /* allocate spare buffer */
4184 ret = resize_buffer_duplicate_size(&max_tr,
4185 &global_trace, RING_BUFFER_ALL_CPUS);
4186 if (ret < 0)
4187 break;
4188 current_trace->allocated_snapshot = true;
4189 }
4190
4191 local_irq_disable();
4192 /* Now, we're going to swap */
4193 update_max_tr(&global_trace, current, smp_processor_id());
4194 local_irq_enable();
4195 break;
4196 default:
4197 if (current_trace->allocated_snapshot)
4198 tracing_reset_online_cpus(&max_tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004199 break;
4200 }
4201
4202 if (ret >= 0) {
4203 *ppos += cnt;
4204 ret = cnt;
4205 }
4206out:
4207 mutex_unlock(&trace_types_lock);
4208 return ret;
4209}
4210#endif /* CONFIG_TRACER_SNAPSHOT */
4211
4212
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004213static const struct file_operations tracing_max_lat_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004214 .open = tracing_open_generic,
4215 .read = tracing_max_lat_read,
4216 .write = tracing_max_lat_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004217 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004218};
4219
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004220static const struct file_operations set_tracer_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004221 .open = tracing_open_generic,
4222 .read = tracing_set_trace_read,
4223 .write = tracing_set_trace_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004224 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004225};
4226
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004227static const struct file_operations tracing_pipe_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004228 .open = tracing_open_pipe,
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004229 .poll = tracing_poll_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004230 .read = tracing_read_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004231 .splice_read = tracing_splice_read_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004232 .release = tracing_release_pipe,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004233 .llseek = no_llseek,
Steven Rostedtb3806b42008-05-12 21:20:46 +02004234};
4235
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004236static const struct file_operations tracing_entries_fops = {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004237 .open = tracing_entries_open,
Steven Rostedta98a3c32008-05-12 21:20:59 +02004238 .read = tracing_entries_read,
4239 .write = tracing_entries_write,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004240 .release = tracing_entries_release,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004241 .llseek = generic_file_llseek,
Steven Rostedta98a3c32008-05-12 21:20:59 +02004242};
4243
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004244static const struct file_operations tracing_total_entries_fops = {
4245 .open = tracing_open_generic,
4246 .read = tracing_total_entries_read,
4247 .llseek = generic_file_llseek,
4248};
4249
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004250static const struct file_operations tracing_free_buffer_fops = {
4251 .write = tracing_free_buffer_write,
4252 .release = tracing_free_buffer_release,
4253};
4254
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004255static const struct file_operations tracing_mark_fops = {
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02004256 .open = tracing_open_generic,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004257 .write = tracing_mark_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004258 .llseek = generic_file_llseek,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004259};
4260
Zhaolei5079f322009-08-25 16:12:56 +08004261static const struct file_operations trace_clock_fops = {
Li Zefan13f16d22009-12-08 11:16:11 +08004262 .open = tracing_clock_open,
4263 .read = seq_read,
4264 .llseek = seq_lseek,
4265 .release = single_release,
Zhaolei5079f322009-08-25 16:12:56 +08004266 .write = tracing_clock_write,
4267};
4268
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004269#ifdef CONFIG_TRACER_SNAPSHOT
4270static const struct file_operations snapshot_fops = {
4271 .open = tracing_snapshot_open,
4272 .read = seq_read,
4273 .write = tracing_snapshot_write,
4274 .llseek = tracing_seek,
4275 .release = tracing_release,
4276};
4277#endif /* CONFIG_TRACER_SNAPSHOT */
4278
Steven Rostedt2cadf912008-12-01 22:20:19 -05004279struct ftrace_buffer_info {
4280 struct trace_array *tr;
4281 void *spare;
4282 int cpu;
4283 unsigned int read;
4284};
4285
4286static int tracing_buffers_open(struct inode *inode, struct file *filp)
4287{
4288 int cpu = (int)(long)inode->i_private;
4289 struct ftrace_buffer_info *info;
4290
4291 if (tracing_disabled)
4292 return -ENODEV;
4293
4294 info = kzalloc(sizeof(*info), GFP_KERNEL);
4295 if (!info)
4296 return -ENOMEM;
4297
4298 info->tr = &global_trace;
4299 info->cpu = cpu;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004300 info->spare = NULL;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004301 /* Force reading ring buffer for first read */
4302 info->read = (unsigned int)-1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004303
4304 filp->private_data = info;
4305
Lai Jiangshand1e7e022009-04-02 15:16:56 +08004306 return nonseekable_open(inode, filp);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004307}
4308
4309static ssize_t
4310tracing_buffers_read(struct file *filp, char __user *ubuf,
4311 size_t count, loff_t *ppos)
4312{
4313 struct ftrace_buffer_info *info = filp->private_data;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004314 ssize_t ret;
4315 size_t size;
4316
Steven Rostedt2dc5d122009-03-04 19:10:05 -05004317 if (!count)
4318 return 0;
4319
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004320 if (!info->spare)
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07004321 info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004322 if (!info->spare)
4323 return -ENOMEM;
4324
Steven Rostedt2cadf912008-12-01 22:20:19 -05004325 /* Do we have previous read data to read? */
4326 if (info->read < PAGE_SIZE)
4327 goto read;
4328
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004329 trace_access_lock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004330 ret = ring_buffer_read_page(info->tr->buffer,
4331 &info->spare,
4332 count,
4333 info->cpu, 0);
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004334 trace_access_unlock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004335 if (ret < 0)
4336 return 0;
4337
Steven Rostedt436fc282011-10-14 10:44:25 -04004338 info->read = 0;
4339
Steven Rostedt2cadf912008-12-01 22:20:19 -05004340read:
4341 size = PAGE_SIZE - info->read;
4342 if (size > count)
4343 size = count;
4344
4345 ret = copy_to_user(ubuf, info->spare + info->read, size);
Steven Rostedt2dc5d122009-03-04 19:10:05 -05004346 if (ret == size)
Steven Rostedt2cadf912008-12-01 22:20:19 -05004347 return -EFAULT;
Steven Rostedt2dc5d122009-03-04 19:10:05 -05004348 size -= ret;
4349
Steven Rostedt2cadf912008-12-01 22:20:19 -05004350 *ppos += size;
4351 info->read += size;
4352
4353 return size;
4354}
4355
4356static int tracing_buffers_release(struct inode *inode, struct file *file)
4357{
4358 struct ftrace_buffer_info *info = file->private_data;
4359
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004360 if (info->spare)
4361 ring_buffer_free_read_page(info->tr->buffer, info->spare);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004362 kfree(info);
4363
4364 return 0;
4365}
4366
4367struct buffer_ref {
4368 struct ring_buffer *buffer;
4369 void *page;
4370 int ref;
4371};
4372
4373static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4374 struct pipe_buffer *buf)
4375{
4376 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4377
4378 if (--ref->ref)
4379 return;
4380
4381 ring_buffer_free_read_page(ref->buffer, ref->page);
4382 kfree(ref);
4383 buf->private = 0;
4384}
4385
Steven Rostedt2cadf912008-12-01 22:20:19 -05004386static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4387 struct pipe_buffer *buf)
4388{
4389 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4390
4391 ref->ref++;
4392}
4393
4394/* Pipe buffer operations for a buffer. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08004395static const struct pipe_buf_operations buffer_pipe_buf_ops = {
Steven Rostedt2cadf912008-12-01 22:20:19 -05004396 .can_merge = 0,
4397 .map = generic_pipe_buf_map,
4398 .unmap = generic_pipe_buf_unmap,
4399 .confirm = generic_pipe_buf_confirm,
4400 .release = buffer_pipe_buf_release,
Masami Hiramatsud55cb6c2012-08-09 21:31:10 +09004401 .steal = generic_pipe_buf_steal,
Steven Rostedt2cadf912008-12-01 22:20:19 -05004402 .get = buffer_pipe_buf_get,
4403};
4404
4405/*
4406 * Callback from splice_to_pipe(), if we need to release some pages
4407 * at the end of the spd in case we error'ed out in filling the pipe.
4408 */
4409static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4410{
4411 struct buffer_ref *ref =
4412 (struct buffer_ref *)spd->partial[i].private;
4413
4414 if (--ref->ref)
4415 return;
4416
4417 ring_buffer_free_read_page(ref->buffer, ref->page);
4418 kfree(ref);
4419 spd->partial[i].private = 0;
4420}
4421
4422static ssize_t
4423tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4424 struct pipe_inode_info *pipe, size_t len,
4425 unsigned int flags)
4426{
4427 struct ftrace_buffer_info *info = file->private_data;
Jens Axboe35f3d142010-05-20 10:43:18 +02004428 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4429 struct page *pages_def[PIPE_DEF_BUFFERS];
Steven Rostedt2cadf912008-12-01 22:20:19 -05004430 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02004431 .pages = pages_def,
4432 .partial = partial_def,
Eric Dumazet047fe362012-06-12 15:24:40 +02004433 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt2cadf912008-12-01 22:20:19 -05004434 .flags = flags,
4435 .ops = &buffer_pipe_buf_ops,
4436 .spd_release = buffer_spd_release,
4437 };
4438 struct buffer_ref *ref;
Steven Rostedt93459c62009-04-29 00:23:13 -04004439 int entries, size, i;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004440 size_t ret;
4441
Jens Axboe35f3d142010-05-20 10:43:18 +02004442 if (splice_grow_spd(pipe, &spd))
4443 return -ENOMEM;
4444
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004445 if (*ppos & (PAGE_SIZE - 1)) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004446 ret = -EINVAL;
4447 goto out;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004448 }
4449
4450 if (len & (PAGE_SIZE - 1)) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004451 if (len < PAGE_SIZE) {
4452 ret = -EINVAL;
4453 goto out;
4454 }
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004455 len &= PAGE_MASK;
4456 }
4457
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004458 trace_access_lock(info->cpu);
Steven Rostedt93459c62009-04-29 00:23:13 -04004459 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4460
Jens Axboe35f3d142010-05-20 10:43:18 +02004461 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05004462 struct page *page;
4463 int r;
4464
4465 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4466 if (!ref)
4467 break;
4468
Steven Rostedt7267fa62009-04-29 00:16:21 -04004469 ref->ref = 1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004470 ref->buffer = info->tr->buffer;
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07004471 ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004472 if (!ref->page) {
4473 kfree(ref);
4474 break;
4475 }
4476
4477 r = ring_buffer_read_page(ref->buffer, &ref->page,
Steven Rostedtf2957f12009-04-29 00:26:30 -04004478 len, info->cpu, 1);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004479 if (r < 0) {
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07004480 ring_buffer_free_read_page(ref->buffer, ref->page);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004481 kfree(ref);
4482 break;
4483 }
4484
4485 /*
4486 * zero out any left over data, this is going to
4487 * user land.
4488 */
4489 size = ring_buffer_page_len(ref->page);
4490 if (size < PAGE_SIZE)
4491 memset(ref->page + size, 0, PAGE_SIZE - size);
4492
4493 page = virt_to_page(ref->page);
4494
4495 spd.pages[i] = page;
4496 spd.partial[i].len = PAGE_SIZE;
4497 spd.partial[i].offset = 0;
4498 spd.partial[i].private = (unsigned long)ref;
4499 spd.nr_pages++;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004500 *ppos += PAGE_SIZE;
Steven Rostedt93459c62009-04-29 00:23:13 -04004501
4502 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004503 }
4504
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004505 trace_access_unlock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004506 spd.nr_pages = i;
4507
4508 /* did we read anything? */
4509 if (!spd.nr_pages) {
4510 if (flags & SPLICE_F_NONBLOCK)
4511 ret = -EAGAIN;
4512 else
4513 ret = 0;
4514 /* TODO: block */
Jens Axboe35f3d142010-05-20 10:43:18 +02004515 goto out;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004516 }
4517
4518 ret = splice_to_pipe(pipe, &spd);
Eric Dumazet047fe362012-06-12 15:24:40 +02004519 splice_shrink_spd(&spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02004520out:
Steven Rostedt2cadf912008-12-01 22:20:19 -05004521 return ret;
4522}
4523
4524static const struct file_operations tracing_buffers_fops = {
4525 .open = tracing_buffers_open,
4526 .read = tracing_buffers_read,
4527 .release = tracing_buffers_release,
4528 .splice_read = tracing_buffers_splice_read,
4529 .llseek = no_llseek,
4530};
4531
Steven Rostedtc8d77182009-04-29 18:03:45 -04004532static ssize_t
4533tracing_stats_read(struct file *filp, char __user *ubuf,
4534 size_t count, loff_t *ppos)
4535{
4536 unsigned long cpu = (unsigned long)filp->private_data;
4537 struct trace_array *tr = &global_trace;
4538 struct trace_seq *s;
4539 unsigned long cnt;
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07004540 unsigned long long t;
4541 unsigned long usec_rem;
Steven Rostedtc8d77182009-04-29 18:03:45 -04004542
Li Zefane4f2d102009-06-15 10:57:28 +08004543 s = kmalloc(sizeof(*s), GFP_KERNEL);
Steven Rostedtc8d77182009-04-29 18:03:45 -04004544 if (!s)
Roel Kluina6463652009-11-11 22:26:35 +01004545 return -ENOMEM;
Steven Rostedtc8d77182009-04-29 18:03:45 -04004546
4547 trace_seq_init(s);
4548
4549 cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4550 trace_seq_printf(s, "entries: %ld\n", cnt);
4551
4552 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4553 trace_seq_printf(s, "overrun: %ld\n", cnt);
4554
4555 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4556 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4557
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07004558 cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4559 trace_seq_printf(s, "bytes: %ld\n", cnt);
4560
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08004561 if (trace_clocks[trace_clock_id].in_ns) {
4562 /* local or global for trace_clock */
4563 t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4564 usec_rem = do_div(t, USEC_PER_SEC);
4565 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
4566 t, usec_rem);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07004567
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08004568 t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4569 usec_rem = do_div(t, USEC_PER_SEC);
4570 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4571 } else {
4572 /* counter or tsc mode for trace_clock */
4573 trace_seq_printf(s, "oldest event ts: %llu\n",
4574 ring_buffer_oldest_event_ts(tr->buffer, cpu));
4575
4576 trace_seq_printf(s, "now ts: %llu\n",
4577 ring_buffer_time_stamp(tr->buffer, cpu));
4578 }
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07004579
Slava Pestov884bfe82011-07-15 14:23:58 -07004580 cnt = ring_buffer_dropped_events_cpu(tr->buffer, cpu);
4581 trace_seq_printf(s, "dropped events: %ld\n", cnt);
4582
Steven Rostedt (Red Hat)ad964702013-01-29 17:45:49 -05004583 cnt = ring_buffer_read_events_cpu(tr->buffer, cpu);
4584 trace_seq_printf(s, "read events: %ld\n", cnt);
4585
Steven Rostedtc8d77182009-04-29 18:03:45 -04004586 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4587
4588 kfree(s);
4589
4590 return count;
4591}
4592
4593static const struct file_operations tracing_stats_fops = {
4594 .open = tracing_open_generic,
4595 .read = tracing_stats_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004596 .llseek = generic_file_llseek,
Steven Rostedtc8d77182009-04-29 18:03:45 -04004597};
4598
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004599#ifdef CONFIG_DYNAMIC_FTRACE
4600
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004601int __weak ftrace_arch_read_dyn_info(char *buf, int size)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004602{
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004603 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004604}
4605
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004606static ssize_t
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004607tracing_read_dyn_info(struct file *filp, char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004608 size_t cnt, loff_t *ppos)
4609{
Steven Rostedta26a2a22008-10-31 00:03:22 -04004610 static char ftrace_dyn_info_buffer[1024];
4611 static DEFINE_MUTEX(dyn_info_mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004612 unsigned long *p = filp->private_data;
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004613 char *buf = ftrace_dyn_info_buffer;
Steven Rostedta26a2a22008-10-31 00:03:22 -04004614 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004615 int r;
4616
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004617 mutex_lock(&dyn_info_mutex);
4618 r = sprintf(buf, "%ld ", *p);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004619
Steven Rostedta26a2a22008-10-31 00:03:22 -04004620 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004621 buf[r++] = '\n';
4622
4623 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4624
4625 mutex_unlock(&dyn_info_mutex);
4626
4627 return r;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004628}
4629
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004630static const struct file_operations tracing_dyn_info_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004631 .open = tracing_open_generic,
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004632 .read = tracing_read_dyn_info,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004633 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004634};
4635#endif
4636
4637static struct dentry *d_tracer;
4638
4639struct dentry *tracing_init_dentry(void)
4640{
4641 static int once;
4642
4643 if (d_tracer)
4644 return d_tracer;
4645
Frederic Weisbecker3e1f60b2009-03-22 23:10:45 +01004646 if (!debugfs_initialized())
4647 return NULL;
4648
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004649 d_tracer = debugfs_create_dir("tracing", NULL);
4650
4651 if (!d_tracer && !once) {
4652 once = 1;
4653 pr_warning("Could not create debugfs directory 'tracing'\n");
4654 return NULL;
4655 }
4656
4657 return d_tracer;
4658}
4659
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004660static struct dentry *d_percpu;
4661
Josh Triplettb736f482012-11-18 21:27:45 -08004662static struct dentry *tracing_dentry_percpu(void)
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004663{
4664 static int once;
4665 struct dentry *d_tracer;
4666
4667 if (d_percpu)
4668 return d_percpu;
4669
4670 d_tracer = tracing_init_dentry();
4671
4672 if (!d_tracer)
4673 return NULL;
4674
4675 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4676
4677 if (!d_percpu && !once) {
4678 once = 1;
4679 pr_warning("Could not create debugfs directory 'per_cpu'\n");
4680 return NULL;
4681 }
4682
4683 return d_percpu;
4684}
4685
4686static void tracing_init_debugfs_percpu(long cpu)
4687{
4688 struct dentry *d_percpu = tracing_dentry_percpu();
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004689 struct dentry *d_cpu;
Steven Rostedtdd49a382010-10-20 21:51:26 -04004690 char cpu_dir[30]; /* 30 characters should be more than enough */
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004691
Namhyung Kim0a3d7ce2012-04-23 10:11:57 +09004692 if (!d_percpu)
4693 return;
4694
Steven Rostedtdd49a382010-10-20 21:51:26 -04004695 snprintf(cpu_dir, 30, "cpu%ld", cpu);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004696 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4697 if (!d_cpu) {
4698 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4699 return;
4700 }
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004701
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004702 /* per cpu trace_pipe */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004703 trace_create_file("trace_pipe", 0444, d_cpu,
4704 (void *) cpu, &tracing_pipe_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004705
4706 /* per cpu trace */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004707 trace_create_file("trace", 0644, d_cpu,
4708 (void *) cpu, &tracing_fops);
Steven Rostedt7f96f932009-03-13 00:37:42 -04004709
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004710 trace_create_file("trace_pipe_raw", 0444, d_cpu,
4711 (void *) cpu, &tracing_buffers_fops);
Steven Rostedtc8d77182009-04-29 18:03:45 -04004712
4713 trace_create_file("stats", 0444, d_cpu,
4714 (void *) cpu, &tracing_stats_fops);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004715
4716 trace_create_file("buffer_size_kb", 0444, d_cpu,
4717 (void *) cpu, &tracing_entries_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004718}
4719
Steven Rostedt60a11772008-05-12 21:20:44 +02004720#ifdef CONFIG_FTRACE_SELFTEST
4721/* Let selftest have access to static functions in this file */
4722#include "trace_selftest.c"
4723#endif
4724
Steven Rostedt577b7852009-02-26 23:43:05 -05004725struct trace_option_dentry {
4726 struct tracer_opt *opt;
4727 struct tracer_flags *flags;
4728 struct dentry *entry;
4729};
4730
4731static ssize_t
4732trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4733 loff_t *ppos)
4734{
4735 struct trace_option_dentry *topt = filp->private_data;
4736 char *buf;
4737
4738 if (topt->flags->val & topt->opt->bit)
4739 buf = "1\n";
4740 else
4741 buf = "0\n";
4742
4743 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4744}
4745
4746static ssize_t
4747trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4748 loff_t *ppos)
4749{
4750 struct trace_option_dentry *topt = filp->private_data;
4751 unsigned long val;
Steven Rostedt577b7852009-02-26 23:43:05 -05004752 int ret;
4753
Peter Huewe22fe9b52011-06-07 21:58:27 +02004754 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4755 if (ret)
Steven Rostedt577b7852009-02-26 23:43:05 -05004756 return ret;
4757
Li Zefan8d18eaa2009-12-08 11:17:06 +08004758 if (val != 0 && val != 1)
Steven Rostedt577b7852009-02-26 23:43:05 -05004759 return -EINVAL;
Li Zefan8d18eaa2009-12-08 11:17:06 +08004760
4761 if (!!(topt->flags->val & topt->opt->bit) != val) {
4762 mutex_lock(&trace_types_lock);
4763 ret = __set_tracer_option(current_trace, topt->flags,
Steven Rostedtc757bea2009-12-21 22:35:16 -05004764 topt->opt, !val);
Li Zefan8d18eaa2009-12-08 11:17:06 +08004765 mutex_unlock(&trace_types_lock);
4766 if (ret)
4767 return ret;
Steven Rostedt577b7852009-02-26 23:43:05 -05004768 }
4769
4770 *ppos += cnt;
4771
4772 return cnt;
4773}
4774
4775
4776static const struct file_operations trace_options_fops = {
4777 .open = tracing_open_generic,
4778 .read = trace_options_read,
4779 .write = trace_options_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004780 .llseek = generic_file_llseek,
Steven Rostedt577b7852009-02-26 23:43:05 -05004781};
4782
Steven Rostedta8259072009-02-26 22:19:12 -05004783static ssize_t
4784trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4785 loff_t *ppos)
4786{
4787 long index = (long)filp->private_data;
4788 char *buf;
4789
4790 if (trace_flags & (1 << index))
4791 buf = "1\n";
4792 else
4793 buf = "0\n";
4794
4795 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4796}
4797
4798static ssize_t
4799trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4800 loff_t *ppos)
4801{
4802 long index = (long)filp->private_data;
Steven Rostedta8259072009-02-26 22:19:12 -05004803 unsigned long val;
4804 int ret;
4805
Peter Huewe22fe9b52011-06-07 21:58:27 +02004806 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4807 if (ret)
Steven Rostedta8259072009-02-26 22:19:12 -05004808 return ret;
4809
Zhaoleif2d84b62009-08-07 18:55:48 +08004810 if (val != 0 && val != 1)
Steven Rostedta8259072009-02-26 22:19:12 -05004811 return -EINVAL;
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04004812
4813 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04004814 ret = set_tracer_flag(1 << index, val);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04004815 mutex_unlock(&trace_types_lock);
Steven Rostedta8259072009-02-26 22:19:12 -05004816
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04004817 if (ret < 0)
4818 return ret;
4819
Steven Rostedta8259072009-02-26 22:19:12 -05004820 *ppos += cnt;
4821
4822 return cnt;
4823}
4824
Steven Rostedta8259072009-02-26 22:19:12 -05004825static const struct file_operations trace_options_core_fops = {
4826 .open = tracing_open_generic,
4827 .read = trace_options_core_read,
4828 .write = trace_options_core_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004829 .llseek = generic_file_llseek,
Steven Rostedta8259072009-02-26 22:19:12 -05004830};
4831
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004832struct dentry *trace_create_file(const char *name,
Al Virof4ae40a2011-07-24 04:33:43 -04004833 umode_t mode,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004834 struct dentry *parent,
4835 void *data,
4836 const struct file_operations *fops)
4837{
4838 struct dentry *ret;
4839
4840 ret = debugfs_create_file(name, mode, parent, data, fops);
4841 if (!ret)
4842 pr_warning("Could not create debugfs '%s' entry\n", name);
4843
4844 return ret;
4845}
4846
4847
Steven Rostedta8259072009-02-26 22:19:12 -05004848static struct dentry *trace_options_init_dentry(void)
4849{
4850 struct dentry *d_tracer;
4851 static struct dentry *t_options;
4852
4853 if (t_options)
4854 return t_options;
4855
4856 d_tracer = tracing_init_dentry();
4857 if (!d_tracer)
4858 return NULL;
4859
4860 t_options = debugfs_create_dir("options", d_tracer);
4861 if (!t_options) {
4862 pr_warning("Could not create debugfs directory 'options'\n");
4863 return NULL;
4864 }
4865
4866 return t_options;
4867}
4868
Steven Rostedt577b7852009-02-26 23:43:05 -05004869static void
4870create_trace_option_file(struct trace_option_dentry *topt,
4871 struct tracer_flags *flags,
4872 struct tracer_opt *opt)
4873{
4874 struct dentry *t_options;
Steven Rostedt577b7852009-02-26 23:43:05 -05004875
4876 t_options = trace_options_init_dentry();
4877 if (!t_options)
4878 return;
4879
4880 topt->flags = flags;
4881 topt->opt = opt;
4882
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004883 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05004884 &trace_options_fops);
4885
Steven Rostedt577b7852009-02-26 23:43:05 -05004886}
4887
4888static struct trace_option_dentry *
4889create_trace_option_files(struct tracer *tracer)
4890{
4891 struct trace_option_dentry *topts;
4892 struct tracer_flags *flags;
4893 struct tracer_opt *opts;
4894 int cnt;
4895
4896 if (!tracer)
4897 return NULL;
4898
4899 flags = tracer->flags;
4900
4901 if (!flags || !flags->opts)
4902 return NULL;
4903
4904 opts = flags->opts;
4905
4906 for (cnt = 0; opts[cnt].name; cnt++)
4907 ;
4908
Steven Rostedt0cfe8242009-02-27 10:51:10 -05004909 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
Steven Rostedt577b7852009-02-26 23:43:05 -05004910 if (!topts)
4911 return NULL;
4912
4913 for (cnt = 0; opts[cnt].name; cnt++)
4914 create_trace_option_file(&topts[cnt], flags,
4915 &opts[cnt]);
4916
4917 return topts;
4918}
4919
4920static void
4921destroy_trace_option_files(struct trace_option_dentry *topts)
4922{
4923 int cnt;
4924
4925 if (!topts)
4926 return;
4927
4928 for (cnt = 0; topts[cnt].opt; cnt++) {
4929 if (topts[cnt].entry)
4930 debugfs_remove(topts[cnt].entry);
4931 }
4932
4933 kfree(topts);
4934}
4935
Steven Rostedta8259072009-02-26 22:19:12 -05004936static struct dentry *
4937create_trace_option_core_file(const char *option, long index)
4938{
4939 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05004940
4941 t_options = trace_options_init_dentry();
4942 if (!t_options)
4943 return NULL;
4944
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004945 return trace_create_file(option, 0644, t_options, (void *)index,
Steven Rostedta8259072009-02-26 22:19:12 -05004946 &trace_options_core_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05004947}
4948
4949static __init void create_trace_options_dir(void)
4950{
4951 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05004952 int i;
4953
4954 t_options = trace_options_init_dentry();
4955 if (!t_options)
4956 return;
4957
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004958 for (i = 0; trace_options[i]; i++)
4959 create_trace_option_core_file(trace_options[i], i);
Steven Rostedta8259072009-02-26 22:19:12 -05004960}
4961
Steven Rostedt499e5472012-02-22 15:50:28 -05004962static ssize_t
4963rb_simple_read(struct file *filp, char __user *ubuf,
4964 size_t cnt, loff_t *ppos)
4965{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04004966 struct trace_array *tr = filp->private_data;
4967 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt499e5472012-02-22 15:50:28 -05004968 char buf[64];
4969 int r;
4970
4971 if (buffer)
4972 r = ring_buffer_record_is_on(buffer);
4973 else
4974 r = 0;
4975
4976 r = sprintf(buf, "%d\n", r);
4977
4978 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4979}
4980
4981static ssize_t
4982rb_simple_write(struct file *filp, const char __user *ubuf,
4983 size_t cnt, loff_t *ppos)
4984{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04004985 struct trace_array *tr = filp->private_data;
4986 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt499e5472012-02-22 15:50:28 -05004987 unsigned long val;
4988 int ret;
4989
4990 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4991 if (ret)
4992 return ret;
4993
4994 if (buffer) {
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05004995 mutex_lock(&trace_types_lock);
4996 if (val) {
Steven Rostedt499e5472012-02-22 15:50:28 -05004997 ring_buffer_record_on(buffer);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05004998 if (current_trace->start)
4999 current_trace->start(tr);
5000 } else {
Steven Rostedt499e5472012-02-22 15:50:28 -05005001 ring_buffer_record_off(buffer);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05005002 if (current_trace->stop)
5003 current_trace->stop(tr);
5004 }
5005 mutex_unlock(&trace_types_lock);
Steven Rostedt499e5472012-02-22 15:50:28 -05005006 }
5007
5008 (*ppos)++;
5009
5010 return cnt;
5011}
5012
5013static const struct file_operations rb_simple_fops = {
5014 .open = tracing_open_generic,
5015 .read = rb_simple_read,
5016 .write = rb_simple_write,
5017 .llseek = default_llseek,
5018};
5019
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01005020static __init int tracer_init_debugfs(void)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005021{
5022 struct dentry *d_tracer;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005023 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005024
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08005025 trace_access_lock_init();
5026
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005027 d_tracer = tracing_init_dentry();
5028
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005029 trace_create_file("trace_options", 0644, d_tracer,
5030 NULL, &tracing_iter_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005031
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005032 trace_create_file("tracing_cpumask", 0644, d_tracer,
5033 NULL, &tracing_cpumask_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05005034
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005035 trace_create_file("trace", 0644, d_tracer,
5036 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
Ingo Molnarc7078de2008-05-12 21:20:52 +02005037
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005038 trace_create_file("available_tracers", 0444, d_tracer,
5039 &global_trace, &show_traces_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005040
Li Zefan339ae5d2009-04-17 10:34:30 +08005041 trace_create_file("current_tracer", 0644, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005042 &global_trace, &set_tracer_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005043
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04005044#ifdef CONFIG_TRACER_MAX_TRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005045 trace_create_file("tracing_max_latency", 0644, d_tracer,
5046 &tracing_max_latency, &tracing_max_lat_fops);
Tim Bird0e950172010-02-25 15:36:43 -08005047#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005048
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005049 trace_create_file("tracing_thresh", 0644, d_tracer,
5050 &tracing_thresh, &tracing_max_lat_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005051
Li Zefan339ae5d2009-04-17 10:34:30 +08005052 trace_create_file("README", 0444, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005053 NULL, &tracing_readme_fops);
Ingo Molnar7bd2f242008-05-12 21:20:45 +02005054
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005055 trace_create_file("trace_pipe", 0444, d_tracer,
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005056 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005057
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005058 trace_create_file("buffer_size_kb", 0644, d_tracer,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08005059 (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
Steven Rostedta98a3c32008-05-12 21:20:59 +02005060
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005061 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
5062 &global_trace, &tracing_total_entries_fops);
5063
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005064 trace_create_file("free_buffer", 0644, d_tracer,
5065 &global_trace, &tracing_free_buffer_fops);
5066
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005067 trace_create_file("trace_marker", 0220, d_tracer,
5068 NULL, &tracing_mark_fops);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005069
Avadh Patel69abe6a2009-04-10 16:04:48 -04005070 trace_create_file("saved_cmdlines", 0444, d_tracer,
5071 NULL, &tracing_saved_cmdlines_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005072
Zhaolei5079f322009-08-25 16:12:56 +08005073 trace_create_file("trace_clock", 0644, d_tracer, NULL,
5074 &trace_clock_fops);
5075
Steven Rostedt499e5472012-02-22 15:50:28 -05005076 trace_create_file("tracing_on", 0644, d_tracer,
Steven Rostedt348f0fc2012-04-16 15:41:28 -04005077 &global_trace, &rb_simple_fops);
Steven Rostedt499e5472012-02-22 15:50:28 -05005078
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005079#ifdef CONFIG_DYNAMIC_FTRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005080 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
5081 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005082#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005083
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005084#ifdef CONFIG_TRACER_SNAPSHOT
5085 trace_create_file("snapshot", 0644, d_tracer,
5086 (void *) TRACE_PIPE_ALL_CPU, &snapshot_fops);
5087#endif
5088
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005089 create_trace_options_dir();
5090
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005091 for_each_tracing_cpu(cpu)
5092 tracing_init_debugfs_percpu(cpu);
5093
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01005094 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005095}
5096
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005097static int trace_panic_handler(struct notifier_block *this,
5098 unsigned long event, void *unused)
5099{
Steven Rostedt944ac422008-10-23 19:26:08 -04005100 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005101 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005102 return NOTIFY_OK;
5103}
5104
5105static struct notifier_block trace_panic_notifier = {
5106 .notifier_call = trace_panic_handler,
5107 .next = NULL,
5108 .priority = 150 /* priority: INT_MAX >= x >= 0 */
5109};
5110
5111static int trace_die_handler(struct notifier_block *self,
5112 unsigned long val,
5113 void *data)
5114{
5115 switch (val) {
5116 case DIE_OOPS:
Steven Rostedt944ac422008-10-23 19:26:08 -04005117 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005118 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005119 break;
5120 default:
5121 break;
5122 }
5123 return NOTIFY_OK;
5124}
5125
5126static struct notifier_block trace_die_notifier = {
5127 .notifier_call = trace_die_handler,
5128 .priority = 200
5129};
5130
5131/*
5132 * printk is set to max of 1024, we really don't need it that big.
5133 * Nothing should be printing 1000 characters anyway.
5134 */
5135#define TRACE_MAX_PRINT 1000
5136
5137/*
5138 * Define here KERN_TRACE so that we have one place to modify
5139 * it if we decide to change what log level the ftrace dump
5140 * should be at.
5141 */
Steven Rostedt428aee12009-01-14 12:24:42 -05005142#define KERN_TRACE KERN_EMERG
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005143
Jason Wessel955b61e2010-08-05 09:22:23 -05005144void
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005145trace_printk_seq(struct trace_seq *s)
5146{
5147 /* Probably should print a warning here. */
5148 if (s->len >= 1000)
5149 s->len = 1000;
5150
5151 /* should be zero ended, but we are paranoid. */
5152 s->buffer[s->len] = 0;
5153
5154 printk(KERN_TRACE "%s", s->buffer);
5155
Steven Rostedtf9520752009-03-02 14:04:40 -05005156 trace_seq_init(s);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005157}
5158
Jason Wessel955b61e2010-08-05 09:22:23 -05005159void trace_init_global_iter(struct trace_iterator *iter)
5160{
5161 iter->tr = &global_trace;
5162 iter->trace = current_trace;
5163 iter->cpu_file = TRACE_PIPE_ALL_CPU;
5164}
5165
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005166static void
5167__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005168{
Thomas Gleixner445c8952009-12-02 19:49:50 +01005169 static arch_spinlock_t ftrace_dump_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01005170 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005171 /* use static because iter can be a bit big for the stack */
5172 static struct trace_iterator iter;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005173 unsigned int old_userobj;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005174 static int dump_ran;
Steven Rostedtd7690412008-10-01 00:29:53 -04005175 unsigned long flags;
5176 int cnt = 0, cpu;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005177
5178 /* only one dump */
Steven Rostedtcd891ae2009-04-28 11:39:34 -04005179 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01005180 arch_spin_lock(&ftrace_dump_lock);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005181 if (dump_ran)
5182 goto out;
5183
5184 dump_ran = 1;
5185
Steven Rostedt0ee6b6c2009-01-14 14:50:19 -05005186 tracing_off();
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005187
Steven Rostedte0a413f2011-09-29 21:26:16 -04005188 /* Did function tracer already get disabled? */
5189 if (ftrace_is_dead()) {
5190 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
5191 printk("# MAY BE MISSING FUNCTION EVENTS\n");
5192 }
5193
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005194 if (disable_tracing)
5195 ftrace_kill();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005196
Jovi Zhang38dbe0b2013-01-25 18:03:07 +08005197 /* Simulate the iterator */
Jason Wessel955b61e2010-08-05 09:22:23 -05005198 trace_init_global_iter(&iter);
5199
Steven Rostedtd7690412008-10-01 00:29:53 -04005200 for_each_tracing_cpu(cpu) {
Jason Wessel955b61e2010-08-05 09:22:23 -05005201 atomic_inc(&iter.tr->data[cpu]->disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -04005202 }
5203
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005204 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
5205
Török Edwinb54d3de2008-11-22 13:28:48 +02005206 /* don't look at user memory in panic mode */
5207 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
5208
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005209 switch (oops_dump_mode) {
5210 case DUMP_ALL:
5211 iter.cpu_file = TRACE_PIPE_ALL_CPU;
5212 break;
5213 case DUMP_ORIG:
5214 iter.cpu_file = raw_smp_processor_id();
5215 break;
5216 case DUMP_NONE:
5217 goto out_enable;
5218 default:
5219 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
5220 iter.cpu_file = TRACE_PIPE_ALL_CPU;
5221 }
5222
5223 printk(KERN_TRACE "Dumping ftrace buffer:\n");
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005224
5225 /*
5226 * We need to stop all tracing on all CPUS to read the
5227 * the next buffer. This is a bit expensive, but is
5228 * not done often. We fill all what we can read,
5229 * and then release the locks again.
5230 */
5231
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005232 while (!trace_empty(&iter)) {
5233
5234 if (!cnt)
5235 printk(KERN_TRACE "---------------------------------\n");
5236
5237 cnt++;
5238
5239 /* reset all but tr, trace, and overruns */
5240 memset(&iter.seq, 0,
5241 sizeof(struct trace_iterator) -
5242 offsetof(struct trace_iterator, seq));
5243 iter.iter_flags |= TRACE_FILE_LAT_FMT;
5244 iter.pos = -1;
5245
Jason Wessel955b61e2010-08-05 09:22:23 -05005246 if (trace_find_next_entry_inc(&iter) != NULL) {
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08005247 int ret;
5248
5249 ret = print_trace_line(&iter);
5250 if (ret != TRACE_TYPE_NO_CONSUME)
5251 trace_consume(&iter);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005252 }
Steven Rostedtb892e5c2012-03-01 22:06:48 -05005253 touch_nmi_watchdog();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005254
5255 trace_printk_seq(&iter.seq);
5256 }
5257
5258 if (!cnt)
5259 printk(KERN_TRACE " (ftrace buffer empty)\n");
5260 else
5261 printk(KERN_TRACE "---------------------------------\n");
5262
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005263 out_enable:
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005264 /* Re-enable tracing if requested */
5265 if (!disable_tracing) {
5266 trace_flags |= old_userobj;
5267
5268 for_each_tracing_cpu(cpu) {
Jason Wessel955b61e2010-08-05 09:22:23 -05005269 atomic_dec(&iter.tr->data[cpu]->disabled);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005270 }
5271 tracing_on();
5272 }
5273
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005274 out:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01005275 arch_spin_unlock(&ftrace_dump_lock);
Steven Rostedtcd891ae2009-04-28 11:39:34 -04005276 local_irq_restore(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005277}
5278
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005279/* By default: disable tracing after the dump */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005280void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005281{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005282 __ftrace_dump(true, oops_dump_mode);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005283}
Paul E. McKenneya8eecf22011-10-02 11:01:15 -07005284EXPORT_SYMBOL_GPL(ftrace_dump);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005285
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005286__init static int tracer_alloc_buffers(void)
5287{
Steven Rostedt73c51622009-03-11 13:42:01 -04005288 int ring_buf_size;
David Sharp750912f2010-12-08 13:46:47 -08005289 enum ring_buffer_flags rb_flags;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005290 int i;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305291 int ret = -ENOMEM;
5292
David Sharp750912f2010-12-08 13:46:47 -08005293
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305294 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
5295 goto out;
5296
5297 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
5298 goto out_free_buffer_mask;
5299
Steven Rostedt07d777f2011-09-22 14:01:55 -04005300 /* Only allocate trace_printk buffers if a trace_printk exists */
5301 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
Steven Rostedt81698832012-10-11 10:15:05 -04005302 /* Must be called before global_trace.buffer is allocated */
Steven Rostedt07d777f2011-09-22 14:01:55 -04005303 trace_printk_init_buffers();
5304
Steven Rostedt73c51622009-03-11 13:42:01 -04005305 /* To save memory, keep the ring buffer size to its minimum */
5306 if (ring_buffer_expanded)
5307 ring_buf_size = trace_buf_size;
5308 else
5309 ring_buf_size = 1;
5310
David Sharp750912f2010-12-08 13:46:47 -08005311 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5312
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305313 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
5314 cpumask_copy(tracing_cpumask, cpu_all_mask);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005315
Steven Rostedtab464282008-05-12 21:21:00 +02005316 /* TODO: make the number of buffers hot pluggable with CPUS */
David Sharp750912f2010-12-08 13:46:47 -08005317 global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005318 if (!global_trace.buffer) {
5319 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
5320 WARN_ON(1);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305321 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005322 }
Steven Rostedt499e5472012-02-22 15:50:28 -05005323 if (global_trace.buffer_disabled)
5324 tracing_off();
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005325
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305326
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005327#ifdef CONFIG_TRACER_MAX_TRACE
David Sharp750912f2010-12-08 13:46:47 -08005328 max_tr.buffer = ring_buffer_alloc(1, rb_flags);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005329 if (!max_tr.buffer) {
5330 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
5331 WARN_ON(1);
5332 ring_buffer_free(global_trace.buffer);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305333 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005334 }
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005335#endif
5336
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02005337 /* Allocate the first page for all buffers */
Steven Rostedtab464282008-05-12 21:21:00 +02005338 for_each_tracing_cpu(i) {
jolsa@redhat.com566b0aa2009-07-16 21:44:26 +02005339 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
Tejun Heo9705f692009-10-29 22:34:13 +09005340 max_tr.data[i] = &per_cpu(max_tr_data, i);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005341 }
5342
Vaibhav Nagarnaika591c732012-05-03 10:40:34 -07005343 set_buffer_entries(&global_trace,
5344 ring_buffer_size(global_trace.buffer, 0));
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08005345#ifdef CONFIG_TRACER_MAX_TRACE
5346 set_buffer_entries(&max_tr, 1);
5347#endif
5348
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005349 trace_init_cmdlines();
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04005350 init_irq_work(&trace_work_wakeup, trace_wake_up);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005351
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02005352 register_tracer(&nop_trace);
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05005353
Steven Rostedt60a11772008-05-12 21:20:44 +02005354 /* All seems OK, enable tracing */
5355 tracing_disabled = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005356
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005357 atomic_notifier_chain_register(&panic_notifier_list,
5358 &trace_panic_notifier);
5359
5360 register_die_notifier(&trace_die_notifier);
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01005361
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04005362 while (trace_boot_options) {
5363 char *option;
5364
5365 option = strsep(&trace_boot_options, ",");
5366 trace_set_options(option);
5367 }
5368
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01005369 return 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005370
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305371out_free_cpumask:
5372 free_cpumask_var(tracing_cpumask);
5373out_free_buffer_mask:
5374 free_cpumask_var(tracing_buffer_mask);
5375out:
5376 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005377}
Steven Rostedtb2821ae2009-02-02 21:38:32 -05005378
5379__init static int clear_boot_tracer(void)
5380{
5381 /*
5382 * The default tracer at boot buffer is an init section.
5383 * This function is called in lateinit. If we did not
5384 * find the boot tracer, then clear it out, to prevent
5385 * later registration from accessing the buffer that is
5386 * about to be freed.
5387 */
5388 if (!default_bootup_tracer)
5389 return 0;
5390
5391 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5392 default_bootup_tracer);
5393 default_bootup_tracer = NULL;
5394
5395 return 0;
5396}
5397
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01005398early_initcall(tracer_alloc_buffers);
5399fs_initcall(tracer_init_debugfs);
Steven Rostedtb2821ae2009-02-02 21:38:32 -05005400late_initcall(clear_boot_tracer);