blob: d6736b93dc2aed0a1934165cf945012306a2f516 [file] [log] [blame]
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
Steven Rostedt2cadf912008-12-01 22:20:19 -050014#include <linux/ring_buffer.h>
Sam Ravnborg273b2812009-10-18 00:52:28 +020015#include <generated/utsrelease.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050016#include <linux/stacktrace.h>
17#include <linux/writeback.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020018#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
Alexey Dobriyan405f5572009-07-11 22:08:37 +040020#include <linux/smp_lock.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040021#include <linux/notifier.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050022#include <linux/irqflags.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020023#include <linux/debugfs.h>
Steven Rostedt4c11d7a2008-05-12 21:20:43 +020024#include <linux/pagemap.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020025#include <linux/hardirq.h>
26#include <linux/linkage.h>
27#include <linux/uaccess.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050028#include <linux/kprobes.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020029#include <linux/ftrace.h>
30#include <linux/module.h>
31#include <linux/percpu.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050032#include <linux/splice.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040033#include <linux/kdebug.h>
Frederic Weisbecker5f0c6c02009-03-27 14:22:10 +010034#include <linux/string.h>
Lai Jiangshan7e53bd42010-01-06 20:08:50 +080035#include <linux/rwsem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090036#include <linux/slab.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020037#include <linux/ctype.h>
38#include <linux/init.h>
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +020039#include <linux/poll.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020040#include <linux/fs.h>
Ingo Molnar86387f72008-05-12 21:20:51 +020041
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020042#include "trace.h"
Steven Rostedtf0868d12008-12-23 23:24:12 -050043#include "trace_output.h"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020044
Steven Rostedt3928a8a2008-09-29 23:02:41 -040045#define TRACE_BUFFER_FLAGS (RB_FL_OVERWRITE)
46
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010047/*
Steven Rostedt73c51622009-03-11 13:42:01 -040048 * On boot up, the ring buffer is set to the minimum size, so that
49 * we do not waste memory on systems that are not using tracing.
50 */
Li Zefan020e5f82009-07-01 10:47:05 +080051int ring_buffer_expanded;
Steven Rostedt73c51622009-03-11 13:42:01 -040052
53/*
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010054 * We need to change this state when a selftest is running.
Frederic Weisbeckerff325042008-12-04 23:47:35 +010055 * A selftest will lurk into the ring-buffer to count the
56 * entries inserted during the selftest although some concurrent
Ingo Molnar5e1607a2009-03-05 10:24:48 +010057 * insertions into the ring-buffer such as trace_printk could occurred
Frederic Weisbeckerff325042008-12-04 23:47:35 +010058 * at the same time, giving false positive or negative results.
59 */
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010060static bool __read_mostly tracing_selftest_running;
Frederic Weisbeckerff325042008-12-04 23:47:35 +010061
Steven Rostedtb2821ae2009-02-02 21:38:32 -050062/*
63 * If a tracer is running, we do not want to run SELFTEST.
64 */
Li Zefan020e5f82009-07-01 10:47:05 +080065bool __read_mostly tracing_selftest_disabled;
Steven Rostedtb2821ae2009-02-02 21:38:32 -050066
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010067/* For tracers that don't implement custom flags */
68static struct tracer_opt dummy_tracer_opt[] = {
69 { }
70};
71
72static struct tracer_flags dummy_tracer_flags = {
73 .val = 0,
74 .opts = dummy_tracer_opt
75};
76
77static int dummy_set_flag(u32 old_flags, u32 bit, int set)
78{
79 return 0;
80}
Steven Rostedt0f048702008-11-05 16:05:44 -050081
82/*
83 * Kill all tracing for good (never come back).
84 * It is initialized to 1 but will turn to zero if the initialization
85 * of the tracer is successful. But that is the only place that sets
86 * this back to zero.
87 */
Hannes Eder4fd27352009-02-10 19:44:12 +010088static int tracing_disabled = 1;
Steven Rostedt0f048702008-11-05 16:05:44 -050089
Christoph Lameter9288f992009-10-07 19:17:45 -040090DEFINE_PER_CPU(int, ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -040091
92static inline void ftrace_disable_cpu(void)
93{
94 preempt_disable();
Rusty Russelldd17c8f2009-10-29 22:34:15 +090095 __this_cpu_inc(ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -040096}
97
98static inline void ftrace_enable_cpu(void)
99{
Rusty Russelldd17c8f2009-10-29 22:34:15 +0900100 __this_cpu_dec(ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -0400101 preempt_enable();
102}
103
Jason Wessel955b61e2010-08-05 09:22:23 -0500104cpumask_var_t __read_mostly tracing_buffer_mask;
Steven Rostedtab464282008-05-12 21:21:00 +0200105
Steven Rostedt944ac422008-10-23 19:26:08 -0400106/*
107 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
108 *
109 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
110 * is set, then ftrace_dump is called. This will output the contents
111 * of the ftrace buffers to the console. This is very useful for
112 * capturing traces that lead to crashes and outputing it to a
113 * serial console.
114 *
115 * It is default off, but you can enable it with either specifying
116 * "ftrace_dump_on_oops" in the kernel command line, or setting
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200117 * /proc/sys/kernel/ftrace_dump_on_oops
118 * Set 1 if you want to dump buffers of all CPUs
119 * Set 2 if you want to dump the buffer of the CPU that triggered oops
Steven Rostedt944ac422008-10-23 19:26:08 -0400120 */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200121
122enum ftrace_dump_mode ftrace_dump_on_oops;
Steven Rostedt944ac422008-10-23 19:26:08 -0400123
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500124static int tracing_set_tracer(const char *buf);
125
Li Zefanee6c2c12009-09-18 14:06:47 +0800126#define MAX_TRACER_SIZE 100
127static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500128static char *default_bootup_tracer;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100129
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200130static int __init set_cmdline_ftrace(char *str)
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100131{
Li Zefanee6c2c12009-09-18 14:06:47 +0800132 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500133 default_bootup_tracer = bootup_tracer_buf;
Steven Rostedt73c51622009-03-11 13:42:01 -0400134 /* We are using ftrace early, expand it */
135 ring_buffer_expanded = 1;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100136 return 1;
137}
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200138__setup("ftrace=", set_cmdline_ftrace);
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100139
Steven Rostedt944ac422008-10-23 19:26:08 -0400140static int __init set_ftrace_dump_on_oops(char *str)
141{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200142 if (*str++ != '=' || !*str) {
143 ftrace_dump_on_oops = DUMP_ALL;
144 return 1;
145 }
146
147 if (!strcmp("orig_cpu", str)) {
148 ftrace_dump_on_oops = DUMP_ORIG;
149 return 1;
150 }
151
152 return 0;
Steven Rostedt944ac422008-10-23 19:26:08 -0400153}
154__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
Steven Rostedt60a11772008-05-12 21:20:44 +0200155
Lai Jiangshancf8e3472009-03-30 13:48:00 +0800156unsigned long long ns2usecs(cycle_t nsec)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200157{
158 nsec += 500;
159 do_div(nsec, 1000);
160 return nsec;
161}
162
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200163/*
164 * The global_trace is the descriptor that holds the tracing
165 * buffers for the live tracing. For each CPU, it contains
166 * a link list of pages that will store trace entries. The
167 * page descriptor of the pages in the memory is used to hold
168 * the link list by linking the lru item in the page descriptor
169 * to each of the pages in the buffer per CPU.
170 *
171 * For each active CPU there is a data field that holds the
172 * pages for the buffer for that CPU. Each CPU has the same number
173 * of pages allocated for its buffer.
174 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200175static struct trace_array global_trace;
176
177static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
178
Steven Rostedte77405a2009-09-02 14:17:06 -0400179int filter_current_check_discard(struct ring_buffer *buffer,
180 struct ftrace_event_call *call, void *rec,
Tom Zanussieb02ce02009-04-08 03:15:54 -0500181 struct ring_buffer_event *event)
182{
Steven Rostedte77405a2009-09-02 14:17:06 -0400183 return filter_check_discard(call, rec, buffer, event);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500184}
Steven Rostedt17c873e2009-04-10 18:12:50 -0400185EXPORT_SYMBOL_GPL(filter_current_check_discard);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500186
Steven Rostedt37886f62009-03-17 17:22:06 -0400187cycle_t ftrace_now(int cpu)
188{
189 u64 ts;
190
191 /* Early boot up does not have a buffer yet */
192 if (!global_trace.buffer)
193 return trace_clock_local();
194
195 ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
196 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
197
198 return ts;
199}
200
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200201/*
202 * The max_tr is used to snapshot the global_trace when a maximum
203 * latency is reached. Some tracers will use this to store a maximum
204 * trace while it continues examining live traces.
205 *
206 * The buffers for the max_tr are set up the same as the global_trace.
207 * When a snapshot is taken, the link list of the max_tr is swapped
208 * with the link list of the global_trace and the buffers are reset for
209 * the global_trace so the tracing can continue.
210 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200211static struct trace_array max_tr;
212
Tejun Heo9705f692009-10-29 22:34:13 +0900213static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200214
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200215/* tracer_enabled is used to toggle activation of a tracer */
Steven Rostedt26994ea2008-05-12 21:20:48 +0200216static int tracer_enabled = 1;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200217
Steven Rostedt90369902008-11-05 16:05:44 -0500218/**
219 * tracing_is_enabled - return tracer_enabled status
220 *
221 * This function is used by other tracers to know the status
222 * of the tracer_enabled flag. Tracers may use this function
223 * to know if it should enable their features when starting
224 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
225 */
226int tracing_is_enabled(void)
227{
228 return tracer_enabled;
229}
230
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200231/*
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400232 * trace_buf_size is the size in bytes that is allocated
233 * for a buffer. Note, the number of bytes is always rounded
234 * to page size.
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400235 *
236 * This number is purposely set to a low number of 16384.
237 * If the dump on oops happens, it will be much appreciated
238 * to not have to wait for all that output. Anyway this can be
239 * boot time and run time configurable.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200240 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400241#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400242
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400243static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200244
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200245/* trace_types holds a link list of available tracers. */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200246static struct tracer *trace_types __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200247
248/* current_trace points to the tracer that is currently active */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200249static struct tracer *current_trace __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200250
251/*
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200252 * trace_types_lock is used to protect the trace_types list.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200253 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200254static DEFINE_MUTEX(trace_types_lock);
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200255
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800256/*
257 * serialize the access of the ring buffer
258 *
259 * ring buffer serializes readers, but it is low level protection.
260 * The validity of the events (which returns by ring_buffer_peek() ..etc)
261 * are not protected by ring buffer.
262 *
263 * The content of events may become garbage if we allow other process consumes
264 * these events concurrently:
265 * A) the page of the consumed events may become a normal page
266 * (not reader page) in ring buffer, and this page will be rewrited
267 * by events producer.
268 * B) The page of the consumed events may become a page for splice_read,
269 * and this page will be returned to system.
270 *
271 * These primitives allow multi process access to different cpu ring buffer
272 * concurrently.
273 *
274 * These primitives don't distinguish read-only and read-consume access.
275 * Multi read-only access are also serialized.
276 */
277
278#ifdef CONFIG_SMP
279static DECLARE_RWSEM(all_cpu_access_lock);
280static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
281
282static inline void trace_access_lock(int cpu)
283{
284 if (cpu == TRACE_PIPE_ALL_CPU) {
285 /* gain it for accessing the whole ring buffer. */
286 down_write(&all_cpu_access_lock);
287 } else {
288 /* gain it for accessing a cpu ring buffer. */
289
290 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
291 down_read(&all_cpu_access_lock);
292
293 /* Secondly block other access to this @cpu ring buffer. */
294 mutex_lock(&per_cpu(cpu_access_lock, cpu));
295 }
296}
297
298static inline void trace_access_unlock(int cpu)
299{
300 if (cpu == TRACE_PIPE_ALL_CPU) {
301 up_write(&all_cpu_access_lock);
302 } else {
303 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
304 up_read(&all_cpu_access_lock);
305 }
306}
307
308static inline void trace_access_lock_init(void)
309{
310 int cpu;
311
312 for_each_possible_cpu(cpu)
313 mutex_init(&per_cpu(cpu_access_lock, cpu));
314}
315
316#else
317
318static DEFINE_MUTEX(access_lock);
319
320static inline void trace_access_lock(int cpu)
321{
322 (void)cpu;
323 mutex_lock(&access_lock);
324}
325
326static inline void trace_access_unlock(int cpu)
327{
328 (void)cpu;
329 mutex_unlock(&access_lock);
330}
331
332static inline void trace_access_lock_init(void)
333{
334}
335
336#endif
337
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200338/* trace_wait is a waitqueue for tasks blocked on trace_poll */
Ingo Molnar4e655512008-05-12 21:20:52 +0200339static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
340
Steven Rostedtee6bce52008-11-12 17:52:37 -0500341/* trace_flags holds trace_options default values */
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500342unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
Steven Rostedta2a16d62009-03-24 23:17:58 -0400343 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
344 TRACE_ITER_GRAPH_TIME;
Ingo Molnar4e655512008-05-12 21:20:52 +0200345
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400346static int trace_stop_count;
347static DEFINE_SPINLOCK(tracing_start_lock);
348
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200349/**
350 * trace_wake_up - wake up tasks waiting for trace input
351 *
352 * Simply wakes up any task that is blocked on the trace_wait
353 * queue. These is used with trace_poll for tasks polling the trace.
354 */
Ingo Molnar4e655512008-05-12 21:20:52 +0200355void trace_wake_up(void)
356{
Andrew Morton89f19f02009-09-19 11:55:44 -0700357 int cpu;
358
359 if (trace_flags & TRACE_ITER_BLOCK)
360 return;
Ingo Molnar017730c2008-05-12 21:20:52 +0200361 /*
362 * The runqueue_is_locked() can fail, but this is the best we
363 * have for now:
364 */
Andrew Morton89f19f02009-09-19 11:55:44 -0700365 cpu = get_cpu();
366 if (!runqueue_is_locked(cpu))
Ingo Molnar4e655512008-05-12 21:20:52 +0200367 wake_up(&trace_wait);
Andrew Morton89f19f02009-09-19 11:55:44 -0700368 put_cpu();
Ingo Molnar4e655512008-05-12 21:20:52 +0200369}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200370
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400371static int __init set_buf_size(char *str)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200372{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400373 unsigned long buf_size;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200374
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200375 if (!str)
376 return 0;
Li Zefan9d612be2009-06-24 17:33:15 +0800377 buf_size = memparse(str, &str);
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200378 /* nr_entries can not be zero */
Li Zefan9d612be2009-06-24 17:33:15 +0800379 if (buf_size == 0)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200380 return 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400381 trace_buf_size = buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200382 return 1;
383}
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400384__setup("trace_buf_size=", set_buf_size);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200385
Tim Bird0e950172010-02-25 15:36:43 -0800386static int __init set_tracing_thresh(char *str)
387{
388 unsigned long threshhold;
389 int ret;
390
391 if (!str)
392 return 0;
393 ret = strict_strtoul(str, 0, &threshhold);
394 if (ret < 0)
395 return 0;
396 tracing_thresh = threshhold * 1000;
397 return 1;
398}
399__setup("tracing_thresh=", set_tracing_thresh);
400
Steven Rostedt57f50be2008-05-12 21:20:44 +0200401unsigned long nsecs_to_usecs(unsigned long nsecs)
402{
403 return nsecs / 1000;
404}
405
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200406/* These must match the bit postions in trace_iterator_flags */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200407static const char *trace_options[] = {
408 "print-parent",
409 "sym-offset",
410 "sym-addr",
411 "verbose",
Ingo Molnarf9896bf2008-05-12 21:20:47 +0200412 "raw",
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +0200413 "hex",
Ingo Molnarcb0f12a2008-05-12 21:20:47 +0200414 "bin",
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +0200415 "block",
Ingo Molnar86387f72008-05-12 21:20:51 +0200416 "stacktrace",
Ingo Molnar5e1607a2009-03-05 10:24:48 +0100417 "trace_printk",
Steven Rostedtb2a866f2008-11-03 23:15:57 -0500418 "ftrace_preempt",
Steven Rostedt9f029e82008-11-12 15:24:24 -0500419 "branch",
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500420 "annotate",
Török Edwin02b67512008-11-22 13:28:47 +0200421 "userstacktrace",
Török Edwinb54d3de2008-11-22 13:28:48 +0200422 "sym-userobj",
Frederic Weisbecker66896a82008-12-13 20:18:13 +0100423 "printk-msg-only",
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -0200424 "context-info",
Steven Rostedtc032ef642009-03-04 20:34:24 -0500425 "latency-format",
Steven Rostedtbe6f1642009-03-24 11:06:24 -0400426 "sleep-time",
Steven Rostedta2a16d62009-03-24 23:17:58 -0400427 "graph-time",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200428 NULL
429};
430
Zhaolei5079f322009-08-25 16:12:56 +0800431static struct {
432 u64 (*func)(void);
433 const char *name;
434} trace_clocks[] = {
435 { trace_clock_local, "local" },
436 { trace_clock_global, "global" },
437};
438
439int trace_clock_id;
440
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200441/*
442 * trace_parser_get_init - gets the buffer for trace parser
443 */
444int trace_parser_get_init(struct trace_parser *parser, int size)
445{
446 memset(parser, 0, sizeof(*parser));
447
448 parser->buffer = kmalloc(size, GFP_KERNEL);
449 if (!parser->buffer)
450 return 1;
451
452 parser->size = size;
453 return 0;
454}
455
456/*
457 * trace_parser_put - frees the buffer for trace parser
458 */
459void trace_parser_put(struct trace_parser *parser)
460{
461 kfree(parser->buffer);
462}
463
464/*
465 * trace_get_user - reads the user input string separated by space
466 * (matched by isspace(ch))
467 *
468 * For each string found the 'struct trace_parser' is updated,
469 * and the function returns.
470 *
471 * Returns number of bytes read.
472 *
473 * See kernel/trace/trace.h for 'struct trace_parser' details.
474 */
475int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
476 size_t cnt, loff_t *ppos)
477{
478 char ch;
479 size_t read = 0;
480 ssize_t ret;
481
482 if (!*ppos)
483 trace_parser_clear(parser);
484
485 ret = get_user(ch, ubuf++);
486 if (ret)
487 goto out;
488
489 read++;
490 cnt--;
491
492 /*
493 * The parser is not finished with the last write,
494 * continue reading the user input without skipping spaces.
495 */
496 if (!parser->cont) {
497 /* skip white space */
498 while (cnt && isspace(ch)) {
499 ret = get_user(ch, ubuf++);
500 if (ret)
501 goto out;
502 read++;
503 cnt--;
504 }
505
506 /* only spaces were written */
507 if (isspace(ch)) {
508 *ppos += read;
509 ret = read;
510 goto out;
511 }
512
513 parser->idx = 0;
514 }
515
516 /* read the non-space input */
517 while (cnt && !isspace(ch)) {
Li Zefan3c235a32009-09-22 13:51:54 +0800518 if (parser->idx < parser->size - 1)
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200519 parser->buffer[parser->idx++] = ch;
520 else {
521 ret = -EINVAL;
522 goto out;
523 }
524 ret = get_user(ch, ubuf++);
525 if (ret)
526 goto out;
527 read++;
528 cnt--;
529 }
530
531 /* We either got finished input or we have to wait for another call. */
532 if (isspace(ch)) {
533 parser->buffer[parser->idx] = 0;
534 parser->cont = false;
535 } else {
536 parser->cont = true;
537 parser->buffer[parser->idx++] = ch;
538 }
539
540 *ppos += read;
541 ret = read;
542
543out:
544 return ret;
545}
546
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200547ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
548{
549 int len;
550 int ret;
551
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500552 if (!cnt)
553 return 0;
554
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200555 if (s->len <= s->readpos)
556 return -EBUSY;
557
558 len = s->len - s->readpos;
559 if (cnt > len)
560 cnt = len;
561 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500562 if (ret == cnt)
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200563 return -EFAULT;
564
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500565 cnt -= ret;
566
Steven Rostedte74da522009-03-04 20:31:11 -0500567 s->readpos += cnt;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200568 return cnt;
Steven Rostedt214023c2008-05-12 21:20:46 +0200569}
570
Dmitri Vorobievb8b94262009-03-22 19:11:11 +0200571static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200572{
573 int len;
574 void *ret;
575
576 if (s->len <= s->readpos)
577 return -EBUSY;
578
579 len = s->len - s->readpos;
580 if (cnt > len)
581 cnt = len;
582 ret = memcpy(buf, s->buffer + s->readpos, cnt);
583 if (!ret)
584 return -EFAULT;
585
Steven Rostedte74da522009-03-04 20:31:11 -0500586 s->readpos += cnt;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200587 return cnt;
588}
589
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400590/*
591 * ftrace_max_lock is used to protect the swapping of buffers
592 * when taking a max snapshot. The buffers themselves are
593 * protected by per_cpu spinlocks. But the action of the swap
594 * needs its own lock.
595 *
Thomas Gleixner445c8952009-12-02 19:49:50 +0100596 * This is defined as a arch_spinlock_t in order to help
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400597 * with performance when lockdep debugging is enabled.
598 *
599 * It is also used in other places outside the update_max_tr
600 * so it needs to be defined outside of the
601 * CONFIG_TRACER_MAX_TRACE.
602 */
Thomas Gleixner445c8952009-12-02 19:49:50 +0100603static arch_spinlock_t ftrace_max_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100604 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400605
Tim Bird0e950172010-02-25 15:36:43 -0800606unsigned long __read_mostly tracing_thresh;
607
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400608#ifdef CONFIG_TRACER_MAX_TRACE
609unsigned long __read_mostly tracing_max_latency;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400610
611/*
612 * Copy the new maximum trace into the separate maximum-trace
613 * structure. (this way the maximum trace is permanently saved,
614 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
615 */
616static void
617__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
618{
619 struct trace_array_cpu *data = tr->data[cpu];
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300620 struct trace_array_cpu *max_data;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400621
622 max_tr.cpu = cpu;
623 max_tr.time_start = data->preempt_timestamp;
624
Steven Rostedt8248ac02009-09-02 12:27:41 -0400625 max_data = max_tr.data[cpu];
626 max_data->saved_latency = tracing_max_latency;
627 max_data->critical_start = data->critical_start;
628 max_data->critical_end = data->critical_end;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400629
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300630 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
Steven Rostedt8248ac02009-09-02 12:27:41 -0400631 max_data->pid = tsk->pid;
632 max_data->uid = task_uid(tsk);
633 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
634 max_data->policy = tsk->policy;
635 max_data->rt_priority = tsk->rt_priority;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400636
637 /* record this tasks comm */
638 tracing_record_cmdline(tsk);
639}
640
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200641/**
642 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
643 * @tr: tracer
644 * @tsk: the task with the latency
645 * @cpu: The cpu that initiated the trace.
646 *
647 * Flip the buffers between the @tr and the max_tr and record information
648 * about which task was the cause of this latency.
649 */
Ingo Molnare309b412008-05-12 21:20:51 +0200650void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200651update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
652{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400653 struct ring_buffer *buf = tr->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200654
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400655 if (trace_stop_count)
656 return;
657
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200658 WARN_ON_ONCE(!irqs_disabled());
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100659 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400660
661 tr->buffer = max_tr.buffer;
662 max_tr.buffer = buf;
663
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200664 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100665 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200666}
667
668/**
669 * update_max_tr_single - only copy one trace over, and reset the rest
670 * @tr - tracer
671 * @tsk - task with the latency
672 * @cpu - the cpu of the buffer to copy.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200673 *
674 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200675 */
Ingo Molnare309b412008-05-12 21:20:51 +0200676void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200677update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
678{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400679 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200680
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400681 if (trace_stop_count)
682 return;
683
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200684 WARN_ON_ONCE(!irqs_disabled());
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100685 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200686
Steven Rostedtd7690412008-10-01 00:29:53 -0400687 ftrace_disable_cpu();
688
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400689 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
690
Steven Rostedte8165db2009-09-03 19:13:05 -0400691 if (ret == -EBUSY) {
692 /*
693 * We failed to swap the buffer due to a commit taking
694 * place on this CPU. We fail to record, but we reset
695 * the max trace buffer (no one writes directly to it)
696 * and flag that it failed.
697 */
698 trace_array_printk(&max_tr, _THIS_IP_,
699 "Failed to swap buffers due to commit in progress\n");
700 }
701
Steven Rostedtd7690412008-10-01 00:29:53 -0400702 ftrace_enable_cpu();
703
Steven Rostedte8165db2009-09-03 19:13:05 -0400704 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200705
706 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100707 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200708}
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400709#endif /* CONFIG_TRACER_MAX_TRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200710
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200711/**
712 * register_tracer - register a tracer with the ftrace system.
713 * @type - the plugin for the tracer
714 *
715 * Register a new plugin tracer.
716 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200717int register_tracer(struct tracer *type)
Hannes Edere7669b82009-02-10 19:44:45 +0100718__releases(kernel_lock)
719__acquires(kernel_lock)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200720{
721 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200722 int ret = 0;
723
724 if (!type->name) {
725 pr_info("Tracer must have a name\n");
726 return -1;
727 }
728
Li Zefanee6c2c12009-09-18 14:06:47 +0800729 if (strlen(type->name) > MAX_TRACER_SIZE) {
730 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
731 return -1;
732 }
733
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100734 /*
735 * When this gets called we hold the BKL which means that
736 * preemption is disabled. Various trace selftests however
737 * need to disable and enable preemption for successful tests.
738 * So we drop the BKL here and grab it after the tests again.
739 */
740 unlock_kernel();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200741 mutex_lock(&trace_types_lock);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100742
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +0100743 tracing_selftest_running = true;
744
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200745 for (t = trace_types; t; t = t->next) {
746 if (strcmp(type->name, t->name) == 0) {
747 /* already found */
Li Zefanee6c2c12009-09-18 14:06:47 +0800748 pr_info("Tracer %s already registered\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200749 type->name);
750 ret = -1;
751 goto out;
752 }
753 }
754
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +0100755 if (!type->set_flag)
756 type->set_flag = &dummy_set_flag;
757 if (!type->flags)
758 type->flags = &dummy_tracer_flags;
759 else
760 if (!type->flags->opts)
761 type->flags->opts = dummy_tracer_opt;
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +0100762 if (!type->wait_pipe)
763 type->wait_pipe = default_wait_pipe;
764
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +0100765
Steven Rostedt60a11772008-05-12 21:20:44 +0200766#ifdef CONFIG_FTRACE_STARTUP_TEST
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500767 if (type->selftest && !tracing_selftest_disabled) {
Steven Rostedt60a11772008-05-12 21:20:44 +0200768 struct tracer *saved_tracer = current_trace;
Steven Rostedt60a11772008-05-12 21:20:44 +0200769 struct trace_array *tr = &global_trace;
Frederic Weisbeckerff325042008-12-04 23:47:35 +0100770
Steven Rostedt60a11772008-05-12 21:20:44 +0200771 /*
772 * Run a selftest on this tracer.
773 * Here we reset the trace buffer, and set the current
774 * tracer to be this tracer. The tracer can then run some
775 * internal tracing to verify that everything is in order.
776 * If we fail, we do not register this tracer.
777 */
Steven Rostedt76f0d072009-09-04 12:12:39 -0400778 tracing_reset_online_cpus(tr);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100779
Steven Rostedt60a11772008-05-12 21:20:44 +0200780 current_trace = type;
Steven Rostedt60a11772008-05-12 21:20:44 +0200781 /* the test is responsible for initializing and enabling */
782 pr_info("Testing tracer %s: ", type->name);
783 ret = type->selftest(type, tr);
784 /* the test is responsible for resetting too */
785 current_trace = saved_tracer;
Steven Rostedt60a11772008-05-12 21:20:44 +0200786 if (ret) {
787 printk(KERN_CONT "FAILED!\n");
788 goto out;
789 }
Steven Rostedt1d4db002008-05-12 21:20:45 +0200790 /* Only reset on passing, to avoid touching corrupted buffers */
Steven Rostedt76f0d072009-09-04 12:12:39 -0400791 tracing_reset_online_cpus(tr);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100792
Steven Rostedt60a11772008-05-12 21:20:44 +0200793 printk(KERN_CONT "PASSED\n");
794 }
795#endif
796
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200797 type->next = trace_types;
798 trace_types = type;
Steven Rostedt60a11772008-05-12 21:20:44 +0200799
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200800 out:
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +0100801 tracing_selftest_running = false;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200802 mutex_unlock(&trace_types_lock);
803
Steven Rostedtdac74942009-02-05 01:13:38 -0500804 if (ret || !default_bootup_tracer)
805 goto out_unlock;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500806
Li Zefanee6c2c12009-09-18 14:06:47 +0800807 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
Steven Rostedtdac74942009-02-05 01:13:38 -0500808 goto out_unlock;
809
810 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
811 /* Do we want this tracer to start on bootup? */
812 tracing_set_tracer(type->name);
813 default_bootup_tracer = NULL;
814 /* disable other selftests, since this will break it. */
815 tracing_selftest_disabled = 1;
816#ifdef CONFIG_FTRACE_STARTUP_TEST
817 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
818 type->name);
819#endif
820
821 out_unlock:
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500822 lock_kernel();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200823 return ret;
824}
825
826void unregister_tracer(struct tracer *type)
827{
828 struct tracer **t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200829
830 mutex_lock(&trace_types_lock);
831 for (t = &trace_types; *t; t = &(*t)->next) {
832 if (*t == type)
833 goto found;
834 }
Li Zefanee6c2c12009-09-18 14:06:47 +0800835 pr_info("Tracer %s not registered\n", type->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200836 goto out;
837
838 found:
839 *t = (*t)->next;
Arnaldo Carvalho de Melob5db03c2009-02-07 18:52:59 -0200840
841 if (type == current_trace && tracer_enabled) {
842 tracer_enabled = 0;
843 tracing_stop();
844 if (current_trace->stop)
845 current_trace->stop(&global_trace);
846 current_trace = &nop_trace;
847 }
Li Zefanee6c2c12009-09-18 14:06:47 +0800848out:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200849 mutex_unlock(&trace_types_lock);
850}
851
Steven Rostedt283740c2010-03-12 19:48:41 -0500852static void __tracing_reset(struct ring_buffer *buffer, int cpu)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200853{
Steven Rostedtd7690412008-10-01 00:29:53 -0400854 ftrace_disable_cpu();
Steven Rostedt283740c2010-03-12 19:48:41 -0500855 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedtd7690412008-10-01 00:29:53 -0400856 ftrace_enable_cpu();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200857}
858
Steven Rostedtf6339032009-09-04 12:35:16 -0400859void tracing_reset(struct trace_array *tr, int cpu)
860{
861 struct ring_buffer *buffer = tr->buffer;
862
863 ring_buffer_record_disable(buffer);
864
865 /* Make sure all commits have finished */
866 synchronize_sched();
Steven Rostedt283740c2010-03-12 19:48:41 -0500867 __tracing_reset(buffer, cpu);
Steven Rostedtf6339032009-09-04 12:35:16 -0400868
869 ring_buffer_record_enable(buffer);
870}
871
Pekka J Enberg213cc062008-12-19 12:08:39 +0200872void tracing_reset_online_cpus(struct trace_array *tr)
873{
Steven Rostedt621968c2009-09-04 12:02:35 -0400874 struct ring_buffer *buffer = tr->buffer;
Pekka J Enberg213cc062008-12-19 12:08:39 +0200875 int cpu;
876
Steven Rostedt621968c2009-09-04 12:02:35 -0400877 ring_buffer_record_disable(buffer);
878
879 /* Make sure all commits have finished */
880 synchronize_sched();
881
Pekka J Enberg213cc062008-12-19 12:08:39 +0200882 tr->time_start = ftrace_now(tr->cpu);
883
884 for_each_online_cpu(cpu)
Steven Rostedt283740c2010-03-12 19:48:41 -0500885 __tracing_reset(buffer, cpu);
Steven Rostedt621968c2009-09-04 12:02:35 -0400886
887 ring_buffer_record_enable(buffer);
Pekka J Enberg213cc062008-12-19 12:08:39 +0200888}
889
Steven Rostedt9456f0f2009-05-06 21:54:09 -0400890void tracing_reset_current(int cpu)
891{
892 tracing_reset(&global_trace, cpu);
893}
894
895void tracing_reset_current_online_cpus(void)
896{
897 tracing_reset_online_cpus(&global_trace);
898}
899
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200900#define SAVED_CMDLINES 128
Thomas Gleixner2c7eea42009-03-18 09:03:19 +0100901#define NO_CMDLINE_MAP UINT_MAX
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200902static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
903static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
904static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
905static int cmdline_idx;
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100906static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt25b0b442008-05-12 21:21:00 +0200907
Steven Rostedt25b0b442008-05-12 21:21:00 +0200908/* temporary disable recording */
Hannes Eder4fd27352009-02-10 19:44:12 +0100909static atomic_t trace_record_cmdline_disabled __read_mostly;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200910
911static void trace_init_cmdlines(void)
912{
Thomas Gleixner2c7eea42009-03-18 09:03:19 +0100913 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
914 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200915 cmdline_idx = 0;
916}
917
Carsten Emdeb5130b12009-09-13 01:43:07 +0200918int is_tracing_stopped(void)
919{
920 return trace_stop_count;
921}
922
Steven Rostedt0f048702008-11-05 16:05:44 -0500923/**
Steven Rostedt69bb54e2008-11-21 12:59:38 -0500924 * ftrace_off_permanent - disable all ftrace code permanently
925 *
926 * This should only be called when a serious anomally has
927 * been detected. This will turn off the function tracing,
928 * ring buffers, and other tracing utilites. It takes no
929 * locks and can be called from any context.
930 */
931void ftrace_off_permanent(void)
932{
933 tracing_disabled = 1;
934 ftrace_stop();
935 tracing_off_permanent();
936}
937
938/**
Steven Rostedt0f048702008-11-05 16:05:44 -0500939 * tracing_start - quick start of the tracer
940 *
941 * If tracing is enabled but was stopped by tracing_stop,
942 * this will start the tracer back up.
943 */
944void tracing_start(void)
945{
946 struct ring_buffer *buffer;
947 unsigned long flags;
948
949 if (tracing_disabled)
950 return;
951
952 spin_lock_irqsave(&tracing_start_lock, flags);
Steven Rostedtb06a8302009-01-22 14:26:15 -0500953 if (--trace_stop_count) {
954 if (trace_stop_count < 0) {
955 /* Someone screwed up their debugging */
956 WARN_ON_ONCE(1);
957 trace_stop_count = 0;
958 }
Steven Rostedt0f048702008-11-05 16:05:44 -0500959 goto out;
960 }
961
Steven Rostedta2f80712010-03-12 19:56:00 -0500962 /* Prevent the buffers from switching */
963 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt0f048702008-11-05 16:05:44 -0500964
965 buffer = global_trace.buffer;
966 if (buffer)
967 ring_buffer_record_enable(buffer);
968
969 buffer = max_tr.buffer;
970 if (buffer)
971 ring_buffer_record_enable(buffer);
972
Steven Rostedta2f80712010-03-12 19:56:00 -0500973 arch_spin_unlock(&ftrace_max_lock);
974
Steven Rostedt0f048702008-11-05 16:05:44 -0500975 ftrace_start();
976 out:
977 spin_unlock_irqrestore(&tracing_start_lock, flags);
978}
979
980/**
981 * tracing_stop - quick stop of the tracer
982 *
983 * Light weight way to stop tracing. Use in conjunction with
984 * tracing_start.
985 */
986void tracing_stop(void)
987{
988 struct ring_buffer *buffer;
989 unsigned long flags;
990
991 ftrace_stop();
992 spin_lock_irqsave(&tracing_start_lock, flags);
993 if (trace_stop_count++)
994 goto out;
995
Steven Rostedta2f80712010-03-12 19:56:00 -0500996 /* Prevent the buffers from switching */
997 arch_spin_lock(&ftrace_max_lock);
998
Steven Rostedt0f048702008-11-05 16:05:44 -0500999 buffer = global_trace.buffer;
1000 if (buffer)
1001 ring_buffer_record_disable(buffer);
1002
1003 buffer = max_tr.buffer;
1004 if (buffer)
1005 ring_buffer_record_disable(buffer);
1006
Steven Rostedta2f80712010-03-12 19:56:00 -05001007 arch_spin_unlock(&ftrace_max_lock);
1008
Steven Rostedt0f048702008-11-05 16:05:44 -05001009 out:
1010 spin_unlock_irqrestore(&tracing_start_lock, flags);
1011}
1012
Ingo Molnare309b412008-05-12 21:20:51 +02001013void trace_stop_cmdline_recording(void);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001014
Ingo Molnare309b412008-05-12 21:20:51 +02001015static void trace_save_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001016{
Carsten Emdea635cf02009-03-18 09:00:41 +01001017 unsigned pid, idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001018
1019 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1020 return;
1021
1022 /*
1023 * It's not the end of the world if we don't get
1024 * the lock, but we also don't want to spin
1025 * nor do we want to disable interrupts,
1026 * so if we miss here, then better luck next time.
1027 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001028 if (!arch_spin_trylock(&trace_cmdline_lock))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001029 return;
1030
1031 idx = map_pid_to_cmdline[tsk->pid];
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001032 if (idx == NO_CMDLINE_MAP) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001033 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1034
Carsten Emdea635cf02009-03-18 09:00:41 +01001035 /*
1036 * Check whether the cmdline buffer at idx has a pid
1037 * mapped. We are going to overwrite that entry so we
1038 * need to clear the map_pid_to_cmdline. Otherwise we
1039 * would read the new comm for the old pid.
1040 */
1041 pid = map_cmdline_to_pid[idx];
1042 if (pid != NO_CMDLINE_MAP)
1043 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001044
Carsten Emdea635cf02009-03-18 09:00:41 +01001045 map_cmdline_to_pid[idx] = tsk->pid;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001046 map_pid_to_cmdline[tsk->pid] = idx;
1047
1048 cmdline_idx = idx;
1049 }
1050
1051 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1052
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001053 arch_spin_unlock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001054}
1055
Steven Rostedt4ca53082009-03-16 19:20:15 -04001056void trace_find_cmdline(int pid, char comm[])
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001057{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001058 unsigned map;
1059
Steven Rostedt4ca53082009-03-16 19:20:15 -04001060 if (!pid) {
1061 strcpy(comm, "<idle>");
1062 return;
1063 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001064
Steven Rostedt74bf4072010-01-25 15:11:53 -05001065 if (WARN_ON_ONCE(pid < 0)) {
1066 strcpy(comm, "<XXX>");
1067 return;
1068 }
1069
Steven Rostedt4ca53082009-03-16 19:20:15 -04001070 if (pid > PID_MAX_DEFAULT) {
1071 strcpy(comm, "<...>");
1072 return;
1073 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001074
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001075 preempt_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001076 arch_spin_lock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001077 map = map_pid_to_cmdline[pid];
Thomas Gleixner50d88752009-03-18 08:58:44 +01001078 if (map != NO_CMDLINE_MAP)
1079 strcpy(comm, saved_cmdlines[map]);
1080 else
1081 strcpy(comm, "<...>");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001082
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001083 arch_spin_unlock(&trace_cmdline_lock);
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001084 preempt_enable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001085}
1086
Ingo Molnare309b412008-05-12 21:20:51 +02001087void tracing_record_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001088{
Thomas Gleixner18aecd32009-03-18 08:56:58 +01001089 if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1090 !tracing_is_on())
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001091 return;
1092
1093 trace_save_cmdline(tsk);
1094}
1095
Pekka Paalanen45dcd8b2008-09-16 21:56:41 +03001096void
Steven Rostedt38697052008-10-01 13:14:09 -04001097tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1098 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001099{
1100 struct task_struct *tsk = current;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001101
Steven Rostedt777e2082008-09-29 23:02:42 -04001102 entry->preempt_count = pc & 0xff;
1103 entry->pid = (tsk) ? tsk->pid : 0;
Steven Rostedt637e7e82009-09-11 13:55:35 -04001104 entry->lock_depth = (tsk) ? tsk->lock_depth : 0;
Steven Rostedt777e2082008-09-29 23:02:42 -04001105 entry->flags =
Steven Rostedt92444892008-10-24 09:42:59 -04001106#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
Steven Rostedt2e2ca152008-08-01 12:26:40 -04001107 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
Steven Rostedt92444892008-10-24 09:42:59 -04001108#else
1109 TRACE_FLAG_IRQS_NOSUPPORT |
1110#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001111 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1112 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1113 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1114}
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +02001115EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001116
Steven Rostedte77405a2009-09-02 14:17:06 -04001117struct ring_buffer_event *
1118trace_buffer_lock_reserve(struct ring_buffer *buffer,
1119 int type,
1120 unsigned long len,
1121 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001122{
1123 struct ring_buffer_event *event;
1124
Steven Rostedte77405a2009-09-02 14:17:06 -04001125 event = ring_buffer_lock_reserve(buffer, len);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001126 if (event != NULL) {
1127 struct trace_entry *ent = ring_buffer_event_data(event);
1128
1129 tracing_generic_entry_update(ent, flags, pc);
1130 ent->type = type;
1131 }
1132
1133 return event;
1134}
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001135
Steven Rostedte77405a2009-09-02 14:17:06 -04001136static inline void
1137__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1138 struct ring_buffer_event *event,
1139 unsigned long flags, int pc,
1140 int wake)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001141{
Steven Rostedte77405a2009-09-02 14:17:06 -04001142 ring_buffer_unlock_commit(buffer, event);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001143
Steven Rostedte77405a2009-09-02 14:17:06 -04001144 ftrace_trace_stack(buffer, flags, 6, pc);
1145 ftrace_trace_userstack(buffer, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001146
1147 if (wake)
1148 trace_wake_up();
1149}
1150
Steven Rostedte77405a2009-09-02 14:17:06 -04001151void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1152 struct ring_buffer_event *event,
1153 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001154{
Steven Rostedte77405a2009-09-02 14:17:06 -04001155 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001156}
1157
Steven Rostedtef5580d2009-02-27 19:38:04 -05001158struct ring_buffer_event *
Steven Rostedte77405a2009-09-02 14:17:06 -04001159trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1160 int type, unsigned long len,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001161 unsigned long flags, int pc)
1162{
Steven Rostedte77405a2009-09-02 14:17:06 -04001163 *current_rb = global_trace.buffer;
1164 return trace_buffer_lock_reserve(*current_rb,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001165 type, len, flags, pc);
1166}
Steven Rostedt94487d62009-05-05 19:22:53 -04001167EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001168
Steven Rostedte77405a2009-09-02 14:17:06 -04001169void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1170 struct ring_buffer_event *event,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001171 unsigned long flags, int pc)
1172{
Steven Rostedte77405a2009-09-02 14:17:06 -04001173 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001174}
Steven Rostedt94487d62009-05-05 19:22:53 -04001175EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001176
Steven Rostedte77405a2009-09-02 14:17:06 -04001177void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1178 struct ring_buffer_event *event,
1179 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001180{
Steven Rostedte77405a2009-09-02 14:17:06 -04001181 __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001182}
Steven Rostedt94487d62009-05-05 19:22:53 -04001183EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
Steven Rostedt77d9f462009-04-02 01:16:59 -04001184
Steven Rostedte77405a2009-09-02 14:17:06 -04001185void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1186 struct ring_buffer_event *event)
Steven Rostedt77d9f462009-04-02 01:16:59 -04001187{
Steven Rostedte77405a2009-09-02 14:17:06 -04001188 ring_buffer_discard_commit(buffer, event);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001189}
Steven Rostedt12acd472009-04-17 16:01:56 -04001190EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001191
Ingo Molnare309b412008-05-12 21:20:51 +02001192void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001193trace_function(struct trace_array *tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001194 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1195 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001196{
Tom Zanussie1112b42009-03-31 00:48:49 -05001197 struct ftrace_event_call *call = &event_function;
Steven Rostedte77405a2009-09-02 14:17:06 -04001198 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001199 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001200 struct ftrace_entry *entry;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001201
Steven Rostedtd7690412008-10-01 00:29:53 -04001202 /* If we are reading the ring buffer, don't trace */
Rusty Russelldd17c8f2009-10-29 22:34:15 +09001203 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
Steven Rostedtd7690412008-10-01 00:29:53 -04001204 return;
1205
Steven Rostedte77405a2009-09-02 14:17:06 -04001206 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001207 flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001208 if (!event)
1209 return;
1210 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001211 entry->ip = ip;
1212 entry->parent_ip = parent_ip;
Tom Zanussie1112b42009-03-31 00:48:49 -05001213
Steven Rostedte77405a2009-09-02 14:17:06 -04001214 if (!filter_check_discard(call, entry, buffer, event))
1215 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001216}
1217
Ingo Molnare309b412008-05-12 21:20:51 +02001218void
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001219ftrace(struct trace_array *tr, struct trace_array_cpu *data,
Steven Rostedt38697052008-10-01 13:14:09 -04001220 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1221 int pc)
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001222{
1223 if (likely(!atomic_read(&data->disabled)))
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001224 trace_function(tr, ip, parent_ip, flags, pc);
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001225}
1226
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001227#ifdef CONFIG_STACKTRACE
Steven Rostedte77405a2009-09-02 14:17:06 -04001228static void __ftrace_trace_stack(struct ring_buffer *buffer,
Steven Rostedt53614992009-01-15 19:12:40 -05001229 unsigned long flags,
1230 int skip, int pc)
Ingo Molnar86387f72008-05-12 21:20:51 +02001231{
Tom Zanussie1112b42009-03-31 00:48:49 -05001232 struct ftrace_event_call *call = &event_kernel_stack;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001233 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001234 struct stack_entry *entry;
Ingo Molnar86387f72008-05-12 21:20:51 +02001235 struct stack_trace trace;
1236
Steven Rostedte77405a2009-09-02 14:17:06 -04001237 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001238 sizeof(*entry), flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001239 if (!event)
1240 return;
1241 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001242 memset(&entry->caller, 0, sizeof(entry->caller));
Ingo Molnar86387f72008-05-12 21:20:51 +02001243
1244 trace.nr_entries = 0;
1245 trace.max_entries = FTRACE_STACK_ENTRIES;
1246 trace.skip = skip;
Steven Rostedt777e2082008-09-29 23:02:42 -04001247 trace.entries = entry->caller;
Ingo Molnar86387f72008-05-12 21:20:51 +02001248
1249 save_stack_trace(&trace);
Steven Rostedte77405a2009-09-02 14:17:06 -04001250 if (!filter_check_discard(call, entry, buffer, event))
1251 ring_buffer_unlock_commit(buffer, event);
Ingo Molnarf0a920d2008-05-12 21:20:47 +02001252}
1253
Steven Rostedte77405a2009-09-02 14:17:06 -04001254void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1255 int skip, int pc)
Steven Rostedt53614992009-01-15 19:12:40 -05001256{
1257 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1258 return;
1259
Steven Rostedte77405a2009-09-02 14:17:06 -04001260 __ftrace_trace_stack(buffer, flags, skip, pc);
Steven Rostedt53614992009-01-15 19:12:40 -05001261}
1262
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001263void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1264 int pc)
Steven Rostedt38697052008-10-01 13:14:09 -04001265{
Steven Rostedte77405a2009-09-02 14:17:06 -04001266 __ftrace_trace_stack(tr->buffer, flags, skip, pc);
Steven Rostedt38697052008-10-01 13:14:09 -04001267}
1268
Steven Rostedt03889382009-12-11 09:48:22 -05001269/**
1270 * trace_dump_stack - record a stack back trace in the trace buffer
1271 */
1272void trace_dump_stack(void)
1273{
1274 unsigned long flags;
1275
1276 if (tracing_disabled || tracing_selftest_running)
Steven Rostedte36c5452009-12-14 15:58:33 -05001277 return;
Steven Rostedt03889382009-12-11 09:48:22 -05001278
1279 local_save_flags(flags);
1280
1281 /* skipping 3 traces, seems to get us at the caller of this function */
1282 __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count());
1283}
1284
Steven Rostedte77405a2009-09-02 14:17:06 -04001285void
1286ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
Török Edwin02b67512008-11-22 13:28:47 +02001287{
Tom Zanussie1112b42009-03-31 00:48:49 -05001288 struct ftrace_event_call *call = &event_user_stack;
Török Edwin8d7c6a92008-11-23 12:39:06 +02001289 struct ring_buffer_event *event;
Török Edwin02b67512008-11-22 13:28:47 +02001290 struct userstack_entry *entry;
1291 struct stack_trace trace;
Török Edwin02b67512008-11-22 13:28:47 +02001292
1293 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1294 return;
1295
Steven Rostedtb6345872010-03-12 20:03:30 -05001296 /*
1297 * NMIs can not handle page faults, even with fix ups.
1298 * The save user stack can (and often does) fault.
1299 */
1300 if (unlikely(in_nmi()))
1301 return;
1302
Steven Rostedte77405a2009-09-02 14:17:06 -04001303 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001304 sizeof(*entry), flags, pc);
Török Edwin02b67512008-11-22 13:28:47 +02001305 if (!event)
1306 return;
1307 entry = ring_buffer_event_data(event);
Török Edwin02b67512008-11-22 13:28:47 +02001308
Steven Rostedt48659d32009-09-11 11:36:23 -04001309 entry->tgid = current->tgid;
Török Edwin02b67512008-11-22 13:28:47 +02001310 memset(&entry->caller, 0, sizeof(entry->caller));
1311
1312 trace.nr_entries = 0;
1313 trace.max_entries = FTRACE_STACK_ENTRIES;
1314 trace.skip = 0;
1315 trace.entries = entry->caller;
1316
1317 save_stack_trace_user(&trace);
Steven Rostedte77405a2009-09-02 14:17:06 -04001318 if (!filter_check_discard(call, entry, buffer, event))
1319 ring_buffer_unlock_commit(buffer, event);
Török Edwin02b67512008-11-22 13:28:47 +02001320}
1321
Hannes Eder4fd27352009-02-10 19:44:12 +01001322#ifdef UNUSED
1323static void __trace_userstack(struct trace_array *tr, unsigned long flags)
Török Edwin02b67512008-11-22 13:28:47 +02001324{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001325 ftrace_trace_userstack(tr, flags, preempt_count());
Török Edwin02b67512008-11-22 13:28:47 +02001326}
Hannes Eder4fd27352009-02-10 19:44:12 +01001327#endif /* UNUSED */
Török Edwin02b67512008-11-22 13:28:47 +02001328
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001329#endif /* CONFIG_STACKTRACE */
1330
Steven Rostedt38697052008-10-01 13:14:09 -04001331static void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001332ftrace_trace_special(void *__tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001333 unsigned long arg1, unsigned long arg2, unsigned long arg3,
1334 int pc)
Ingo Molnara4feb832008-05-12 21:21:02 +02001335{
Steven Rostedt60ba7702009-09-12 23:34:04 -04001336 struct ftrace_event_call *call = &event_special;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001337 struct ring_buffer_event *event;
Ingo Molnara4feb832008-05-12 21:21:02 +02001338 struct trace_array *tr = __tr;
Steven Rostedte77405a2009-09-02 14:17:06 -04001339 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt777e2082008-09-29 23:02:42 -04001340 struct special_entry *entry;
Ingo Molnara4feb832008-05-12 21:21:02 +02001341
Steven Rostedte77405a2009-09-02 14:17:06 -04001342 event = trace_buffer_lock_reserve(buffer, TRACE_SPECIAL,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001343 sizeof(*entry), 0, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001344 if (!event)
1345 return;
1346 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001347 entry->arg1 = arg1;
1348 entry->arg2 = arg2;
1349 entry->arg3 = arg3;
Steven Rostedt60ba7702009-09-12 23:34:04 -04001350
1351 if (!filter_check_discard(call, entry, buffer, event))
1352 trace_buffer_unlock_commit(buffer, event, 0, pc);
Ingo Molnara4feb832008-05-12 21:21:02 +02001353}
1354
1355void
Steven Rostedt38697052008-10-01 13:14:09 -04001356__trace_special(void *__tr, void *__data,
1357 unsigned long arg1, unsigned long arg2, unsigned long arg3)
1358{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001359 ftrace_trace_special(__tr, arg1, arg2, arg3, preempt_count());
Steven Rostedt38697052008-10-01 13:14:09 -04001360}
1361
1362void
Steven Rostedt4902f882008-05-22 00:22:18 -04001363ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3)
1364{
1365 struct trace_array *tr = &global_trace;
1366 struct trace_array_cpu *data;
Steven Rostedt5aa1ba62008-11-10 23:07:30 -05001367 unsigned long flags;
Steven Rostedt4902f882008-05-22 00:22:18 -04001368 int cpu;
Steven Rostedt38697052008-10-01 13:14:09 -04001369 int pc;
Steven Rostedt4902f882008-05-22 00:22:18 -04001370
Steven Rostedtc76f0692008-11-07 22:36:02 -05001371 if (tracing_disabled)
Steven Rostedt4902f882008-05-22 00:22:18 -04001372 return;
1373
Steven Rostedt38697052008-10-01 13:14:09 -04001374 pc = preempt_count();
Steven Rostedt5aa1ba62008-11-10 23:07:30 -05001375 local_irq_save(flags);
Steven Rostedt4902f882008-05-22 00:22:18 -04001376 cpu = raw_smp_processor_id();
1377 data = tr->data[cpu];
Steven Rostedt4902f882008-05-22 00:22:18 -04001378
Steven Rostedt5aa1ba62008-11-10 23:07:30 -05001379 if (likely(atomic_inc_return(&data->disabled) == 1))
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001380 ftrace_trace_special(tr, arg1, arg2, arg3, pc);
Steven Rostedt4902f882008-05-22 00:22:18 -04001381
Steven Rostedt5aa1ba62008-11-10 23:07:30 -05001382 atomic_dec(&data->disabled);
1383 local_irq_restore(flags);
Steven Rostedt4902f882008-05-22 00:22:18 -04001384}
1385
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001386/**
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001387 * trace_vbprintk - write binary msg to tracing buffer
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001388 *
1389 */
Steven Rostedt40ce74f2009-03-19 14:03:53 -04001390int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001391{
Thomas Gleixner445c8952009-12-02 19:49:50 +01001392 static arch_spinlock_t trace_buf_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001393 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001394 static u32 trace_buf[TRACE_BUF_SIZE];
1395
Tom Zanussie1112b42009-03-31 00:48:49 -05001396 struct ftrace_event_call *call = &event_bprint;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001397 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001398 struct ring_buffer *buffer;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001399 struct trace_array *tr = &global_trace;
1400 struct trace_array_cpu *data;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001401 struct bprint_entry *entry;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001402 unsigned long flags;
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001403 int disable;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001404 int resched;
1405 int cpu, len = 0, size, pc;
1406
1407 if (unlikely(tracing_selftest_running || tracing_disabled))
1408 return 0;
1409
1410 /* Don't pollute graph traces with trace_vprintk internals */
1411 pause_graph_tracing();
1412
1413 pc = preempt_count();
1414 resched = ftrace_preempt_disable();
1415 cpu = raw_smp_processor_id();
1416 data = tr->data[cpu];
1417
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001418 disable = atomic_inc_return(&data->disabled);
1419 if (unlikely(disable != 1))
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001420 goto out;
1421
Steven Rostedt80370cb2009-03-10 17:16:35 -04001422 /* Lockdep uses trace_printk for lock tracing */
1423 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001424 arch_spin_lock(&trace_buf_lock);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001425 len = vbin_printf(trace_buf, TRACE_BUF_SIZE, fmt, args);
1426
1427 if (len > TRACE_BUF_SIZE || len < 0)
1428 goto out_unlock;
1429
1430 size = sizeof(*entry) + sizeof(u32) * len;
Steven Rostedte77405a2009-09-02 14:17:06 -04001431 buffer = tr->buffer;
1432 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1433 flags, pc);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001434 if (!event)
1435 goto out_unlock;
1436 entry = ring_buffer_event_data(event);
1437 entry->ip = ip;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001438 entry->fmt = fmt;
1439
1440 memcpy(entry->buf, trace_buf, sizeof(u32) * len);
Steven Rostedtd9313692010-01-06 17:27:11 -05001441 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedte77405a2009-09-02 14:17:06 -04001442 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtd9313692010-01-06 17:27:11 -05001443 ftrace_trace_stack(buffer, flags, 6, pc);
1444 }
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001445
1446out_unlock:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001447 arch_spin_unlock(&trace_buf_lock);
Steven Rostedt80370cb2009-03-10 17:16:35 -04001448 local_irq_restore(flags);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001449
1450out:
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001451 atomic_dec_return(&data->disabled);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001452 ftrace_preempt_enable(resched);
1453 unpause_graph_tracing();
1454
1455 return len;
1456}
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001457EXPORT_SYMBOL_GPL(trace_vbprintk);
1458
Steven Rostedt659372d2009-09-03 19:11:07 -04001459int trace_array_printk(struct trace_array *tr,
1460 unsigned long ip, const char *fmt, ...)
1461{
1462 int ret;
1463 va_list ap;
1464
1465 if (!(trace_flags & TRACE_ITER_PRINTK))
1466 return 0;
1467
1468 va_start(ap, fmt);
1469 ret = trace_array_vprintk(tr, ip, fmt, ap);
1470 va_end(ap);
1471 return ret;
1472}
1473
1474int trace_array_vprintk(struct trace_array *tr,
1475 unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001476{
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001477 static arch_spinlock_t trace_buf_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001478 static char trace_buf[TRACE_BUF_SIZE];
1479
Tom Zanussie1112b42009-03-31 00:48:49 -05001480 struct ftrace_event_call *call = &event_print;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001481 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001482 struct ring_buffer *buffer;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001483 struct trace_array_cpu *data;
1484 int cpu, len = 0, size, pc;
1485 struct print_entry *entry;
1486 unsigned long irq_flags;
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001487 int disable;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001488
1489 if (tracing_disabled || tracing_selftest_running)
1490 return 0;
1491
1492 pc = preempt_count();
1493 preempt_disable_notrace();
1494 cpu = raw_smp_processor_id();
1495 data = tr->data[cpu];
1496
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001497 disable = atomic_inc_return(&data->disabled);
1498 if (unlikely(disable != 1))
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001499 goto out;
1500
1501 pause_graph_tracing();
1502 raw_local_irq_save(irq_flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001503 arch_spin_lock(&trace_buf_lock);
Carsten Emdef2942482009-12-06 14:02:44 +01001504 len = vsnprintf(trace_buf, TRACE_BUF_SIZE, fmt, args);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001505
1506 size = sizeof(*entry) + len + 1;
Steven Rostedte77405a2009-09-02 14:17:06 -04001507 buffer = tr->buffer;
1508 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1509 irq_flags, pc);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001510 if (!event)
1511 goto out_unlock;
1512 entry = ring_buffer_event_data(event);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001513 entry->ip = ip;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001514
1515 memcpy(&entry->buf, trace_buf, len);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001516 entry->buf[len] = '\0';
Steven Rostedtd9313692010-01-06 17:27:11 -05001517 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedte77405a2009-09-02 14:17:06 -04001518 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtd9313692010-01-06 17:27:11 -05001519 ftrace_trace_stack(buffer, irq_flags, 6, pc);
1520 }
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001521
1522 out_unlock:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001523 arch_spin_unlock(&trace_buf_lock);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001524 raw_local_irq_restore(irq_flags);
1525 unpause_graph_tracing();
1526 out:
Steven Rostedt3189cdb2009-04-17 16:13:55 -04001527 atomic_dec_return(&data->disabled);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001528 preempt_enable_notrace();
1529
1530 return len;
1531}
Steven Rostedt659372d2009-09-03 19:11:07 -04001532
1533int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1534{
Steven Rostedta813a152009-10-09 01:41:35 -04001535 return trace_array_vprintk(&global_trace, ip, fmt, args);
Steven Rostedt659372d2009-09-03 19:11:07 -04001536}
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001537EXPORT_SYMBOL_GPL(trace_vprintk);
1538
Robert Richtere2ac8ef2008-11-12 12:59:32 +01001539static void trace_iterator_increment(struct trace_iterator *iter)
Steven Rostedt5a90f572008-09-03 17:42:51 -04001540{
Steven Rostedtd7690412008-10-01 00:29:53 -04001541 /* Don't allow ftrace to trace into the ring buffers */
1542 ftrace_disable_cpu();
1543
Steven Rostedt5a90f572008-09-03 17:42:51 -04001544 iter->idx++;
Steven Rostedtd7690412008-10-01 00:29:53 -04001545 if (iter->buffer_iter[iter->cpu])
1546 ring_buffer_read(iter->buffer_iter[iter->cpu], NULL);
1547
1548 ftrace_enable_cpu();
Steven Rostedt5a90f572008-09-03 17:42:51 -04001549}
1550
Ingo Molnare309b412008-05-12 21:20:51 +02001551static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04001552peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1553 unsigned long *lost_events)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001554{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001555 struct ring_buffer_event *event;
1556 struct ring_buffer_iter *buf_iter = iter->buffer_iter[cpu];
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001557
Steven Rostedtd7690412008-10-01 00:29:53 -04001558 /* Don't allow ftrace to trace into the ring buffers */
1559 ftrace_disable_cpu();
1560
1561 if (buf_iter)
1562 event = ring_buffer_iter_peek(buf_iter, ts);
1563 else
Steven Rostedtbc21b472010-03-31 19:49:26 -04001564 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1565 lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04001566
1567 ftrace_enable_cpu();
1568
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001569 return event ? ring_buffer_event_data(event) : NULL;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001570}
Steven Rostedtd7690412008-10-01 00:29:53 -04001571
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001572static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04001573__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1574 unsigned long *missing_events, u64 *ent_ts)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001575{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001576 struct ring_buffer *buffer = iter->tr->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001577 struct trace_entry *ent, *next = NULL;
Lai Jiangshanaa274972010-04-05 17:11:05 +08001578 unsigned long lost_events = 0, next_lost = 0;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001579 int cpu_file = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001580 u64 next_ts = 0, ts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001581 int next_cpu = -1;
1582 int cpu;
1583
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001584 /*
1585 * If we are in a per_cpu trace file, don't bother by iterating over
1586 * all cpu and peek directly.
1587 */
1588 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1589 if (ring_buffer_empty_cpu(buffer, cpu_file))
1590 return NULL;
Steven Rostedtbc21b472010-03-31 19:49:26 -04001591 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001592 if (ent_cpu)
1593 *ent_cpu = cpu_file;
1594
1595 return ent;
1596 }
1597
Steven Rostedtab464282008-05-12 21:21:00 +02001598 for_each_tracing_cpu(cpu) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001599
1600 if (ring_buffer_empty_cpu(buffer, cpu))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001601 continue;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001602
Steven Rostedtbc21b472010-03-31 19:49:26 -04001603 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001604
Ingo Molnarcdd31cd2008-05-12 21:20:46 +02001605 /*
1606 * Pick the entry with the smallest timestamp:
1607 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001608 if (ent && (!next || ts < next_ts)) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001609 next = ent;
1610 next_cpu = cpu;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001611 next_ts = ts;
Steven Rostedtbc21b472010-03-31 19:49:26 -04001612 next_lost = lost_events;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001613 }
1614 }
1615
1616 if (ent_cpu)
1617 *ent_cpu = next_cpu;
1618
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001619 if (ent_ts)
1620 *ent_ts = next_ts;
1621
Steven Rostedtbc21b472010-03-31 19:49:26 -04001622 if (missing_events)
1623 *missing_events = next_lost;
1624
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001625 return next;
1626}
1627
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001628/* Find the next real entry, without updating the iterator itself */
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001629struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1630 int *ent_cpu, u64 *ent_ts)
Steven Rostedtb3806b42008-05-12 21:20:46 +02001631{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001632 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001633}
Ingo Molnar8c523a92008-05-12 21:20:46 +02001634
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001635/* Find the next real entry, and increment the iterator to the next entry */
Jason Wessel955b61e2010-08-05 09:22:23 -05001636void *trace_find_next_entry_inc(struct trace_iterator *iter)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001637{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001638 iter->ent = __find_next_entry(iter, &iter->cpu,
1639 &iter->lost_events, &iter->ts);
Steven Rostedtb3806b42008-05-12 21:20:46 +02001640
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001641 if (iter->ent)
Robert Richtere2ac8ef2008-11-12 12:59:32 +01001642 trace_iterator_increment(iter);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001643
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001644 return iter->ent ? iter : NULL;
Steven Rostedtb3806b42008-05-12 21:20:46 +02001645}
1646
Ingo Molnare309b412008-05-12 21:20:51 +02001647static void trace_consume(struct trace_iterator *iter)
Steven Rostedtb3806b42008-05-12 21:20:46 +02001648{
Steven Rostedtd7690412008-10-01 00:29:53 -04001649 /* Don't allow ftrace to trace into the ring buffers */
1650 ftrace_disable_cpu();
Steven Rostedtbc21b472010-03-31 19:49:26 -04001651 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1652 &iter->lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04001653 ftrace_enable_cpu();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001654}
1655
Ingo Molnare309b412008-05-12 21:20:51 +02001656static void *s_next(struct seq_file *m, void *v, loff_t *pos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001657{
1658 struct trace_iterator *iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001659 int i = (int)*pos;
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001660 void *ent;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001661
Steven Rostedta63ce5b2009-12-07 09:11:39 -05001662 WARN_ON_ONCE(iter->leftover);
1663
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001664 (*pos)++;
1665
1666 /* can't go backwards */
1667 if (iter->idx > i)
1668 return NULL;
1669
1670 if (iter->idx < 0)
Jason Wessel955b61e2010-08-05 09:22:23 -05001671 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001672 else
1673 ent = iter;
1674
1675 while (ent && iter->idx < i)
Jason Wessel955b61e2010-08-05 09:22:23 -05001676 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001677
1678 iter->pos = *pos;
1679
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001680 return ent;
1681}
1682
Jason Wessel955b61e2010-08-05 09:22:23 -05001683void tracing_iter_reset(struct trace_iterator *iter, int cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001684{
1685 struct trace_array *tr = iter->tr;
1686 struct ring_buffer_event *event;
1687 struct ring_buffer_iter *buf_iter;
1688 unsigned long entries = 0;
1689 u64 ts;
1690
1691 tr->data[cpu]->skipped_entries = 0;
1692
1693 if (!iter->buffer_iter[cpu])
1694 return;
1695
1696 buf_iter = iter->buffer_iter[cpu];
1697 ring_buffer_iter_reset(buf_iter);
1698
1699 /*
1700 * We could have the case with the max latency tracers
1701 * that a reset never took place on a cpu. This is evident
1702 * by the timestamp being before the start of the buffer.
1703 */
1704 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1705 if (ts >= iter->tr->time_start)
1706 break;
1707 entries++;
1708 ring_buffer_read(buf_iter, NULL);
1709 }
1710
1711 tr->data[cpu]->skipped_entries = entries;
1712}
1713
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001714/*
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001715 * The current tracer is copied to avoid a global locking
1716 * all around.
1717 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001718static void *s_start(struct seq_file *m, loff_t *pos)
1719{
1720 struct trace_iterator *iter = m->private;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001721 static struct tracer *old_tracer;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001722 int cpu_file = iter->cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001723 void *p = NULL;
1724 loff_t l = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001725 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001726
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001727 /* copy the tracer to avoid using a global lock all around */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001728 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001729 if (unlikely(old_tracer != current_trace && current_trace)) {
1730 old_tracer = current_trace;
1731 *iter->trace = *current_trace;
Steven Rostedtd15f57f2008-05-12 21:20:56 +02001732 }
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001733 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001734
1735 atomic_inc(&trace_record_cmdline_disabled);
1736
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001737 if (*pos != iter->pos) {
1738 iter->ent = NULL;
1739 iter->cpu = 0;
1740 iter->idx = -1;
1741
Steven Rostedtd7690412008-10-01 00:29:53 -04001742 ftrace_disable_cpu();
1743
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001744 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1745 for_each_tracing_cpu(cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001746 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001747 } else
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001748 tracing_iter_reset(iter, cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001749
Steven Rostedtd7690412008-10-01 00:29:53 -04001750 ftrace_enable_cpu();
1751
Lai Jiangshanac91d852010-03-02 17:54:50 +08001752 iter->leftover = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001753 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1754 ;
1755
1756 } else {
Steven Rostedta63ce5b2009-12-07 09:11:39 -05001757 /*
1758 * If we overflowed the seq_file before, then we want
1759 * to just reuse the trace_seq buffer again.
1760 */
1761 if (iter->leftover)
1762 p = iter;
1763 else {
1764 l = *pos - 1;
1765 p = s_next(m, p, &l);
1766 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001767 }
1768
Lai Jiangshan4f535962009-05-18 19:35:34 +08001769 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001770 trace_access_lock(cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001771 return p;
1772}
1773
1774static void s_stop(struct seq_file *m, void *p)
1775{
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001776 struct trace_iterator *iter = m->private;
1777
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001778 atomic_dec(&trace_record_cmdline_disabled);
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001779 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08001780 trace_event_read_unlock();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001781}
1782
Ingo Molnare309b412008-05-12 21:20:51 +02001783static void print_lat_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001784{
Michael Ellermana6168352008-08-20 16:36:11 -07001785 seq_puts(m, "# _------=> CPU# \n");
1786 seq_puts(m, "# / _-----=> irqs-off \n");
1787 seq_puts(m, "# | / _----=> need-resched \n");
1788 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1789 seq_puts(m, "# ||| / _--=> preempt-depth \n");
Steven Rostedt637e7e82009-09-11 13:55:35 -04001790 seq_puts(m, "# |||| /_--=> lock-depth \n");
1791 seq_puts(m, "# |||||/ delay \n");
1792 seq_puts(m, "# cmd pid |||||| time | caller \n");
1793 seq_puts(m, "# \\ / |||||| \\ | / \n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001794}
1795
Ingo Molnare309b412008-05-12 21:20:51 +02001796static void print_func_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001797{
Michael Ellermana6168352008-08-20 16:36:11 -07001798 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
1799 seq_puts(m, "# | | | | |\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001800}
1801
1802
Jiri Olsa62b915f2010-04-02 19:01:22 +02001803void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001804print_trace_header(struct seq_file *m, struct trace_iterator *iter)
1805{
1806 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
1807 struct trace_array *tr = iter->tr;
1808 struct trace_array_cpu *data = tr->data[tr->cpu];
1809 struct tracer *type = current_trace;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001810 unsigned long entries = 0;
1811 unsigned long total = 0;
1812 unsigned long count;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001813 const char *name = "preemption";
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001814 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001815
1816 if (type)
1817 name = type->name;
1818
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001819
1820 for_each_tracing_cpu(cpu) {
1821 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1822 /*
1823 * If this buffer has skipped entries, then we hold all
1824 * entries for the trace and we need to ignore the
1825 * ones before the time stamp.
1826 */
1827 if (tr->data[cpu]->skipped_entries) {
1828 count -= tr->data[cpu]->skipped_entries;
1829 /* total is the same as the entries */
1830 total += count;
1831 } else
1832 total += count +
1833 ring_buffer_overrun_cpu(tr->buffer, cpu);
1834 entries += count;
1835 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001836
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001837 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001838 name, UTS_RELEASE);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001839 seq_puts(m, "# -----------------------------------"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001840 "---------------------------------\n");
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001841 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001842 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
Steven Rostedt57f50be2008-05-12 21:20:44 +02001843 nsecs_to_usecs(data->saved_latency),
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001844 entries,
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02001845 total,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001846 tr->cpu,
1847#if defined(CONFIG_PREEMPT_NONE)
1848 "server",
1849#elif defined(CONFIG_PREEMPT_VOLUNTARY)
1850 "desktop",
Steven Rostedtb5c21b42008-07-10 20:58:12 -04001851#elif defined(CONFIG_PREEMPT)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001852 "preempt",
1853#else
1854 "unknown",
1855#endif
1856 /* These are reserved for later use */
1857 0, 0, 0, 0);
1858#ifdef CONFIG_SMP
1859 seq_printf(m, " #P:%d)\n", num_online_cpus());
1860#else
1861 seq_puts(m, ")\n");
1862#endif
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001863 seq_puts(m, "# -----------------\n");
1864 seq_printf(m, "# | task: %.16s-%d "
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001865 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
1866 data->comm, data->pid, data->uid, data->nice,
1867 data->policy, data->rt_priority);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001868 seq_puts(m, "# -----------------\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001869
1870 if (data->critical_start) {
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001871 seq_puts(m, "# => started at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02001872 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
1873 trace_print_seq(m, &iter->seq);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001874 seq_puts(m, "\n# => ended at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02001875 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
1876 trace_print_seq(m, &iter->seq);
Steven Rostedt8248ac02009-09-02 12:27:41 -04001877 seq_puts(m, "\n#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001878 }
1879
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09001880 seq_puts(m, "#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001881}
1882
Steven Rostedta3097202008-11-07 22:36:02 -05001883static void test_cpu_buff_start(struct trace_iterator *iter)
1884{
1885 struct trace_seq *s = &iter->seq;
1886
Steven Rostedt12ef7d42008-11-12 17:52:38 -05001887 if (!(trace_flags & TRACE_ITER_ANNOTATE))
1888 return;
1889
1890 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
1891 return;
1892
Rusty Russell44623442009-01-01 10:12:23 +10301893 if (cpumask_test_cpu(iter->cpu, iter->started))
Steven Rostedta3097202008-11-07 22:36:02 -05001894 return;
1895
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001896 if (iter->tr->data[iter->cpu]->skipped_entries)
1897 return;
1898
Rusty Russell44623442009-01-01 10:12:23 +10301899 cpumask_set_cpu(iter->cpu, iter->started);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02001900
1901 /* Don't print started cpu buffer for the first entry of the trace */
1902 if (iter->idx > 1)
1903 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
1904 iter->cpu);
Steven Rostedta3097202008-11-07 22:36:02 -05001905}
1906
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001907static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001908{
Steven Rostedt214023c2008-05-12 21:20:46 +02001909 struct trace_seq *s = &iter->seq;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001910 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001911 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05001912 struct trace_event *event;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001913
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001914 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001915
Steven Rostedta3097202008-11-07 22:36:02 -05001916 test_cpu_buff_start(iter);
1917
Steven Rostedtf633cef2008-12-23 23:24:13 -05001918 event = ftrace_find_event(entry->type);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001919
1920 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt27d48be2009-03-04 21:57:29 -05001921 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
1922 if (!trace_print_lat_context(iter))
1923 goto partial;
1924 } else {
1925 if (!trace_print_context(iter))
1926 goto partial;
1927 }
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001928 }
1929
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02001930 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04001931 return event->funcs->trace(iter, sym_flags, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001932
1933 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
1934 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04001935
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001936 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001937partial:
1938 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001939}
1940
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001941static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02001942{
1943 struct trace_seq *s = &iter->seq;
1944 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05001945 struct trace_event *event;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02001946
1947 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001948
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001949 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001950 if (!trace_seq_printf(s, "%d %d %llu ",
1951 entry->pid, iter->cpu, iter->ts))
1952 goto partial;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001953 }
Ingo Molnarf9896bf2008-05-12 21:20:47 +02001954
Steven Rostedtf633cef2008-12-23 23:24:13 -05001955 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02001956 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04001957 return event->funcs->raw(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001958
1959 if (!trace_seq_printf(s, "%d ?\n", entry->type))
1960 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04001961
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001962 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001963partial:
1964 return TRACE_TYPE_PARTIAL_LINE;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02001965}
1966
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001967static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02001968{
1969 struct trace_seq *s = &iter->seq;
1970 unsigned char newline = '\n';
1971 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05001972 struct trace_event *event;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02001973
1974 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001975
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001976 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
1977 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
1978 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
1979 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
1980 }
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02001981
Steven Rostedtf633cef2008-12-23 23:24:13 -05001982 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02001983 if (event) {
Steven Rostedta9a57762010-04-22 18:46:14 -04001984 enum print_line_t ret = event->funcs->hex(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02001985 if (ret != TRACE_TYPE_HANDLED)
1986 return ret;
1987 }
Steven Rostedt7104f302008-10-01 10:52:51 -04001988
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02001989 SEQ_PUT_FIELD_RET(s, newline);
1990
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001991 return TRACE_TYPE_HANDLED;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02001992}
1993
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02001994static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02001995{
1996 struct trace_seq *s = &iter->seq;
1997 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05001998 struct trace_event *event;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02001999
2000 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002001
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002002 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2003 SEQ_PUT_FIELD_RET(s, entry->pid);
Steven Rostedt1830b52d2009-02-07 19:38:43 -05002004 SEQ_PUT_FIELD_RET(s, iter->cpu);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002005 SEQ_PUT_FIELD_RET(s, iter->ts);
2006 }
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002007
Steven Rostedtf633cef2008-12-23 23:24:13 -05002008 event = ftrace_find_event(entry->type);
Steven Rostedta9a57762010-04-22 18:46:14 -04002009 return event ? event->funcs->binary(iter, 0, event) :
2010 TRACE_TYPE_HANDLED;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002011}
2012
Jiri Olsa62b915f2010-04-02 19:01:22 +02002013int trace_empty(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002014{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002015 int cpu;
2016
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002017 /* If we are looking at one CPU buffer, only check that one */
2018 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2019 cpu = iter->cpu_file;
2020 if (iter->buffer_iter[cpu]) {
2021 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2022 return 0;
2023 } else {
2024 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2025 return 0;
2026 }
2027 return 1;
2028 }
2029
Steven Rostedtab464282008-05-12 21:21:00 +02002030 for_each_tracing_cpu(cpu) {
Steven Rostedtd7690412008-10-01 00:29:53 -04002031 if (iter->buffer_iter[cpu]) {
2032 if (!ring_buffer_iter_empty(iter->buffer_iter[cpu]))
2033 return 0;
2034 } else {
2035 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2036 return 0;
2037 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002038 }
Steven Rostedtd7690412008-10-01 00:29:53 -04002039
Frederic Weisbecker797d3712008-09-30 18:13:45 +02002040 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002041}
2042
Lai Jiangshan4f535962009-05-18 19:35:34 +08002043/* Called with trace_event_read_lock() held. */
Jason Wessel955b61e2010-08-05 09:22:23 -05002044enum print_line_t print_trace_line(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002045{
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002046 enum print_line_t ret;
2047
Steven Rostedtbc21b472010-03-31 19:49:26 -04002048 if (iter->lost_events)
2049 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2050 iter->cpu, iter->lost_events);
2051
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002052 if (iter->trace && iter->trace->print_line) {
2053 ret = iter->trace->print_line(iter);
2054 if (ret != TRACE_TYPE_UNHANDLED)
2055 return ret;
2056 }
Thomas Gleixner72829bc2008-05-23 21:37:28 +02002057
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002058 if (iter->ent->type == TRACE_BPRINT &&
2059 trace_flags & TRACE_ITER_PRINTK &&
2060 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002061 return trace_print_bprintk_msg_only(iter);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002062
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002063 if (iter->ent->type == TRACE_PRINT &&
2064 trace_flags & TRACE_ITER_PRINTK &&
2065 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002066 return trace_print_printk_msg_only(iter);
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002067
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002068 if (trace_flags & TRACE_ITER_BIN)
2069 return print_bin_fmt(iter);
2070
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002071 if (trace_flags & TRACE_ITER_HEX)
2072 return print_hex_fmt(iter);
2073
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002074 if (trace_flags & TRACE_ITER_RAW)
2075 return print_raw_fmt(iter);
2076
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002077 return print_trace_fmt(iter);
2078}
2079
Jiri Olsa62b915f2010-04-02 19:01:22 +02002080void trace_default_header(struct seq_file *m)
2081{
2082 struct trace_iterator *iter = m->private;
2083
2084 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2085 /* print nothing if the buffers are empty */
2086 if (trace_empty(iter))
2087 return;
2088 print_trace_header(m, iter);
2089 if (!(trace_flags & TRACE_ITER_VERBOSE))
2090 print_lat_help_header(m);
2091 } else {
2092 if (!(trace_flags & TRACE_ITER_VERBOSE))
2093 print_func_help_header(m);
2094 }
2095}
2096
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002097static int s_show(struct seq_file *m, void *v)
2098{
2099 struct trace_iterator *iter = v;
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002100 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002101
2102 if (iter->ent == NULL) {
2103 if (iter->tr) {
2104 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2105 seq_puts(m, "#\n");
2106 }
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002107 if (iter->trace && iter->trace->print_header)
2108 iter->trace->print_header(m);
Jiri Olsa62b915f2010-04-02 19:01:22 +02002109 else
2110 trace_default_header(m);
2111
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002112 } else if (iter->leftover) {
2113 /*
2114 * If we filled the seq_file buffer earlier, we
2115 * want to just show it now.
2116 */
2117 ret = trace_print_seq(m, &iter->seq);
2118
2119 /* ret should this time be zero, but you never know */
2120 iter->leftover = ret;
2121
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002122 } else {
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002123 print_trace_line(iter);
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002124 ret = trace_print_seq(m, &iter->seq);
2125 /*
2126 * If we overflow the seq_file buffer, then it will
2127 * ask us for this data again at start up.
2128 * Use that instead.
2129 * ret is 0 if seq_file write succeeded.
2130 * -1 otherwise.
2131 */
2132 iter->leftover = ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002133 }
2134
2135 return 0;
2136}
2137
James Morris88e9d342009-09-22 16:43:43 -07002138static const struct seq_operations tracer_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002139 .start = s_start,
2140 .next = s_next,
2141 .stop = s_stop,
2142 .show = s_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002143};
2144
Ingo Molnare309b412008-05-12 21:20:51 +02002145static struct trace_iterator *
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002146__tracing_open(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002147{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002148 long cpu_file = (long) inode->i_private;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002149 void *fail_ret = ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002150 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002151 struct seq_file *m;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002152 int cpu, ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002153
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002154 if (tracing_disabled)
2155 return ERR_PTR(-ENODEV);
Steven Rostedt60a11772008-05-12 21:20:44 +02002156
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002157 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002158 if (!iter)
2159 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002160
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002161 /*
2162 * We make a copy of the current tracer to avoid concurrent
2163 * changes on it while we are reading.
2164 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002165 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002166 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002167 if (!iter->trace)
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002168 goto fail;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002169
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002170 if (current_trace)
2171 *iter->trace = *current_trace;
2172
Li Zefan79f55992009-06-15 14:58:26 +08002173 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002174 goto fail;
2175
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002176 if (current_trace && current_trace->print_max)
2177 iter->tr = &max_tr;
2178 else
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002179 iter->tr = &global_trace;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002180 iter->pos = -1;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002181 mutex_init(&iter->mutex);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002182 iter->cpu_file = cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002183
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002184 /* Notify the tracer early; before we stop tracing. */
2185 if (iter->trace && iter->trace->open)
Markus Metzgera93751c2008-12-11 13:53:26 +01002186 iter->trace->open(iter);
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002187
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002188 /* Annotate start of buffers if we had overruns */
2189 if (ring_buffer_overruns(iter->tr->buffer))
2190 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2191
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002192 /* stop the trace while dumping */
2193 tracing_stop();
2194
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002195 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2196 for_each_tracing_cpu(cpu) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002197 iter->buffer_iter[cpu] =
David Miller72c9ddf2010-04-20 15:47:11 -07002198 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2199 }
2200 ring_buffer_read_prepare_sync();
2201 for_each_tracing_cpu(cpu) {
2202 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002203 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002204 }
2205 } else {
2206 cpu = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002207 iter->buffer_iter[cpu] =
David Miller72c9ddf2010-04-20 15:47:11 -07002208 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2209 ring_buffer_read_prepare_sync();
2210 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002211 tracing_iter_reset(iter, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002212 }
2213
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002214 ret = seq_open(file, &tracer_seq_ops);
2215 if (ret < 0) {
2216 fail_ret = ERR_PTR(ret);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002217 goto fail_buffer;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002218 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002219
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002220 m = file->private_data;
2221 m->private = iter;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002222
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002223 mutex_unlock(&trace_types_lock);
2224
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002225 return iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002226
2227 fail_buffer:
2228 for_each_tracing_cpu(cpu) {
2229 if (iter->buffer_iter[cpu])
2230 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2231 }
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002232 free_cpumask_var(iter->started);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002233 tracing_start();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002234 fail:
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002235 mutex_unlock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002236 kfree(iter->trace);
Julia Lawall0bb943c2008-11-14 19:05:31 +01002237 kfree(iter);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002238
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002239 return fail_ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002240}
2241
2242int tracing_open_generic(struct inode *inode, struct file *filp)
2243{
Steven Rostedt60a11772008-05-12 21:20:44 +02002244 if (tracing_disabled)
2245 return -ENODEV;
2246
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002247 filp->private_data = inode->i_private;
2248 return 0;
2249}
2250
Hannes Eder4fd27352009-02-10 19:44:12 +01002251static int tracing_release(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002252{
2253 struct seq_file *m = (struct seq_file *)file->private_data;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002254 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002255 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002256
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002257 if (!(file->f_mode & FMODE_READ))
2258 return 0;
2259
2260 iter = m->private;
2261
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002262 mutex_lock(&trace_types_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002263 for_each_tracing_cpu(cpu) {
2264 if (iter->buffer_iter[cpu])
2265 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2266 }
2267
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002268 if (iter->trace && iter->trace->close)
2269 iter->trace->close(iter);
2270
2271 /* reenable tracing if it was previously enabled */
Steven Rostedt90369902008-11-05 16:05:44 -05002272 tracing_start();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002273 mutex_unlock(&trace_types_lock);
2274
2275 seq_release(inode, file);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002276 mutex_destroy(&iter->mutex);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002277 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002278 kfree(iter->trace);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002279 kfree(iter);
2280 return 0;
2281}
2282
2283static int tracing_open(struct inode *inode, struct file *file)
2284{
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002285 struct trace_iterator *iter;
2286 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002287
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002288 /* If this file was open for write, then erase contents */
2289 if ((file->f_mode & FMODE_WRITE) &&
Steven Rostedt8650ae32009-07-22 23:29:30 -04002290 (file->f_flags & O_TRUNC)) {
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002291 long cpu = (long) inode->i_private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002292
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002293 if (cpu == TRACE_PIPE_ALL_CPU)
2294 tracing_reset_online_cpus(&global_trace);
2295 else
2296 tracing_reset(&global_trace, cpu);
2297 }
2298
2299 if (file->f_mode & FMODE_READ) {
2300 iter = __tracing_open(inode, file);
2301 if (IS_ERR(iter))
2302 ret = PTR_ERR(iter);
2303 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2304 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2305 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002306 return ret;
2307}
2308
Ingo Molnare309b412008-05-12 21:20:51 +02002309static void *
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002310t_next(struct seq_file *m, void *v, loff_t *pos)
2311{
Li Zefanf129e962009-06-24 09:53:44 +08002312 struct tracer *t = v;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002313
2314 (*pos)++;
2315
2316 if (t)
2317 t = t->next;
2318
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002319 return t;
2320}
2321
2322static void *t_start(struct seq_file *m, loff_t *pos)
2323{
Li Zefanf129e962009-06-24 09:53:44 +08002324 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002325 loff_t l = 0;
2326
2327 mutex_lock(&trace_types_lock);
Li Zefanf129e962009-06-24 09:53:44 +08002328 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002329 ;
2330
2331 return t;
2332}
2333
2334static void t_stop(struct seq_file *m, void *p)
2335{
2336 mutex_unlock(&trace_types_lock);
2337}
2338
2339static int t_show(struct seq_file *m, void *v)
2340{
2341 struct tracer *t = v;
2342
2343 if (!t)
2344 return 0;
2345
2346 seq_printf(m, "%s", t->name);
2347 if (t->next)
2348 seq_putc(m, ' ');
2349 else
2350 seq_putc(m, '\n');
2351
2352 return 0;
2353}
2354
James Morris88e9d342009-09-22 16:43:43 -07002355static const struct seq_operations show_traces_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002356 .start = t_start,
2357 .next = t_next,
2358 .stop = t_stop,
2359 .show = t_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002360};
2361
2362static int show_traces_open(struct inode *inode, struct file *file)
2363{
Steven Rostedt60a11772008-05-12 21:20:44 +02002364 if (tracing_disabled)
2365 return -ENODEV;
2366
Li Zefanf129e962009-06-24 09:53:44 +08002367 return seq_open(file, &show_traces_seq_ops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002368}
2369
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002370static ssize_t
2371tracing_write_stub(struct file *filp, const char __user *ubuf,
2372 size_t count, loff_t *ppos)
2373{
2374 return count;
2375}
2376
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002377static const struct file_operations tracing_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002378 .open = tracing_open,
2379 .read = seq_read,
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002380 .write = tracing_write_stub,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002381 .llseek = seq_lseek,
2382 .release = tracing_release,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002383};
2384
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002385static const struct file_operations show_traces_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002386 .open = show_traces_open,
2387 .read = seq_read,
2388 .release = seq_release,
2389};
2390
Ingo Molnar36dfe922008-05-12 21:20:52 +02002391/*
2392 * Only trace on a CPU if the bitmask is set:
2393 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302394static cpumask_var_t tracing_cpumask;
Ingo Molnar36dfe922008-05-12 21:20:52 +02002395
2396/*
2397 * The tracer itself will not take this lock, but still we want
2398 * to provide a consistent cpumask to user-space:
2399 */
2400static DEFINE_MUTEX(tracing_cpumask_update_lock);
2401
2402/*
2403 * Temporary storage for the character representation of the
2404 * CPU bitmask (and one more byte for the newline):
2405 */
2406static char mask_str[NR_CPUS + 1];
2407
Ingo Molnarc7078de2008-05-12 21:20:52 +02002408static ssize_t
2409tracing_cpumask_read(struct file *filp, char __user *ubuf,
2410 size_t count, loff_t *ppos)
2411{
Ingo Molnar36dfe922008-05-12 21:20:52 +02002412 int len;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002413
2414 mutex_lock(&tracing_cpumask_update_lock);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002415
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302416 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002417 if (count - len < 2) {
2418 count = -EINVAL;
2419 goto out_err;
2420 }
2421 len += sprintf(mask_str + len, "\n");
2422 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2423
2424out_err:
Ingo Molnarc7078de2008-05-12 21:20:52 +02002425 mutex_unlock(&tracing_cpumask_update_lock);
2426
2427 return count;
2428}
2429
2430static ssize_t
2431tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2432 size_t count, loff_t *ppos)
2433{
Ingo Molnar36dfe922008-05-12 21:20:52 +02002434 int err, cpu;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302435 cpumask_var_t tracing_cpumask_new;
2436
2437 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2438 return -ENOMEM;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002439
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302440 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002441 if (err)
2442 goto err_unlock;
2443
Li Zefan215368e2009-06-15 10:56:42 +08002444 mutex_lock(&tracing_cpumask_update_lock);
2445
Steven Rostedta5e25882008-12-02 15:34:05 -05002446 local_irq_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01002447 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtab464282008-05-12 21:21:00 +02002448 for_each_tracing_cpu(cpu) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002449 /*
2450 * Increase/decrease the disabled counter if we are
2451 * about to flip a bit in the cpumask:
2452 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302453 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2454 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002455 atomic_inc(&global_trace.data[cpu]->disabled);
2456 }
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302457 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2458 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002459 atomic_dec(&global_trace.data[cpu]->disabled);
2460 }
2461 }
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01002462 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedta5e25882008-12-02 15:34:05 -05002463 local_irq_enable();
Ingo Molnar36dfe922008-05-12 21:20:52 +02002464
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302465 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002466
Ingo Molnarc7078de2008-05-12 21:20:52 +02002467 mutex_unlock(&tracing_cpumask_update_lock);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302468 free_cpumask_var(tracing_cpumask_new);
Ingo Molnarc7078de2008-05-12 21:20:52 +02002469
Ingo Molnarc7078de2008-05-12 21:20:52 +02002470 return count;
Ingo Molnar36dfe922008-05-12 21:20:52 +02002471
2472err_unlock:
Li Zefan215368e2009-06-15 10:56:42 +08002473 free_cpumask_var(tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002474
2475 return err;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002476}
2477
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002478static const struct file_operations tracing_cpumask_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002479 .open = tracing_open_generic,
2480 .read = tracing_cpumask_read,
2481 .write = tracing_cpumask_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002482};
2483
Li Zefanfdb372e2009-12-08 11:15:59 +08002484static int tracing_trace_options_show(struct seq_file *m, void *v)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002485{
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002486 struct tracer_opt *trace_opts;
2487 u32 tracer_flags;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002488 int i;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002489
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002490 mutex_lock(&trace_types_lock);
2491 tracer_flags = current_trace->flags->val;
2492 trace_opts = current_trace->flags->opts;
2493
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002494 for (i = 0; trace_options[i]; i++) {
2495 if (trace_flags & (1 << i))
Li Zefanfdb372e2009-12-08 11:15:59 +08002496 seq_printf(m, "%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002497 else
Li Zefanfdb372e2009-12-08 11:15:59 +08002498 seq_printf(m, "no%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002499 }
2500
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002501 for (i = 0; trace_opts[i].name; i++) {
2502 if (tracer_flags & trace_opts[i].bit)
Li Zefanfdb372e2009-12-08 11:15:59 +08002503 seq_printf(m, "%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002504 else
Li Zefanfdb372e2009-12-08 11:15:59 +08002505 seq_printf(m, "no%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002506 }
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002507 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002508
Li Zefanfdb372e2009-12-08 11:15:59 +08002509 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002510}
2511
Li Zefan8d18eaa2009-12-08 11:17:06 +08002512static int __set_tracer_option(struct tracer *trace,
2513 struct tracer_flags *tracer_flags,
2514 struct tracer_opt *opts, int neg)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002515{
Li Zefan8d18eaa2009-12-08 11:17:06 +08002516 int ret;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002517
Li Zefan8d18eaa2009-12-08 11:17:06 +08002518 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002519 if (ret)
2520 return ret;
2521
2522 if (neg)
Zhaolei77708412009-08-07 18:53:21 +08002523 tracer_flags->val &= ~opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002524 else
Zhaolei77708412009-08-07 18:53:21 +08002525 tracer_flags->val |= opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002526 return 0;
2527}
2528
Li Zefan8d18eaa2009-12-08 11:17:06 +08002529/* Try to assign a tracer specific option */
2530static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2531{
2532 struct tracer_flags *tracer_flags = trace->flags;
2533 struct tracer_opt *opts = NULL;
2534 int i;
2535
2536 for (i = 0; tracer_flags->opts[i].name; i++) {
2537 opts = &tracer_flags->opts[i];
2538
2539 if (strcmp(cmp, opts->name) == 0)
2540 return __set_tracer_option(trace, trace->flags,
2541 opts, neg);
2542 }
2543
2544 return -EINVAL;
2545}
2546
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002547static void set_tracer_flags(unsigned int mask, int enabled)
2548{
2549 /* do nothing if flag is already set */
2550 if (!!(trace_flags & mask) == !!enabled)
2551 return;
2552
2553 if (enabled)
2554 trace_flags |= mask;
2555 else
2556 trace_flags &= ~mask;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002557}
2558
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002559static ssize_t
Steven Rostedtee6bce52008-11-12 17:52:37 -05002560tracing_trace_options_write(struct file *filp, const char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002561 size_t cnt, loff_t *ppos)
2562{
2563 char buf[64];
Li Zefan8d18eaa2009-12-08 11:17:06 +08002564 char *cmp;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002565 int neg = 0;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002566 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002567 int i;
2568
Steven Rostedtcffae432008-05-12 21:21:00 +02002569 if (cnt >= sizeof(buf))
2570 return -EINVAL;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002571
2572 if (copy_from_user(&buf, ubuf, cnt))
2573 return -EFAULT;
2574
2575 buf[cnt] = 0;
Li Zefan8d18eaa2009-12-08 11:17:06 +08002576 cmp = strstrip(buf);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002577
Li Zefan8d18eaa2009-12-08 11:17:06 +08002578 if (strncmp(cmp, "no", 2) == 0) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002579 neg = 1;
2580 cmp += 2;
2581 }
2582
2583 for (i = 0; trace_options[i]; i++) {
Li Zefan8d18eaa2009-12-08 11:17:06 +08002584 if (strcmp(cmp, trace_options[i]) == 0) {
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002585 set_tracer_flags(1 << i, !neg);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002586 break;
2587 }
2588 }
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002589
2590 /* If no option could be set, test the specific tracer options */
2591 if (!trace_options[i]) {
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002592 mutex_lock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002593 ret = set_tracer_option(current_trace, cmp, neg);
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002594 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002595 if (ret)
2596 return ret;
2597 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002598
Jiri Olsacf8517c2009-10-23 19:36:16 -04002599 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002600
2601 return cnt;
2602}
2603
Li Zefanfdb372e2009-12-08 11:15:59 +08002604static int tracing_trace_options_open(struct inode *inode, struct file *file)
2605{
2606 if (tracing_disabled)
2607 return -ENODEV;
2608 return single_open(file, tracing_trace_options_show, NULL);
2609}
2610
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002611static const struct file_operations tracing_iter_fops = {
Li Zefanfdb372e2009-12-08 11:15:59 +08002612 .open = tracing_trace_options_open,
2613 .read = seq_read,
2614 .llseek = seq_lseek,
2615 .release = single_release,
Steven Rostedtee6bce52008-11-12 17:52:37 -05002616 .write = tracing_trace_options_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002617};
2618
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002619static const char readme_msg[] =
2620 "tracing mini-HOWTO:\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002621 "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2622 "# cat /sys/kernel/debug/tracing/available_tracers\n"
Nikanth Karthikesanbc2b6872009-03-23 11:58:31 +05302623 "wakeup preemptirqsoff preemptoff irqsoff function sched_switch nop\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002624 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Nikanth Karthikesanbc2b6872009-03-23 11:58:31 +05302625 "nop\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002626 "# echo sched_switch > /sys/kernel/debug/tracing/current_tracer\n"
2627 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002628 "sched_switch\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002629 "# cat /sys/kernel/debug/tracing/trace_options\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002630 "noprint-parent nosym-offset nosym-addr noverbose\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002631 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
2632 "# echo 1 > /sys/kernel/debug/tracing/tracing_enabled\n"
2633 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
2634 "# echo 0 > /sys/kernel/debug/tracing/tracing_enabled\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002635;
2636
2637static ssize_t
2638tracing_readme_read(struct file *filp, char __user *ubuf,
2639 size_t cnt, loff_t *ppos)
2640{
2641 return simple_read_from_buffer(ubuf, cnt, ppos,
2642 readme_msg, strlen(readme_msg));
2643}
2644
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002645static const struct file_operations tracing_readme_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002646 .open = tracing_open_generic,
2647 .read = tracing_readme_read,
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002648};
2649
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002650static ssize_t
Avadh Patel69abe6a2009-04-10 16:04:48 -04002651tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2652 size_t cnt, loff_t *ppos)
2653{
2654 char *buf_comm;
2655 char *file_buf;
2656 char *buf;
2657 int len = 0;
2658 int pid;
2659 int i;
2660
2661 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2662 if (!file_buf)
2663 return -ENOMEM;
2664
2665 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2666 if (!buf_comm) {
2667 kfree(file_buf);
2668 return -ENOMEM;
2669 }
2670
2671 buf = file_buf;
2672
2673 for (i = 0; i < SAVED_CMDLINES; i++) {
2674 int r;
2675
2676 pid = map_cmdline_to_pid[i];
2677 if (pid == -1 || pid == NO_CMDLINE_MAP)
2678 continue;
2679
2680 trace_find_cmdline(pid, buf_comm);
2681 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2682 buf += r;
2683 len += r;
2684 }
2685
2686 len = simple_read_from_buffer(ubuf, cnt, ppos,
2687 file_buf, len);
2688
2689 kfree(file_buf);
2690 kfree(buf_comm);
2691
2692 return len;
2693}
2694
2695static const struct file_operations tracing_saved_cmdlines_fops = {
2696 .open = tracing_open_generic,
2697 .read = tracing_saved_cmdlines_read,
2698};
2699
2700static ssize_t
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002701tracing_ctrl_read(struct file *filp, char __user *ubuf,
2702 size_t cnt, loff_t *ppos)
2703{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002704 char buf[64];
2705 int r;
2706
Steven Rostedt90369902008-11-05 16:05:44 -05002707 r = sprintf(buf, "%u\n", tracer_enabled);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002708 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002709}
2710
2711static ssize_t
2712tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2713 size_t cnt, loff_t *ppos)
2714{
2715 struct trace_array *tr = filp->private_data;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002716 char buf[64];
Hannes Eder5e398412009-02-10 19:44:34 +01002717 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002718 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002719
Steven Rostedtcffae432008-05-12 21:21:00 +02002720 if (cnt >= sizeof(buf))
2721 return -EINVAL;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002722
2723 if (copy_from_user(&buf, ubuf, cnt))
2724 return -EFAULT;
2725
2726 buf[cnt] = 0;
2727
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002728 ret = strict_strtoul(buf, 10, &val);
2729 if (ret < 0)
2730 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002731
2732 val = !!val;
2733
2734 mutex_lock(&trace_types_lock);
Steven Rostedt90369902008-11-05 16:05:44 -05002735 if (tracer_enabled ^ val) {
2736 if (val) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002737 tracer_enabled = 1;
Steven Rostedt90369902008-11-05 16:05:44 -05002738 if (current_trace->start)
2739 current_trace->start(tr);
2740 tracing_start();
2741 } else {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002742 tracer_enabled = 0;
Steven Rostedt90369902008-11-05 16:05:44 -05002743 tracing_stop();
2744 if (current_trace->stop)
2745 current_trace->stop(tr);
2746 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002747 }
2748 mutex_unlock(&trace_types_lock);
2749
Jiri Olsacf8517c2009-10-23 19:36:16 -04002750 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002751
2752 return cnt;
2753}
2754
2755static ssize_t
2756tracing_set_trace_read(struct file *filp, char __user *ubuf,
2757 size_t cnt, loff_t *ppos)
2758{
Li Zefanee6c2c12009-09-18 14:06:47 +08002759 char buf[MAX_TRACER_SIZE+2];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002760 int r;
2761
2762 mutex_lock(&trace_types_lock);
2763 if (current_trace)
2764 r = sprintf(buf, "%s\n", current_trace->name);
2765 else
2766 r = sprintf(buf, "\n");
2767 mutex_unlock(&trace_types_lock);
2768
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002769 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002770}
2771
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02002772int tracer_init(struct tracer *t, struct trace_array *tr)
2773{
2774 tracing_reset_online_cpus(tr);
2775 return t->init(tr);
2776}
2777
Steven Rostedt73c51622009-03-11 13:42:01 -04002778static int tracing_resize_ring_buffer(unsigned long size)
2779{
2780 int ret;
2781
2782 /*
2783 * If kernel or user changes the size of the ring buffer
Steven Rostedta123c522009-03-12 11:21:08 -04002784 * we use the size that was given, and we can forget about
2785 * expanding it later.
Steven Rostedt73c51622009-03-11 13:42:01 -04002786 */
2787 ring_buffer_expanded = 1;
2788
2789 ret = ring_buffer_resize(global_trace.buffer, size);
2790 if (ret < 0)
2791 return ret;
2792
2793 ret = ring_buffer_resize(max_tr.buffer, size);
2794 if (ret < 0) {
2795 int r;
2796
2797 r = ring_buffer_resize(global_trace.buffer,
2798 global_trace.entries);
2799 if (r < 0) {
Steven Rostedta123c522009-03-12 11:21:08 -04002800 /*
2801 * AARGH! We are left with different
2802 * size max buffer!!!!
2803 * The max buffer is our "snapshot" buffer.
2804 * When a tracer needs a snapshot (one of the
2805 * latency tracers), it swaps the max buffer
2806 * with the saved snap shot. We succeeded to
2807 * update the size of the main buffer, but failed to
2808 * update the size of the max buffer. But when we tried
2809 * to reset the main buffer to the original size, we
2810 * failed there too. This is very unlikely to
2811 * happen, but if it does, warn and kill all
2812 * tracing.
2813 */
Steven Rostedt73c51622009-03-11 13:42:01 -04002814 WARN_ON(1);
2815 tracing_disabled = 1;
2816 }
2817 return ret;
2818 }
2819
2820 global_trace.entries = size;
2821
2822 return ret;
2823}
2824
Steven Rostedt1852fcc2009-03-11 14:33:00 -04002825/**
2826 * tracing_update_buffers - used by tracing facility to expand ring buffers
2827 *
2828 * To save on memory when the tracing is never used on a system with it
2829 * configured in. The ring buffers are set to a minimum size. But once
2830 * a user starts to use the tracing facility, then they need to grow
2831 * to their default size.
2832 *
2833 * This function is to be called when a tracer is about to be used.
2834 */
2835int tracing_update_buffers(void)
2836{
2837 int ret = 0;
2838
Steven Rostedt1027fcb2009-03-12 11:33:20 -04002839 mutex_lock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04002840 if (!ring_buffer_expanded)
2841 ret = tracing_resize_ring_buffer(trace_buf_size);
Steven Rostedt1027fcb2009-03-12 11:33:20 -04002842 mutex_unlock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04002843
2844 return ret;
2845}
2846
Steven Rostedt577b7852009-02-26 23:43:05 -05002847struct trace_option_dentry;
2848
2849static struct trace_option_dentry *
2850create_trace_option_files(struct tracer *tracer);
2851
2852static void
2853destroy_trace_option_files(struct trace_option_dentry *topts);
2854
Steven Rostedtb2821ae2009-02-02 21:38:32 -05002855static int tracing_set_tracer(const char *buf)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002856{
Steven Rostedt577b7852009-02-26 23:43:05 -05002857 static struct trace_option_dentry *topts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002858 struct trace_array *tr = &global_trace;
2859 struct tracer *t;
Peter Zijlstrad9e54072008-11-01 19:57:37 +01002860 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002861
Steven Rostedt1027fcb2009-03-12 11:33:20 -04002862 mutex_lock(&trace_types_lock);
2863
Steven Rostedt73c51622009-03-11 13:42:01 -04002864 if (!ring_buffer_expanded) {
2865 ret = tracing_resize_ring_buffer(trace_buf_size);
2866 if (ret < 0)
Frederic Weisbecker59f586d2009-03-15 22:10:39 +01002867 goto out;
Steven Rostedt73c51622009-03-11 13:42:01 -04002868 ret = 0;
2869 }
2870
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002871 for (t = trace_types; t; t = t->next) {
2872 if (strcmp(t->name, buf) == 0)
2873 break;
2874 }
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02002875 if (!t) {
2876 ret = -EINVAL;
2877 goto out;
2878 }
2879 if (t == current_trace)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002880 goto out;
2881
Steven Rostedt9f029e82008-11-12 15:24:24 -05002882 trace_branch_disable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002883 if (current_trace && current_trace->reset)
2884 current_trace->reset(tr);
2885
Steven Rostedt577b7852009-02-26 23:43:05 -05002886 destroy_trace_option_files(topts);
2887
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002888 current_trace = t;
Steven Rostedt577b7852009-02-26 23:43:05 -05002889
2890 topts = create_trace_option_files(current_trace);
2891
Frederic Weisbecker1c800252008-11-16 05:57:26 +01002892 if (t->init) {
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02002893 ret = tracer_init(t, tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +01002894 if (ret)
2895 goto out;
2896 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002897
Steven Rostedt9f029e82008-11-12 15:24:24 -05002898 trace_branch_enable(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002899 out:
2900 mutex_unlock(&trace_types_lock);
2901
Peter Zijlstrad9e54072008-11-01 19:57:37 +01002902 return ret;
2903}
2904
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002905static ssize_t
2906tracing_set_trace_write(struct file *filp, const char __user *ubuf,
2907 size_t cnt, loff_t *ppos)
2908{
Li Zefanee6c2c12009-09-18 14:06:47 +08002909 char buf[MAX_TRACER_SIZE+1];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002910 int i;
2911 size_t ret;
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01002912 int err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002913
Steven Rostedt60063a62008-10-28 10:44:24 -04002914 ret = cnt;
2915
Li Zefanee6c2c12009-09-18 14:06:47 +08002916 if (cnt > MAX_TRACER_SIZE)
2917 cnt = MAX_TRACER_SIZE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002918
2919 if (copy_from_user(&buf, ubuf, cnt))
2920 return -EFAULT;
2921
2922 buf[cnt] = 0;
2923
2924 /* strip ending whitespace. */
2925 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
2926 buf[i] = 0;
2927
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01002928 err = tracing_set_tracer(buf);
2929 if (err)
2930 return err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002931
Jiri Olsacf8517c2009-10-23 19:36:16 -04002932 *ppos += ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002933
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02002934 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002935}
2936
2937static ssize_t
2938tracing_max_lat_read(struct file *filp, char __user *ubuf,
2939 size_t cnt, loff_t *ppos)
2940{
2941 unsigned long *ptr = filp->private_data;
2942 char buf[64];
2943 int r;
2944
Steven Rostedtcffae432008-05-12 21:21:00 +02002945 r = snprintf(buf, sizeof(buf), "%ld\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002946 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
Steven Rostedtcffae432008-05-12 21:21:00 +02002947 if (r > sizeof(buf))
2948 r = sizeof(buf);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002949 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002950}
2951
2952static ssize_t
2953tracing_max_lat_write(struct file *filp, const char __user *ubuf,
2954 size_t cnt, loff_t *ppos)
2955{
Hannes Eder5e398412009-02-10 19:44:34 +01002956 unsigned long *ptr = filp->private_data;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002957 char buf[64];
Hannes Eder5e398412009-02-10 19:44:34 +01002958 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002959 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002960
Steven Rostedtcffae432008-05-12 21:21:00 +02002961 if (cnt >= sizeof(buf))
2962 return -EINVAL;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002963
2964 if (copy_from_user(&buf, ubuf, cnt))
2965 return -EFAULT;
2966
2967 buf[cnt] = 0;
2968
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002969 ret = strict_strtoul(buf, 10, &val);
2970 if (ret < 0)
2971 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002972
2973 *ptr = val * 1000;
2974
2975 return cnt;
2976}
2977
Steven Rostedtb3806b42008-05-12 21:20:46 +02002978static int tracing_open_pipe(struct inode *inode, struct file *filp)
2979{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002980 long cpu_file = (long) inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02002981 struct trace_iterator *iter;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002982 int ret = 0;
Steven Rostedtb3806b42008-05-12 21:20:46 +02002983
2984 if (tracing_disabled)
2985 return -ENODEV;
2986
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002987 mutex_lock(&trace_types_lock);
2988
Steven Rostedtb3806b42008-05-12 21:20:46 +02002989 /* create a buffer to store the information to pass to userspace */
2990 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002991 if (!iter) {
2992 ret = -ENOMEM;
2993 goto out;
2994 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02002995
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002996 /*
2997 * We make a copy of the current tracer to avoid concurrent
2998 * changes on it while we are reading.
2999 */
3000 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3001 if (!iter->trace) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003002 ret = -ENOMEM;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003003 goto fail;
3004 }
3005 if (current_trace)
3006 *iter->trace = *current_trace;
3007
3008 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3009 ret = -ENOMEM;
3010 goto fail;
Rusty Russell44623442009-01-01 10:12:23 +10303011 }
3012
Steven Rostedta3097202008-11-07 22:36:02 -05003013 /* trace pipe does not show start of buffer */
Rusty Russell44623442009-01-01 10:12:23 +10303014 cpumask_setall(iter->started);
Steven Rostedta3097202008-11-07 22:36:02 -05003015
Steven Rostedt112f38a72009-06-01 15:16:05 -04003016 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3017 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3018
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003019 iter->cpu_file = cpu_file;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003020 iter->tr = &global_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003021 mutex_init(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003022 filp->private_data = iter;
3023
Steven Rostedt107bad82008-05-12 21:21:01 +02003024 if (iter->trace->pipe_open)
3025 iter->trace->pipe_open(iter);
Steven Rostedt107bad82008-05-12 21:21:01 +02003026
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003027out:
3028 mutex_unlock(&trace_types_lock);
3029 return ret;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003030
3031fail:
3032 kfree(iter->trace);
3033 kfree(iter);
3034 mutex_unlock(&trace_types_lock);
3035 return ret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003036}
3037
3038static int tracing_release_pipe(struct inode *inode, struct file *file)
3039{
3040 struct trace_iterator *iter = file->private_data;
3041
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003042 mutex_lock(&trace_types_lock);
3043
Steven Rostedt29bf4a52009-12-09 12:37:43 -05003044 if (iter->trace->pipe_close)
Steven Rostedtc521efd2009-12-07 09:06:24 -05003045 iter->trace->pipe_close(iter);
3046
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003047 mutex_unlock(&trace_types_lock);
3048
Rusty Russell44623442009-01-01 10:12:23 +10303049 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003050 mutex_destroy(&iter->mutex);
3051 kfree(iter->trace);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003052 kfree(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003053
3054 return 0;
3055}
3056
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003057static unsigned int
3058tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3059{
3060 struct trace_iterator *iter = filp->private_data;
3061
3062 if (trace_flags & TRACE_ITER_BLOCK) {
3063 /*
3064 * Always select as readable when in blocking mode
3065 */
3066 return POLLIN | POLLRDNORM;
Ingo Molnarafc2abc2008-05-12 21:21:00 +02003067 } else {
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003068 if (!trace_empty(iter))
3069 return POLLIN | POLLRDNORM;
3070 poll_wait(filp, &trace_wait, poll_table);
3071 if (!trace_empty(iter))
3072 return POLLIN | POLLRDNORM;
3073
3074 return 0;
3075 }
3076}
3077
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003078
3079void default_wait_pipe(struct trace_iterator *iter)
3080{
3081 DEFINE_WAIT(wait);
3082
3083 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3084
3085 if (trace_empty(iter))
3086 schedule();
3087
3088 finish_wait(&trace_wait, &wait);
3089}
3090
3091/*
3092 * This is a make-shift waitqueue.
3093 * A tracer might use this callback on some rare cases:
3094 *
3095 * 1) the current tracer might hold the runqueue lock when it wakes up
3096 * a reader, hence a deadlock (sched, function, and function graph tracers)
3097 * 2) the function tracers, trace all functions, we don't want
3098 * the overhead of calling wake_up and friends
3099 * (and tracing them too)
3100 *
3101 * Anyway, this is really very primitive wakeup.
3102 */
3103void poll_wait_pipe(struct trace_iterator *iter)
3104{
3105 set_current_state(TASK_INTERRUPTIBLE);
3106 /* sleep for 100 msecs, and try again. */
3107 schedule_timeout(HZ / 10);
3108}
3109
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003110/* Must be called with trace_types_lock mutex held. */
3111static int tracing_wait_pipe(struct file *filp)
3112{
3113 struct trace_iterator *iter = filp->private_data;
3114
3115 while (trace_empty(iter)) {
3116
3117 if ((filp->f_flags & O_NONBLOCK)) {
3118 return -EAGAIN;
3119 }
3120
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003121 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003122
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003123 iter->trace->wait_pipe(iter);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003124
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003125 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003126
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003127 if (signal_pending(current))
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003128 return -EINTR;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003129
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003130 /*
3131 * We block until we read something and tracing is disabled.
3132 * We still block if tracing is disabled, but we have never
3133 * read anything. This allows a user to cat this file, and
3134 * then enable tracing. But after we have read something,
3135 * we give an EOF when tracing is again disabled.
3136 *
3137 * iter->pos will be 0 if we haven't read anything.
3138 */
3139 if (!tracer_enabled && iter->pos)
3140 break;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003141 }
3142
3143 return 1;
3144}
3145
Steven Rostedtb3806b42008-05-12 21:20:46 +02003146/*
3147 * Consumer reader.
3148 */
3149static ssize_t
3150tracing_read_pipe(struct file *filp, char __user *ubuf,
3151 size_t cnt, loff_t *ppos)
3152{
3153 struct trace_iterator *iter = filp->private_data;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003154 static struct tracer *old_tracer;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003155 ssize_t sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003156
3157 /* return any leftover data */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003158 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3159 if (sret != -EBUSY)
3160 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003161
Steven Rostedtf9520752009-03-02 14:04:40 -05003162 trace_seq_init(&iter->seq);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003163
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003164 /* copy the tracer to avoid using a global lock all around */
Steven Rostedt107bad82008-05-12 21:21:01 +02003165 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003166 if (unlikely(old_tracer != current_trace && current_trace)) {
3167 old_tracer = current_trace;
3168 *iter->trace = *current_trace;
3169 }
3170 mutex_unlock(&trace_types_lock);
3171
3172 /*
3173 * Avoid more than one consumer on a single file descriptor
3174 * This is just a matter of traces coherency, the ring buffer itself
3175 * is protected.
3176 */
3177 mutex_lock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003178 if (iter->trace->read) {
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003179 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3180 if (sret)
Steven Rostedt107bad82008-05-12 21:21:01 +02003181 goto out;
Steven Rostedt107bad82008-05-12 21:21:01 +02003182 }
3183
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003184waitagain:
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003185 sret = tracing_wait_pipe(filp);
3186 if (sret <= 0)
3187 goto out;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003188
3189 /* stop when tracing is finished */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003190 if (trace_empty(iter)) {
3191 sret = 0;
Steven Rostedt107bad82008-05-12 21:21:01 +02003192 goto out;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003193 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003194
3195 if (cnt >= PAGE_SIZE)
3196 cnt = PAGE_SIZE - 1;
3197
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003198 /* reset all but tr, trace, and overruns */
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003199 memset(&iter->seq, 0,
3200 sizeof(struct trace_iterator) -
3201 offsetof(struct trace_iterator, seq));
Steven Rostedt4823ed72008-05-12 21:21:01 +02003202 iter->pos = -1;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003203
Lai Jiangshan4f535962009-05-18 19:35:34 +08003204 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003205 trace_access_lock(iter->cpu_file);
Jason Wessel955b61e2010-08-05 09:22:23 -05003206 while (trace_find_next_entry_inc(iter) != NULL) {
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003207 enum print_line_t ret;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003208 int len = iter->seq.len;
3209
Ingo Molnarf9896bf2008-05-12 21:20:47 +02003210 ret = print_trace_line(iter);
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003211 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt088b1e422008-05-12 21:20:48 +02003212 /* don't print partial lines */
3213 iter->seq.len = len;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003214 break;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003215 }
Frederic Weisbeckerb91facc2009-02-06 18:30:44 +01003216 if (ret != TRACE_TYPE_NO_CONSUME)
3217 trace_consume(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003218
3219 if (iter->seq.len >= cnt)
3220 break;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003221 }
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003222 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003223 trace_event_read_unlock();
Steven Rostedtb3806b42008-05-12 21:20:46 +02003224
Steven Rostedtb3806b42008-05-12 21:20:46 +02003225 /* Now copy what we have to the user */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003226 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3227 if (iter->seq.readpos >= iter->seq.len)
Steven Rostedtf9520752009-03-02 14:04:40 -05003228 trace_seq_init(&iter->seq);
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003229
3230 /*
3231 * If there was nothing to send to user, inspite of consuming trace
3232 * entries, go back to wait for more entries.
3233 */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003234 if (sret == -EBUSY)
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003235 goto waitagain;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003236
Steven Rostedt107bad82008-05-12 21:21:01 +02003237out:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003238 mutex_unlock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003239
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003240 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003241}
3242
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003243static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3244 struct pipe_buffer *buf)
3245{
3246 __free_page(buf->page);
3247}
3248
3249static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3250 unsigned int idx)
3251{
3252 __free_page(spd->pages[idx]);
3253}
3254
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08003255static const struct pipe_buf_operations tracing_pipe_buf_ops = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003256 .can_merge = 0,
3257 .map = generic_pipe_buf_map,
3258 .unmap = generic_pipe_buf_unmap,
3259 .confirm = generic_pipe_buf_confirm,
3260 .release = tracing_pipe_buf_release,
3261 .steal = generic_pipe_buf_steal,
3262 .get = generic_pipe_buf_get,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003263};
3264
Steven Rostedt34cd4992009-02-09 12:06:29 -05003265static size_t
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01003266tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003267{
3268 size_t count;
3269 int ret;
3270
3271 /* Seq buffer is page-sized, exactly what we need. */
3272 for (;;) {
3273 count = iter->seq.len;
3274 ret = print_trace_line(iter);
3275 count = iter->seq.len - count;
3276 if (rem < count) {
3277 rem = 0;
3278 iter->seq.len -= count;
3279 break;
3280 }
3281 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3282 iter->seq.len -= count;
3283 break;
3284 }
3285
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08003286 if (ret != TRACE_TYPE_NO_CONSUME)
3287 trace_consume(iter);
Steven Rostedt34cd4992009-02-09 12:06:29 -05003288 rem -= count;
Jason Wessel955b61e2010-08-05 09:22:23 -05003289 if (!trace_find_next_entry_inc(iter)) {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003290 rem = 0;
3291 iter->ent = NULL;
3292 break;
3293 }
3294 }
3295
3296 return rem;
3297}
3298
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003299static ssize_t tracing_splice_read_pipe(struct file *filp,
3300 loff_t *ppos,
3301 struct pipe_inode_info *pipe,
3302 size_t len,
3303 unsigned int flags)
3304{
Jens Axboe35f3d142010-05-20 10:43:18 +02003305 struct page *pages_def[PIPE_DEF_BUFFERS];
3306 struct partial_page partial_def[PIPE_DEF_BUFFERS];
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003307 struct trace_iterator *iter = filp->private_data;
3308 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02003309 .pages = pages_def,
3310 .partial = partial_def,
Steven Rostedt34cd4992009-02-09 12:06:29 -05003311 .nr_pages = 0, /* This gets updated below. */
3312 .flags = flags,
3313 .ops = &tracing_pipe_buf_ops,
3314 .spd_release = tracing_spd_release_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003315 };
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003316 static struct tracer *old_tracer;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003317 ssize_t ret;
Steven Rostedt34cd4992009-02-09 12:06:29 -05003318 size_t rem;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003319 unsigned int i;
3320
Jens Axboe35f3d142010-05-20 10:43:18 +02003321 if (splice_grow_spd(pipe, &spd))
3322 return -ENOMEM;
3323
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003324 /* copy the tracer to avoid using a global lock all around */
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003325 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003326 if (unlikely(old_tracer != current_trace && current_trace)) {
3327 old_tracer = current_trace;
3328 *iter->trace = *current_trace;
3329 }
3330 mutex_unlock(&trace_types_lock);
3331
3332 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003333
3334 if (iter->trace->splice_read) {
3335 ret = iter->trace->splice_read(iter, filp,
3336 ppos, pipe, len, flags);
3337 if (ret)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003338 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003339 }
3340
3341 ret = tracing_wait_pipe(filp);
3342 if (ret <= 0)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003343 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003344
Jason Wessel955b61e2010-08-05 09:22:23 -05003345 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003346 ret = -EFAULT;
Steven Rostedt34cd4992009-02-09 12:06:29 -05003347 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003348 }
3349
Lai Jiangshan4f535962009-05-18 19:35:34 +08003350 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003351 trace_access_lock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003352
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003353 /* Fill as many pages as possible. */
Jens Axboe35f3d142010-05-20 10:43:18 +02003354 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3355 spd.pages[i] = alloc_page(GFP_KERNEL);
3356 if (!spd.pages[i])
Steven Rostedt34cd4992009-02-09 12:06:29 -05003357 break;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003358
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01003359 rem = tracing_fill_pipe_page(rem, iter);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003360
3361 /* Copy the data into the page, so we can start over. */
3362 ret = trace_seq_to_buffer(&iter->seq,
Jens Axboe35f3d142010-05-20 10:43:18 +02003363 page_address(spd.pages[i]),
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003364 iter->seq.len);
3365 if (ret < 0) {
Jens Axboe35f3d142010-05-20 10:43:18 +02003366 __free_page(spd.pages[i]);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003367 break;
3368 }
Jens Axboe35f3d142010-05-20 10:43:18 +02003369 spd.partial[i].offset = 0;
3370 spd.partial[i].len = iter->seq.len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003371
Steven Rostedtf9520752009-03-02 14:04:40 -05003372 trace_seq_init(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003373 }
3374
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003375 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003376 trace_event_read_unlock();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003377 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003378
3379 spd.nr_pages = i;
3380
Jens Axboe35f3d142010-05-20 10:43:18 +02003381 ret = splice_to_pipe(pipe, &spd);
3382out:
3383 splice_shrink_spd(pipe, &spd);
3384 return ret;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003385
Steven Rostedt34cd4992009-02-09 12:06:29 -05003386out_err:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003387 mutex_unlock(&iter->mutex);
Jens Axboe35f3d142010-05-20 10:43:18 +02003388 goto out;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003389}
3390
Steven Rostedta98a3c32008-05-12 21:20:59 +02003391static ssize_t
3392tracing_entries_read(struct file *filp, char __user *ubuf,
3393 size_t cnt, loff_t *ppos)
3394{
3395 struct trace_array *tr = filp->private_data;
Steven Rostedtdb526ca2009-03-12 13:53:25 -04003396 char buf[96];
Steven Rostedta98a3c32008-05-12 21:20:59 +02003397 int r;
3398
Steven Rostedtdb526ca2009-03-12 13:53:25 -04003399 mutex_lock(&trace_types_lock);
3400 if (!ring_buffer_expanded)
3401 r = sprintf(buf, "%lu (expanded: %lu)\n",
3402 tr->entries >> 10,
3403 trace_buf_size >> 10);
3404 else
3405 r = sprintf(buf, "%lu\n", tr->entries >> 10);
3406 mutex_unlock(&trace_types_lock);
3407
Steven Rostedta98a3c32008-05-12 21:20:59 +02003408 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3409}
3410
3411static ssize_t
3412tracing_entries_write(struct file *filp, const char __user *ubuf,
3413 size_t cnt, loff_t *ppos)
3414{
3415 unsigned long val;
3416 char buf[64];
Steven Rostedtbf5e6512008-11-10 21:46:00 -05003417 int ret, cpu;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003418
Steven Rostedtcffae432008-05-12 21:21:00 +02003419 if (cnt >= sizeof(buf))
3420 return -EINVAL;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003421
3422 if (copy_from_user(&buf, ubuf, cnt))
3423 return -EFAULT;
3424
3425 buf[cnt] = 0;
3426
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003427 ret = strict_strtoul(buf, 10, &val);
3428 if (ret < 0)
3429 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003430
3431 /* must have at least 1 entry */
3432 if (!val)
3433 return -EINVAL;
3434
3435 mutex_lock(&trace_types_lock);
3436
Steven Rostedtc76f0692008-11-07 22:36:02 -05003437 tracing_stop();
Steven Rostedta98a3c32008-05-12 21:20:59 +02003438
Steven Rostedtbf5e6512008-11-10 21:46:00 -05003439 /* disable all cpu buffers */
3440 for_each_tracing_cpu(cpu) {
3441 if (global_trace.data[cpu])
3442 atomic_inc(&global_trace.data[cpu]->disabled);
3443 if (max_tr.data[cpu])
3444 atomic_inc(&max_tr.data[cpu]->disabled);
3445 }
3446
Steven Rostedt1696b2b2008-11-13 00:09:35 -05003447 /* value is in KB */
3448 val <<= 10;
3449
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003450 if (val != global_trace.entries) {
Steven Rostedt73c51622009-03-11 13:42:01 -04003451 ret = tracing_resize_ring_buffer(val);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003452 if (ret < 0) {
3453 cnt = ret;
Steven Rostedt3eefae92008-05-12 21:21:04 +02003454 goto out;
3455 }
Steven Rostedta98a3c32008-05-12 21:20:59 +02003456 }
3457
Jiri Olsacf8517c2009-10-23 19:36:16 -04003458 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003459
Steven Rostedt19384c02008-05-22 00:22:16 -04003460 /* If check pages failed, return ENOMEM */
3461 if (tracing_disabled)
3462 cnt = -ENOMEM;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003463 out:
Steven Rostedtbf5e6512008-11-10 21:46:00 -05003464 for_each_tracing_cpu(cpu) {
3465 if (global_trace.data[cpu])
3466 atomic_dec(&global_trace.data[cpu]->disabled);
3467 if (max_tr.data[cpu])
3468 atomic_dec(&max_tr.data[cpu]->disabled);
3469 }
3470
Steven Rostedtc76f0692008-11-07 22:36:02 -05003471 tracing_start();
Steven Rostedta98a3c32008-05-12 21:20:59 +02003472 max_tr.entries = global_trace.entries;
3473 mutex_unlock(&trace_types_lock);
3474
3475 return cnt;
3476}
3477
Carsten Emdef2942482009-12-06 14:02:44 +01003478static int mark_printk(const char *fmt, ...)
3479{
3480 int ret;
3481 va_list args;
3482 va_start(args, fmt);
3483 ret = trace_vprintk(0, fmt, args);
3484 va_end(args);
3485 return ret;
3486}
3487
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003488static ssize_t
3489tracing_mark_write(struct file *filp, const char __user *ubuf,
3490 size_t cnt, loff_t *fpos)
3491{
3492 char *buf;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003493
Steven Rostedtc76f0692008-11-07 22:36:02 -05003494 if (tracing_disabled)
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003495 return -EINVAL;
3496
3497 if (cnt > TRACE_BUF_SIZE)
3498 cnt = TRACE_BUF_SIZE;
3499
Carsten Emdec13d2f72009-11-16 20:56:13 +01003500 buf = kmalloc(cnt + 2, GFP_KERNEL);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003501 if (buf == NULL)
3502 return -ENOMEM;
3503
3504 if (copy_from_user(buf, ubuf, cnt)) {
3505 kfree(buf);
3506 return -EFAULT;
3507 }
Carsten Emdec13d2f72009-11-16 20:56:13 +01003508 if (buf[cnt-1] != '\n') {
3509 buf[cnt] = '\n';
3510 buf[cnt+1] = '\0';
3511 } else
3512 buf[cnt] = '\0';
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003513
Carsten Emdef2942482009-12-06 14:02:44 +01003514 cnt = mark_printk("%s", buf);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003515 kfree(buf);
3516 *fpos += cnt;
3517
3518 return cnt;
3519}
3520
Li Zefan13f16d22009-12-08 11:16:11 +08003521static int tracing_clock_show(struct seq_file *m, void *v)
Zhaolei5079f322009-08-25 16:12:56 +08003522{
Zhaolei5079f322009-08-25 16:12:56 +08003523 int i;
3524
3525 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
Li Zefan13f16d22009-12-08 11:16:11 +08003526 seq_printf(m,
Zhaolei5079f322009-08-25 16:12:56 +08003527 "%s%s%s%s", i ? " " : "",
3528 i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3529 i == trace_clock_id ? "]" : "");
Li Zefan13f16d22009-12-08 11:16:11 +08003530 seq_putc(m, '\n');
Zhaolei5079f322009-08-25 16:12:56 +08003531
Li Zefan13f16d22009-12-08 11:16:11 +08003532 return 0;
Zhaolei5079f322009-08-25 16:12:56 +08003533}
3534
3535static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3536 size_t cnt, loff_t *fpos)
3537{
3538 char buf[64];
3539 const char *clockstr;
3540 int i;
3541
3542 if (cnt >= sizeof(buf))
3543 return -EINVAL;
3544
3545 if (copy_from_user(&buf, ubuf, cnt))
3546 return -EFAULT;
3547
3548 buf[cnt] = 0;
3549
3550 clockstr = strstrip(buf);
3551
3552 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3553 if (strcmp(trace_clocks[i].name, clockstr) == 0)
3554 break;
3555 }
3556 if (i == ARRAY_SIZE(trace_clocks))
3557 return -EINVAL;
3558
3559 trace_clock_id = i;
3560
3561 mutex_lock(&trace_types_lock);
3562
3563 ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
3564 if (max_tr.buffer)
3565 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
3566
3567 mutex_unlock(&trace_types_lock);
3568
3569 *fpos += cnt;
3570
3571 return cnt;
3572}
3573
Li Zefan13f16d22009-12-08 11:16:11 +08003574static int tracing_clock_open(struct inode *inode, struct file *file)
3575{
3576 if (tracing_disabled)
3577 return -ENODEV;
3578 return single_open(file, tracing_clock_show, NULL);
3579}
3580
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003581static const struct file_operations tracing_max_lat_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003582 .open = tracing_open_generic,
3583 .read = tracing_max_lat_read,
3584 .write = tracing_max_lat_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003585};
3586
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003587static const struct file_operations tracing_ctrl_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003588 .open = tracing_open_generic,
3589 .read = tracing_ctrl_read,
3590 .write = tracing_ctrl_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003591};
3592
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003593static const struct file_operations set_tracer_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003594 .open = tracing_open_generic,
3595 .read = tracing_set_trace_read,
3596 .write = tracing_set_trace_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003597};
3598
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003599static const struct file_operations tracing_pipe_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003600 .open = tracing_open_pipe,
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003601 .poll = tracing_poll_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003602 .read = tracing_read_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003603 .splice_read = tracing_splice_read_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003604 .release = tracing_release_pipe,
Steven Rostedtb3806b42008-05-12 21:20:46 +02003605};
3606
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003607static const struct file_operations tracing_entries_fops = {
Steven Rostedta98a3c32008-05-12 21:20:59 +02003608 .open = tracing_open_generic,
3609 .read = tracing_entries_read,
3610 .write = tracing_entries_write,
3611};
3612
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003613static const struct file_operations tracing_mark_fops = {
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02003614 .open = tracing_open_generic,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003615 .write = tracing_mark_write,
3616};
3617
Zhaolei5079f322009-08-25 16:12:56 +08003618static const struct file_operations trace_clock_fops = {
Li Zefan13f16d22009-12-08 11:16:11 +08003619 .open = tracing_clock_open,
3620 .read = seq_read,
3621 .llseek = seq_lseek,
3622 .release = single_release,
Zhaolei5079f322009-08-25 16:12:56 +08003623 .write = tracing_clock_write,
3624};
3625
Steven Rostedt2cadf912008-12-01 22:20:19 -05003626struct ftrace_buffer_info {
3627 struct trace_array *tr;
3628 void *spare;
3629 int cpu;
3630 unsigned int read;
3631};
3632
3633static int tracing_buffers_open(struct inode *inode, struct file *filp)
3634{
3635 int cpu = (int)(long)inode->i_private;
3636 struct ftrace_buffer_info *info;
3637
3638 if (tracing_disabled)
3639 return -ENODEV;
3640
3641 info = kzalloc(sizeof(*info), GFP_KERNEL);
3642 if (!info)
3643 return -ENOMEM;
3644
3645 info->tr = &global_trace;
3646 info->cpu = cpu;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08003647 info->spare = NULL;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003648 /* Force reading ring buffer for first read */
3649 info->read = (unsigned int)-1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003650
3651 filp->private_data = info;
3652
Lai Jiangshand1e7e022009-04-02 15:16:56 +08003653 return nonseekable_open(inode, filp);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003654}
3655
3656static ssize_t
3657tracing_buffers_read(struct file *filp, char __user *ubuf,
3658 size_t count, loff_t *ppos)
3659{
3660 struct ftrace_buffer_info *info = filp->private_data;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003661 ssize_t ret;
3662 size_t size;
3663
Steven Rostedt2dc5d122009-03-04 19:10:05 -05003664 if (!count)
3665 return 0;
3666
Lai Jiangshanddd538f2009-04-02 15:16:59 +08003667 if (!info->spare)
3668 info->spare = ring_buffer_alloc_read_page(info->tr->buffer);
3669 if (!info->spare)
3670 return -ENOMEM;
3671
Steven Rostedt2cadf912008-12-01 22:20:19 -05003672 /* Do we have previous read data to read? */
3673 if (info->read < PAGE_SIZE)
3674 goto read;
3675
3676 info->read = 0;
3677
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003678 trace_access_lock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003679 ret = ring_buffer_read_page(info->tr->buffer,
3680 &info->spare,
3681 count,
3682 info->cpu, 0);
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003683 trace_access_unlock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003684 if (ret < 0)
3685 return 0;
3686
Steven Rostedt2cadf912008-12-01 22:20:19 -05003687read:
3688 size = PAGE_SIZE - info->read;
3689 if (size > count)
3690 size = count;
3691
3692 ret = copy_to_user(ubuf, info->spare + info->read, size);
Steven Rostedt2dc5d122009-03-04 19:10:05 -05003693 if (ret == size)
Steven Rostedt2cadf912008-12-01 22:20:19 -05003694 return -EFAULT;
Steven Rostedt2dc5d122009-03-04 19:10:05 -05003695 size -= ret;
3696
Steven Rostedt2cadf912008-12-01 22:20:19 -05003697 *ppos += size;
3698 info->read += size;
3699
3700 return size;
3701}
3702
3703static int tracing_buffers_release(struct inode *inode, struct file *file)
3704{
3705 struct ftrace_buffer_info *info = file->private_data;
3706
Lai Jiangshanddd538f2009-04-02 15:16:59 +08003707 if (info->spare)
3708 ring_buffer_free_read_page(info->tr->buffer, info->spare);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003709 kfree(info);
3710
3711 return 0;
3712}
3713
3714struct buffer_ref {
3715 struct ring_buffer *buffer;
3716 void *page;
3717 int ref;
3718};
3719
3720static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
3721 struct pipe_buffer *buf)
3722{
3723 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3724
3725 if (--ref->ref)
3726 return;
3727
3728 ring_buffer_free_read_page(ref->buffer, ref->page);
3729 kfree(ref);
3730 buf->private = 0;
3731}
3732
3733static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
3734 struct pipe_buffer *buf)
3735{
3736 return 1;
3737}
3738
3739static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
3740 struct pipe_buffer *buf)
3741{
3742 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
3743
3744 ref->ref++;
3745}
3746
3747/* Pipe buffer operations for a buffer. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08003748static const struct pipe_buf_operations buffer_pipe_buf_ops = {
Steven Rostedt2cadf912008-12-01 22:20:19 -05003749 .can_merge = 0,
3750 .map = generic_pipe_buf_map,
3751 .unmap = generic_pipe_buf_unmap,
3752 .confirm = generic_pipe_buf_confirm,
3753 .release = buffer_pipe_buf_release,
3754 .steal = buffer_pipe_buf_steal,
3755 .get = buffer_pipe_buf_get,
3756};
3757
3758/*
3759 * Callback from splice_to_pipe(), if we need to release some pages
3760 * at the end of the spd in case we error'ed out in filling the pipe.
3761 */
3762static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
3763{
3764 struct buffer_ref *ref =
3765 (struct buffer_ref *)spd->partial[i].private;
3766
3767 if (--ref->ref)
3768 return;
3769
3770 ring_buffer_free_read_page(ref->buffer, ref->page);
3771 kfree(ref);
3772 spd->partial[i].private = 0;
3773}
3774
3775static ssize_t
3776tracing_buffers_splice_read(struct file *file, loff_t *ppos,
3777 struct pipe_inode_info *pipe, size_t len,
3778 unsigned int flags)
3779{
3780 struct ftrace_buffer_info *info = file->private_data;
Jens Axboe35f3d142010-05-20 10:43:18 +02003781 struct partial_page partial_def[PIPE_DEF_BUFFERS];
3782 struct page *pages_def[PIPE_DEF_BUFFERS];
Steven Rostedt2cadf912008-12-01 22:20:19 -05003783 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02003784 .pages = pages_def,
3785 .partial = partial_def,
Steven Rostedt2cadf912008-12-01 22:20:19 -05003786 .flags = flags,
3787 .ops = &buffer_pipe_buf_ops,
3788 .spd_release = buffer_spd_release,
3789 };
3790 struct buffer_ref *ref;
Steven Rostedt93459c62009-04-29 00:23:13 -04003791 int entries, size, i;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003792 size_t ret;
3793
Jens Axboe35f3d142010-05-20 10:43:18 +02003794 if (splice_grow_spd(pipe, &spd))
3795 return -ENOMEM;
3796
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08003797 if (*ppos & (PAGE_SIZE - 1)) {
3798 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
Jens Axboe35f3d142010-05-20 10:43:18 +02003799 ret = -EINVAL;
3800 goto out;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08003801 }
3802
3803 if (len & (PAGE_SIZE - 1)) {
3804 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
Jens Axboe35f3d142010-05-20 10:43:18 +02003805 if (len < PAGE_SIZE) {
3806 ret = -EINVAL;
3807 goto out;
3808 }
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08003809 len &= PAGE_MASK;
3810 }
3811
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003812 trace_access_lock(info->cpu);
Steven Rostedt93459c62009-04-29 00:23:13 -04003813 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
3814
Jens Axboe35f3d142010-05-20 10:43:18 +02003815 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05003816 struct page *page;
3817 int r;
3818
3819 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
3820 if (!ref)
3821 break;
3822
Steven Rostedt7267fa62009-04-29 00:16:21 -04003823 ref->ref = 1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003824 ref->buffer = info->tr->buffer;
3825 ref->page = ring_buffer_alloc_read_page(ref->buffer);
3826 if (!ref->page) {
3827 kfree(ref);
3828 break;
3829 }
3830
3831 r = ring_buffer_read_page(ref->buffer, &ref->page,
Steven Rostedtf2957f12009-04-29 00:26:30 -04003832 len, info->cpu, 1);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003833 if (r < 0) {
3834 ring_buffer_free_read_page(ref->buffer,
3835 ref->page);
3836 kfree(ref);
3837 break;
3838 }
3839
3840 /*
3841 * zero out any left over data, this is going to
3842 * user land.
3843 */
3844 size = ring_buffer_page_len(ref->page);
3845 if (size < PAGE_SIZE)
3846 memset(ref->page + size, 0, PAGE_SIZE - size);
3847
3848 page = virt_to_page(ref->page);
3849
3850 spd.pages[i] = page;
3851 spd.partial[i].len = PAGE_SIZE;
3852 spd.partial[i].offset = 0;
3853 spd.partial[i].private = (unsigned long)ref;
3854 spd.nr_pages++;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08003855 *ppos += PAGE_SIZE;
Steven Rostedt93459c62009-04-29 00:23:13 -04003856
3857 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003858 }
3859
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003860 trace_access_unlock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05003861 spd.nr_pages = i;
3862
3863 /* did we read anything? */
3864 if (!spd.nr_pages) {
3865 if (flags & SPLICE_F_NONBLOCK)
3866 ret = -EAGAIN;
3867 else
3868 ret = 0;
3869 /* TODO: block */
Jens Axboe35f3d142010-05-20 10:43:18 +02003870 goto out;
Steven Rostedt2cadf912008-12-01 22:20:19 -05003871 }
3872
3873 ret = splice_to_pipe(pipe, &spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02003874 splice_shrink_spd(pipe, &spd);
3875out:
Steven Rostedt2cadf912008-12-01 22:20:19 -05003876 return ret;
3877}
3878
3879static const struct file_operations tracing_buffers_fops = {
3880 .open = tracing_buffers_open,
3881 .read = tracing_buffers_read,
3882 .release = tracing_buffers_release,
3883 .splice_read = tracing_buffers_splice_read,
3884 .llseek = no_llseek,
3885};
3886
Steven Rostedtc8d77182009-04-29 18:03:45 -04003887static ssize_t
3888tracing_stats_read(struct file *filp, char __user *ubuf,
3889 size_t count, loff_t *ppos)
3890{
3891 unsigned long cpu = (unsigned long)filp->private_data;
3892 struct trace_array *tr = &global_trace;
3893 struct trace_seq *s;
3894 unsigned long cnt;
3895
Li Zefane4f2d102009-06-15 10:57:28 +08003896 s = kmalloc(sizeof(*s), GFP_KERNEL);
Steven Rostedtc8d77182009-04-29 18:03:45 -04003897 if (!s)
Roel Kluina6463652009-11-11 22:26:35 +01003898 return -ENOMEM;
Steven Rostedtc8d77182009-04-29 18:03:45 -04003899
3900 trace_seq_init(s);
3901
3902 cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
3903 trace_seq_printf(s, "entries: %ld\n", cnt);
3904
3905 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
3906 trace_seq_printf(s, "overrun: %ld\n", cnt);
3907
3908 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
3909 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
3910
Steven Rostedtc8d77182009-04-29 18:03:45 -04003911 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
3912
3913 kfree(s);
3914
3915 return count;
3916}
3917
3918static const struct file_operations tracing_stats_fops = {
3919 .open = tracing_open_generic,
3920 .read = tracing_stats_read,
3921};
3922
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003923#ifdef CONFIG_DYNAMIC_FTRACE
3924
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003925int __weak ftrace_arch_read_dyn_info(char *buf, int size)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003926{
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003927 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003928}
3929
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003930static ssize_t
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003931tracing_read_dyn_info(struct file *filp, char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003932 size_t cnt, loff_t *ppos)
3933{
Steven Rostedta26a2a22008-10-31 00:03:22 -04003934 static char ftrace_dyn_info_buffer[1024];
3935 static DEFINE_MUTEX(dyn_info_mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003936 unsigned long *p = filp->private_data;
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003937 char *buf = ftrace_dyn_info_buffer;
Steven Rostedta26a2a22008-10-31 00:03:22 -04003938 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003939 int r;
3940
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003941 mutex_lock(&dyn_info_mutex);
3942 r = sprintf(buf, "%ld ", *p);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003943
Steven Rostedta26a2a22008-10-31 00:03:22 -04003944 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003945 buf[r++] = '\n';
3946
3947 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3948
3949 mutex_unlock(&dyn_info_mutex);
3950
3951 return r;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003952}
3953
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003954static const struct file_operations tracing_dyn_info_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003955 .open = tracing_open_generic,
Steven Rostedtb807c3d2008-10-30 16:08:33 -04003956 .read = tracing_read_dyn_info,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003957};
3958#endif
3959
3960static struct dentry *d_tracer;
3961
3962struct dentry *tracing_init_dentry(void)
3963{
3964 static int once;
3965
3966 if (d_tracer)
3967 return d_tracer;
3968
Frederic Weisbecker3e1f60b2009-03-22 23:10:45 +01003969 if (!debugfs_initialized())
3970 return NULL;
3971
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003972 d_tracer = debugfs_create_dir("tracing", NULL);
3973
3974 if (!d_tracer && !once) {
3975 once = 1;
3976 pr_warning("Could not create debugfs directory 'tracing'\n");
3977 return NULL;
3978 }
3979
3980 return d_tracer;
3981}
3982
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003983static struct dentry *d_percpu;
3984
3985struct dentry *tracing_dentry_percpu(void)
3986{
3987 static int once;
3988 struct dentry *d_tracer;
3989
3990 if (d_percpu)
3991 return d_percpu;
3992
3993 d_tracer = tracing_init_dentry();
3994
3995 if (!d_tracer)
3996 return NULL;
3997
3998 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
3999
4000 if (!d_percpu && !once) {
4001 once = 1;
4002 pr_warning("Could not create debugfs directory 'per_cpu'\n");
4003 return NULL;
4004 }
4005
4006 return d_percpu;
4007}
4008
4009static void tracing_init_debugfs_percpu(long cpu)
4010{
4011 struct dentry *d_percpu = tracing_dentry_percpu();
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004012 struct dentry *d_cpu;
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004013 /* strlen(cpu) + MAX(log10(cpu)) + '\0' */
4014 char cpu_dir[7];
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004015
4016 if (cpu > 999 || cpu < 0)
4017 return;
4018
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004019 sprintf(cpu_dir, "cpu%ld", cpu);
4020 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4021 if (!d_cpu) {
4022 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4023 return;
4024 }
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004025
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004026 /* per cpu trace_pipe */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004027 trace_create_file("trace_pipe", 0444, d_cpu,
4028 (void *) cpu, &tracing_pipe_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004029
4030 /* per cpu trace */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004031 trace_create_file("trace", 0644, d_cpu,
4032 (void *) cpu, &tracing_fops);
Steven Rostedt7f96f932009-03-13 00:37:42 -04004033
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004034 trace_create_file("trace_pipe_raw", 0444, d_cpu,
4035 (void *) cpu, &tracing_buffers_fops);
Steven Rostedtc8d77182009-04-29 18:03:45 -04004036
4037 trace_create_file("stats", 0444, d_cpu,
4038 (void *) cpu, &tracing_stats_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004039}
4040
Steven Rostedt60a11772008-05-12 21:20:44 +02004041#ifdef CONFIG_FTRACE_SELFTEST
4042/* Let selftest have access to static functions in this file */
4043#include "trace_selftest.c"
4044#endif
4045
Steven Rostedt577b7852009-02-26 23:43:05 -05004046struct trace_option_dentry {
4047 struct tracer_opt *opt;
4048 struct tracer_flags *flags;
4049 struct dentry *entry;
4050};
4051
4052static ssize_t
4053trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4054 loff_t *ppos)
4055{
4056 struct trace_option_dentry *topt = filp->private_data;
4057 char *buf;
4058
4059 if (topt->flags->val & topt->opt->bit)
4060 buf = "1\n";
4061 else
4062 buf = "0\n";
4063
4064 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4065}
4066
4067static ssize_t
4068trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4069 loff_t *ppos)
4070{
4071 struct trace_option_dentry *topt = filp->private_data;
4072 unsigned long val;
4073 char buf[64];
4074 int ret;
4075
4076 if (cnt >= sizeof(buf))
4077 return -EINVAL;
4078
4079 if (copy_from_user(&buf, ubuf, cnt))
4080 return -EFAULT;
4081
4082 buf[cnt] = 0;
4083
4084 ret = strict_strtoul(buf, 10, &val);
4085 if (ret < 0)
4086 return ret;
4087
Li Zefan8d18eaa2009-12-08 11:17:06 +08004088 if (val != 0 && val != 1)
Steven Rostedt577b7852009-02-26 23:43:05 -05004089 return -EINVAL;
Li Zefan8d18eaa2009-12-08 11:17:06 +08004090
4091 if (!!(topt->flags->val & topt->opt->bit) != val) {
4092 mutex_lock(&trace_types_lock);
4093 ret = __set_tracer_option(current_trace, topt->flags,
Steven Rostedtc757bea2009-12-21 22:35:16 -05004094 topt->opt, !val);
Li Zefan8d18eaa2009-12-08 11:17:06 +08004095 mutex_unlock(&trace_types_lock);
4096 if (ret)
4097 return ret;
Steven Rostedt577b7852009-02-26 23:43:05 -05004098 }
4099
4100 *ppos += cnt;
4101
4102 return cnt;
4103}
4104
4105
4106static const struct file_operations trace_options_fops = {
4107 .open = tracing_open_generic,
4108 .read = trace_options_read,
4109 .write = trace_options_write,
4110};
4111
Steven Rostedta8259072009-02-26 22:19:12 -05004112static ssize_t
4113trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4114 loff_t *ppos)
4115{
4116 long index = (long)filp->private_data;
4117 char *buf;
4118
4119 if (trace_flags & (1 << index))
4120 buf = "1\n";
4121 else
4122 buf = "0\n";
4123
4124 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4125}
4126
4127static ssize_t
4128trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4129 loff_t *ppos)
4130{
4131 long index = (long)filp->private_data;
4132 char buf[64];
4133 unsigned long val;
4134 int ret;
4135
4136 if (cnt >= sizeof(buf))
4137 return -EINVAL;
4138
4139 if (copy_from_user(&buf, ubuf, cnt))
4140 return -EFAULT;
4141
4142 buf[cnt] = 0;
4143
4144 ret = strict_strtoul(buf, 10, &val);
4145 if (ret < 0)
4146 return ret;
4147
Zhaoleif2d84b62009-08-07 18:55:48 +08004148 if (val != 0 && val != 1)
Steven Rostedta8259072009-02-26 22:19:12 -05004149 return -EINVAL;
Zhaoleif2d84b62009-08-07 18:55:48 +08004150 set_tracer_flags(1 << index, val);
Steven Rostedta8259072009-02-26 22:19:12 -05004151
4152 *ppos += cnt;
4153
4154 return cnt;
4155}
4156
Steven Rostedta8259072009-02-26 22:19:12 -05004157static const struct file_operations trace_options_core_fops = {
4158 .open = tracing_open_generic,
4159 .read = trace_options_core_read,
4160 .write = trace_options_core_write,
4161};
4162
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004163struct dentry *trace_create_file(const char *name,
4164 mode_t mode,
4165 struct dentry *parent,
4166 void *data,
4167 const struct file_operations *fops)
4168{
4169 struct dentry *ret;
4170
4171 ret = debugfs_create_file(name, mode, parent, data, fops);
4172 if (!ret)
4173 pr_warning("Could not create debugfs '%s' entry\n", name);
4174
4175 return ret;
4176}
4177
4178
Steven Rostedta8259072009-02-26 22:19:12 -05004179static struct dentry *trace_options_init_dentry(void)
4180{
4181 struct dentry *d_tracer;
4182 static struct dentry *t_options;
4183
4184 if (t_options)
4185 return t_options;
4186
4187 d_tracer = tracing_init_dentry();
4188 if (!d_tracer)
4189 return NULL;
4190
4191 t_options = debugfs_create_dir("options", d_tracer);
4192 if (!t_options) {
4193 pr_warning("Could not create debugfs directory 'options'\n");
4194 return NULL;
4195 }
4196
4197 return t_options;
4198}
4199
Steven Rostedt577b7852009-02-26 23:43:05 -05004200static void
4201create_trace_option_file(struct trace_option_dentry *topt,
4202 struct tracer_flags *flags,
4203 struct tracer_opt *opt)
4204{
4205 struct dentry *t_options;
Steven Rostedt577b7852009-02-26 23:43:05 -05004206
4207 t_options = trace_options_init_dentry();
4208 if (!t_options)
4209 return;
4210
4211 topt->flags = flags;
4212 topt->opt = opt;
4213
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004214 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05004215 &trace_options_fops);
4216
Steven Rostedt577b7852009-02-26 23:43:05 -05004217}
4218
4219static struct trace_option_dentry *
4220create_trace_option_files(struct tracer *tracer)
4221{
4222 struct trace_option_dentry *topts;
4223 struct tracer_flags *flags;
4224 struct tracer_opt *opts;
4225 int cnt;
4226
4227 if (!tracer)
4228 return NULL;
4229
4230 flags = tracer->flags;
4231
4232 if (!flags || !flags->opts)
4233 return NULL;
4234
4235 opts = flags->opts;
4236
4237 for (cnt = 0; opts[cnt].name; cnt++)
4238 ;
4239
Steven Rostedt0cfe8242009-02-27 10:51:10 -05004240 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
Steven Rostedt577b7852009-02-26 23:43:05 -05004241 if (!topts)
4242 return NULL;
4243
4244 for (cnt = 0; opts[cnt].name; cnt++)
4245 create_trace_option_file(&topts[cnt], flags,
4246 &opts[cnt]);
4247
4248 return topts;
4249}
4250
4251static void
4252destroy_trace_option_files(struct trace_option_dentry *topts)
4253{
4254 int cnt;
4255
4256 if (!topts)
4257 return;
4258
4259 for (cnt = 0; topts[cnt].opt; cnt++) {
4260 if (topts[cnt].entry)
4261 debugfs_remove(topts[cnt].entry);
4262 }
4263
4264 kfree(topts);
4265}
4266
Steven Rostedta8259072009-02-26 22:19:12 -05004267static struct dentry *
4268create_trace_option_core_file(const char *option, long index)
4269{
4270 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05004271
4272 t_options = trace_options_init_dentry();
4273 if (!t_options)
4274 return NULL;
4275
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004276 return trace_create_file(option, 0644, t_options, (void *)index,
Steven Rostedta8259072009-02-26 22:19:12 -05004277 &trace_options_core_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05004278}
4279
4280static __init void create_trace_options_dir(void)
4281{
4282 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05004283 int i;
4284
4285 t_options = trace_options_init_dentry();
4286 if (!t_options)
4287 return;
4288
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004289 for (i = 0; trace_options[i]; i++)
4290 create_trace_option_core_file(trace_options[i], i);
Steven Rostedta8259072009-02-26 22:19:12 -05004291}
4292
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004293static __init int tracer_init_debugfs(void)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004294{
4295 struct dentry *d_tracer;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004296 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004297
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004298 trace_access_lock_init();
4299
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004300 d_tracer = tracing_init_dentry();
4301
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004302 trace_create_file("tracing_enabled", 0644, d_tracer,
4303 &global_trace, &tracing_ctrl_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004304
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004305 trace_create_file("trace_options", 0644, d_tracer,
4306 NULL, &tracing_iter_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004307
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004308 trace_create_file("tracing_cpumask", 0644, d_tracer,
4309 NULL, &tracing_cpumask_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05004310
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004311 trace_create_file("trace", 0644, d_tracer,
4312 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
Ingo Molnarc7078de2008-05-12 21:20:52 +02004313
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004314 trace_create_file("available_tracers", 0444, d_tracer,
4315 &global_trace, &show_traces_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004316
Li Zefan339ae5d2009-04-17 10:34:30 +08004317 trace_create_file("current_tracer", 0644, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004318 &global_trace, &set_tracer_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004319
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04004320#ifdef CONFIG_TRACER_MAX_TRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004321 trace_create_file("tracing_max_latency", 0644, d_tracer,
4322 &tracing_max_latency, &tracing_max_lat_fops);
Tim Bird0e950172010-02-25 15:36:43 -08004323#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004324
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004325 trace_create_file("tracing_thresh", 0644, d_tracer,
4326 &tracing_thresh, &tracing_max_lat_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004327
Li Zefan339ae5d2009-04-17 10:34:30 +08004328 trace_create_file("README", 0444, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004329 NULL, &tracing_readme_fops);
Ingo Molnar7bd2f242008-05-12 21:20:45 +02004330
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004331 trace_create_file("trace_pipe", 0444, d_tracer,
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004332 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004333
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004334 trace_create_file("buffer_size_kb", 0644, d_tracer,
4335 &global_trace, &tracing_entries_fops);
Steven Rostedta98a3c32008-05-12 21:20:59 +02004336
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004337 trace_create_file("trace_marker", 0220, d_tracer,
4338 NULL, &tracing_mark_fops);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004339
Avadh Patel69abe6a2009-04-10 16:04:48 -04004340 trace_create_file("saved_cmdlines", 0444, d_tracer,
4341 NULL, &tracing_saved_cmdlines_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004342
Zhaolei5079f322009-08-25 16:12:56 +08004343 trace_create_file("trace_clock", 0644, d_tracer, NULL,
4344 &trace_clock_fops);
4345
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004346#ifdef CONFIG_DYNAMIC_FTRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004347 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4348 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004349#endif
Ingo Molnard618b3e2008-05-12 21:20:49 +02004350#ifdef CONFIG_SYSPROF_TRACER
4351 init_tracer_sysprof_debugfs(d_tracer);
4352#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004353
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004354 create_trace_options_dir();
4355
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004356 for_each_tracing_cpu(cpu)
4357 tracing_init_debugfs_percpu(cpu);
4358
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004359 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004360}
4361
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004362static int trace_panic_handler(struct notifier_block *this,
4363 unsigned long event, void *unused)
4364{
Steven Rostedt944ac422008-10-23 19:26:08 -04004365 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004366 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004367 return NOTIFY_OK;
4368}
4369
4370static struct notifier_block trace_panic_notifier = {
4371 .notifier_call = trace_panic_handler,
4372 .next = NULL,
4373 .priority = 150 /* priority: INT_MAX >= x >= 0 */
4374};
4375
4376static int trace_die_handler(struct notifier_block *self,
4377 unsigned long val,
4378 void *data)
4379{
4380 switch (val) {
4381 case DIE_OOPS:
Steven Rostedt944ac422008-10-23 19:26:08 -04004382 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004383 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004384 break;
4385 default:
4386 break;
4387 }
4388 return NOTIFY_OK;
4389}
4390
4391static struct notifier_block trace_die_notifier = {
4392 .notifier_call = trace_die_handler,
4393 .priority = 200
4394};
4395
4396/*
4397 * printk is set to max of 1024, we really don't need it that big.
4398 * Nothing should be printing 1000 characters anyway.
4399 */
4400#define TRACE_MAX_PRINT 1000
4401
4402/*
4403 * Define here KERN_TRACE so that we have one place to modify
4404 * it if we decide to change what log level the ftrace dump
4405 * should be at.
4406 */
Steven Rostedt428aee12009-01-14 12:24:42 -05004407#define KERN_TRACE KERN_EMERG
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004408
Jason Wessel955b61e2010-08-05 09:22:23 -05004409void
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004410trace_printk_seq(struct trace_seq *s)
4411{
4412 /* Probably should print a warning here. */
4413 if (s->len >= 1000)
4414 s->len = 1000;
4415
4416 /* should be zero ended, but we are paranoid. */
4417 s->buffer[s->len] = 0;
4418
4419 printk(KERN_TRACE "%s", s->buffer);
4420
Steven Rostedtf9520752009-03-02 14:04:40 -05004421 trace_seq_init(s);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004422}
4423
Jason Wessel955b61e2010-08-05 09:22:23 -05004424void trace_init_global_iter(struct trace_iterator *iter)
4425{
4426 iter->tr = &global_trace;
4427 iter->trace = current_trace;
4428 iter->cpu_file = TRACE_PIPE_ALL_CPU;
4429}
4430
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004431static void
4432__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004433{
Thomas Gleixner445c8952009-12-02 19:49:50 +01004434 static arch_spinlock_t ftrace_dump_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01004435 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004436 /* use static because iter can be a bit big for the stack */
4437 static struct trace_iterator iter;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004438 unsigned int old_userobj;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004439 static int dump_ran;
Steven Rostedtd7690412008-10-01 00:29:53 -04004440 unsigned long flags;
4441 int cnt = 0, cpu;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004442
4443 /* only one dump */
Steven Rostedtcd891ae2009-04-28 11:39:34 -04004444 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01004445 arch_spin_lock(&ftrace_dump_lock);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004446 if (dump_ran)
4447 goto out;
4448
4449 dump_ran = 1;
4450
Steven Rostedt0ee6b6c2009-01-14 14:50:19 -05004451 tracing_off();
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004452
4453 if (disable_tracing)
4454 ftrace_kill();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004455
Jason Wessel955b61e2010-08-05 09:22:23 -05004456 trace_init_global_iter(&iter);
4457
Steven Rostedtd7690412008-10-01 00:29:53 -04004458 for_each_tracing_cpu(cpu) {
Jason Wessel955b61e2010-08-05 09:22:23 -05004459 atomic_inc(&iter.tr->data[cpu]->disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -04004460 }
4461
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004462 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4463
Török Edwinb54d3de2008-11-22 13:28:48 +02004464 /* don't look at user memory in panic mode */
4465 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4466
Steven Rostedte543ad72009-03-04 18:20:36 -05004467 /* Simulate the iterator */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004468 iter.tr = &global_trace;
4469 iter.trace = current_trace;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004470
4471 switch (oops_dump_mode) {
4472 case DUMP_ALL:
4473 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4474 break;
4475 case DUMP_ORIG:
4476 iter.cpu_file = raw_smp_processor_id();
4477 break;
4478 case DUMP_NONE:
4479 goto out_enable;
4480 default:
4481 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
4482 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4483 }
4484
4485 printk(KERN_TRACE "Dumping ftrace buffer:\n");
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004486
4487 /*
4488 * We need to stop all tracing on all CPUS to read the
4489 * the next buffer. This is a bit expensive, but is
4490 * not done often. We fill all what we can read,
4491 * and then release the locks again.
4492 */
4493
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004494 while (!trace_empty(&iter)) {
4495
4496 if (!cnt)
4497 printk(KERN_TRACE "---------------------------------\n");
4498
4499 cnt++;
4500
4501 /* reset all but tr, trace, and overruns */
4502 memset(&iter.seq, 0,
4503 sizeof(struct trace_iterator) -
4504 offsetof(struct trace_iterator, seq));
4505 iter.iter_flags |= TRACE_FILE_LAT_FMT;
4506 iter.pos = -1;
4507
Jason Wessel955b61e2010-08-05 09:22:23 -05004508 if (trace_find_next_entry_inc(&iter) != NULL) {
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08004509 int ret;
4510
4511 ret = print_trace_line(&iter);
4512 if (ret != TRACE_TYPE_NO_CONSUME)
4513 trace_consume(&iter);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004514 }
4515
4516 trace_printk_seq(&iter.seq);
4517 }
4518
4519 if (!cnt)
4520 printk(KERN_TRACE " (ftrace buffer empty)\n");
4521 else
4522 printk(KERN_TRACE "---------------------------------\n");
4523
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004524 out_enable:
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004525 /* Re-enable tracing if requested */
4526 if (!disable_tracing) {
4527 trace_flags |= old_userobj;
4528
4529 for_each_tracing_cpu(cpu) {
Jason Wessel955b61e2010-08-05 09:22:23 -05004530 atomic_dec(&iter.tr->data[cpu]->disabled);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004531 }
4532 tracing_on();
4533 }
4534
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004535 out:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01004536 arch_spin_unlock(&ftrace_dump_lock);
Steven Rostedtcd891ae2009-04-28 11:39:34 -04004537 local_irq_restore(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004538}
4539
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004540/* By default: disable tracing after the dump */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004541void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004542{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004543 __ftrace_dump(true, oops_dump_mode);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004544}
4545
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004546__init static int tracer_alloc_buffers(void)
4547{
Steven Rostedt73c51622009-03-11 13:42:01 -04004548 int ring_buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004549 int i;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304550 int ret = -ENOMEM;
4551
4552 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
4553 goto out;
4554
4555 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
4556 goto out_free_buffer_mask;
4557
Steven Rostedt73c51622009-03-11 13:42:01 -04004558 /* To save memory, keep the ring buffer size to its minimum */
4559 if (ring_buffer_expanded)
4560 ring_buf_size = trace_buf_size;
4561 else
4562 ring_buf_size = 1;
4563
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304564 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
4565 cpumask_copy(tracing_cpumask, cpu_all_mask);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004566
Steven Rostedtab464282008-05-12 21:21:00 +02004567 /* TODO: make the number of buffers hot pluggable with CPUS */
Steven Rostedt73c51622009-03-11 13:42:01 -04004568 global_trace.buffer = ring_buffer_alloc(ring_buf_size,
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004569 TRACE_BUFFER_FLAGS);
4570 if (!global_trace.buffer) {
4571 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
4572 WARN_ON(1);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304573 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004574 }
4575 global_trace.entries = ring_buffer_size(global_trace.buffer);
4576
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304577
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004578#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt73c51622009-03-11 13:42:01 -04004579 max_tr.buffer = ring_buffer_alloc(ring_buf_size,
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004580 TRACE_BUFFER_FLAGS);
4581 if (!max_tr.buffer) {
4582 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
4583 WARN_ON(1);
4584 ring_buffer_free(global_trace.buffer);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304585 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004586 }
4587 max_tr.entries = ring_buffer_size(max_tr.buffer);
4588 WARN_ON(max_tr.entries != global_trace.entries);
4589#endif
4590
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02004591 /* Allocate the first page for all buffers */
Steven Rostedtab464282008-05-12 21:21:00 +02004592 for_each_tracing_cpu(i) {
jolsa@redhat.com566b0aa2009-07-16 21:44:26 +02004593 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
Tejun Heo9705f692009-10-29 22:34:13 +09004594 max_tr.data[i] = &per_cpu(max_tr_data, i);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004595 }
4596
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004597 trace_init_cmdlines();
4598
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02004599 register_tracer(&nop_trace);
Steven Rostedt79fb0762009-02-02 21:38:33 -05004600 current_trace = &nop_trace;
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004601#ifdef CONFIG_BOOT_TRACER
4602 register_tracer(&boot_tracer);
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004603#endif
Steven Rostedt60a11772008-05-12 21:20:44 +02004604 /* All seems OK, enable tracing */
4605 tracing_disabled = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04004606
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004607 atomic_notifier_chain_register(&panic_notifier_list,
4608 &trace_panic_notifier);
4609
4610 register_die_notifier(&trace_die_notifier);
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01004611
4612 return 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004613
Rusty Russell9e01c1b2009-01-01 10:12:22 +10304614out_free_cpumask:
4615 free_cpumask_var(tracing_cpumask);
4616out_free_buffer_mask:
4617 free_cpumask_var(tracing_buffer_mask);
4618out:
4619 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004620}
Steven Rostedtb2821ae2009-02-02 21:38:32 -05004621
4622__init static int clear_boot_tracer(void)
4623{
4624 /*
4625 * The default tracer at boot buffer is an init section.
4626 * This function is called in lateinit. If we did not
4627 * find the boot tracer, then clear it out, to prevent
4628 * later registration from accessing the buffer that is
4629 * about to be freed.
4630 */
4631 if (!default_bootup_tracer)
4632 return 0;
4633
4634 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
4635 default_bootup_tracer);
4636 default_bootup_tracer = NULL;
4637
4638 return 0;
4639}
4640
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004641early_initcall(tracer_alloc_buffers);
4642fs_initcall(tracer_init_debugfs);
Steven Rostedtb2821ae2009-02-02 21:38:32 -05004643late_initcall(clear_boot_tracer);