blob: b2af14e94c28c7c9bcea9ac72550fe40398be33c [file] [log] [blame]
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001/*
2 * ring buffer based function tracer
3 *
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 William Lee Irwin III
13 */
Steven Rostedt2cadf912008-12-01 22:20:19 -050014#include <linux/ring_buffer.h>
Sam Ravnborg273b2812009-10-18 00:52:28 +020015#include <generated/utsrelease.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050016#include <linux/stacktrace.h>
17#include <linux/writeback.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020018#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040020#include <linux/notifier.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050021#include <linux/irqflags.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020022#include <linux/debugfs.h>
Steven Rostedt4c11d7a2008-05-12 21:20:43 +020023#include <linux/pagemap.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020024#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050027#include <linux/kprobes.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020028#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050031#include <linux/splice.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040032#include <linux/kdebug.h>
Frederic Weisbecker5f0c6c02009-03-27 14:22:10 +010033#include <linux/string.h>
Lai Jiangshan7e53bd42010-01-06 20:08:50 +080034#include <linux/rwsem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090035#include <linux/slab.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020036#include <linux/ctype.h>
37#include <linux/init.h>
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +020038#include <linux/poll.h>
Steven Rostedtb892e5c2012-03-01 22:06:48 -050039#include <linux/nmi.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020040#include <linux/fs.h>
Ingo Molnar86387f72008-05-12 21:20:51 +020041
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020042#include "trace.h"
Steven Rostedtf0868d12008-12-23 23:24:12 -050043#include "trace_output.h"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020044
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010045/*
Steven Rostedt73c51622009-03-11 13:42:01 -040046 * On boot up, the ring buffer is set to the minimum size, so that
47 * we do not waste memory on systems that are not using tracing.
48 */
Li Zefan020e5f82009-07-01 10:47:05 +080049int ring_buffer_expanded;
Steven Rostedt73c51622009-03-11 13:42:01 -040050
51/*
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010052 * We need to change this state when a selftest is running.
Frederic Weisbeckerff325042008-12-04 23:47:35 +010053 * A selftest will lurk into the ring-buffer to count the
54 * entries inserted during the selftest although some concurrent
Ingo Molnar5e1607a2009-03-05 10:24:48 +010055 * insertions into the ring-buffer such as trace_printk could occurred
Frederic Weisbeckerff325042008-12-04 23:47:35 +010056 * at the same time, giving false positive or negative results.
57 */
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010058static bool __read_mostly tracing_selftest_running;
Frederic Weisbeckerff325042008-12-04 23:47:35 +010059
Steven Rostedtb2821ae2009-02-02 21:38:32 -050060/*
61 * If a tracer is running, we do not want to run SELFTEST.
62 */
Li Zefan020e5f82009-07-01 10:47:05 +080063bool __read_mostly tracing_selftest_disabled;
Steven Rostedtb2821ae2009-02-02 21:38:32 -050064
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010065/* For tracers that don't implement custom flags */
66static struct tracer_opt dummy_tracer_opt[] = {
67 { }
68};
69
70static struct tracer_flags dummy_tracer_flags = {
71 .val = 0,
72 .opts = dummy_tracer_opt
73};
74
75static int dummy_set_flag(u32 old_flags, u32 bit, int set)
76{
77 return 0;
78}
Steven Rostedt0f048702008-11-05 16:05:44 -050079
80/*
81 * Kill all tracing for good (never come back).
82 * It is initialized to 1 but will turn to zero if the initialization
83 * of the tracer is successful. But that is the only place that sets
84 * this back to zero.
85 */
Hannes Eder4fd27352009-02-10 19:44:12 +010086static int tracing_disabled = 1;
Steven Rostedt0f048702008-11-05 16:05:44 -050087
Christoph Lameter9288f992009-10-07 19:17:45 -040088DEFINE_PER_CPU(int, ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -040089
Jason Wessel955b61e2010-08-05 09:22:23 -050090cpumask_var_t __read_mostly tracing_buffer_mask;
Steven Rostedtab464282008-05-12 21:21:00 +020091
Steven Rostedt944ac422008-10-23 19:26:08 -040092/*
93 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
94 *
95 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
96 * is set, then ftrace_dump is called. This will output the contents
97 * of the ftrace buffers to the console. This is very useful for
98 * capturing traces that lead to crashes and outputing it to a
99 * serial console.
100 *
101 * It is default off, but you can enable it with either specifying
102 * "ftrace_dump_on_oops" in the kernel command line, or setting
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200103 * /proc/sys/kernel/ftrace_dump_on_oops
104 * Set 1 if you want to dump buffers of all CPUs
105 * Set 2 if you want to dump the buffer of the CPU that triggered oops
Steven Rostedt944ac422008-10-23 19:26:08 -0400106 */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200107
108enum ftrace_dump_mode ftrace_dump_on_oops;
Steven Rostedt944ac422008-10-23 19:26:08 -0400109
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500110static int tracing_set_tracer(const char *buf);
111
Li Zefanee6c2c12009-09-18 14:06:47 +0800112#define MAX_TRACER_SIZE 100
113static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500114static char *default_bootup_tracer;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100115
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200116static int __init set_cmdline_ftrace(char *str)
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100117{
Li Zefanee6c2c12009-09-18 14:06:47 +0800118 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500119 default_bootup_tracer = bootup_tracer_buf;
Steven Rostedt73c51622009-03-11 13:42:01 -0400120 /* We are using ftrace early, expand it */
121 ring_buffer_expanded = 1;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100122 return 1;
123}
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200124__setup("ftrace=", set_cmdline_ftrace);
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100125
Steven Rostedt944ac422008-10-23 19:26:08 -0400126static int __init set_ftrace_dump_on_oops(char *str)
127{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200128 if (*str++ != '=' || !*str) {
129 ftrace_dump_on_oops = DUMP_ALL;
130 return 1;
131 }
132
133 if (!strcmp("orig_cpu", str)) {
134 ftrace_dump_on_oops = DUMP_ORIG;
135 return 1;
136 }
137
138 return 0;
Steven Rostedt944ac422008-10-23 19:26:08 -0400139}
140__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
Steven Rostedt60a11772008-05-12 21:20:44 +0200141
Lai Jiangshancf8e3472009-03-30 13:48:00 +0800142unsigned long long ns2usecs(cycle_t nsec)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200143{
144 nsec += 500;
145 do_div(nsec, 1000);
146 return nsec;
147}
148
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200149/*
150 * The global_trace is the descriptor that holds the tracing
151 * buffers for the live tracing. For each CPU, it contains
152 * a link list of pages that will store trace entries. The
153 * page descriptor of the pages in the memory is used to hold
154 * the link list by linking the lru item in the page descriptor
155 * to each of the pages in the buffer per CPU.
156 *
157 * For each active CPU there is a data field that holds the
158 * pages for the buffer for that CPU. Each CPU has the same number
159 * of pages allocated for its buffer.
160 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200161static struct trace_array global_trace;
162
163static DEFINE_PER_CPU(struct trace_array_cpu, global_trace_cpu);
164
Steven Rostedte77405a2009-09-02 14:17:06 -0400165int filter_current_check_discard(struct ring_buffer *buffer,
166 struct ftrace_event_call *call, void *rec,
Tom Zanussieb02ce02009-04-08 03:15:54 -0500167 struct ring_buffer_event *event)
168{
Steven Rostedte77405a2009-09-02 14:17:06 -0400169 return filter_check_discard(call, rec, buffer, event);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500170}
Steven Rostedt17c873e2009-04-10 18:12:50 -0400171EXPORT_SYMBOL_GPL(filter_current_check_discard);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500172
Steven Rostedt37886f62009-03-17 17:22:06 -0400173cycle_t ftrace_now(int cpu)
174{
175 u64 ts;
176
177 /* Early boot up does not have a buffer yet */
178 if (!global_trace.buffer)
179 return trace_clock_local();
180
181 ts = ring_buffer_time_stamp(global_trace.buffer, cpu);
182 ring_buffer_normalize_time_stamp(global_trace.buffer, cpu, &ts);
183
184 return ts;
185}
186
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200187/*
188 * The max_tr is used to snapshot the global_trace when a maximum
189 * latency is reached. Some tracers will use this to store a maximum
190 * trace while it continues examining live traces.
191 *
192 * The buffers for the max_tr are set up the same as the global_trace.
193 * When a snapshot is taken, the link list of the max_tr is swapped
194 * with the link list of the global_trace and the buffers are reset for
195 * the global_trace so the tracing can continue.
196 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200197static struct trace_array max_tr;
198
Tejun Heo9705f692009-10-29 22:34:13 +0900199static DEFINE_PER_CPU(struct trace_array_cpu, max_tr_data);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200200
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200201/* tracer_enabled is used to toggle activation of a tracer */
Steven Rostedt26994ea2008-05-12 21:20:48 +0200202static int tracer_enabled = 1;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200203
Steven Rostedt90369902008-11-05 16:05:44 -0500204/**
205 * tracing_is_enabled - return tracer_enabled status
206 *
207 * This function is used by other tracers to know the status
208 * of the tracer_enabled flag. Tracers may use this function
209 * to know if it should enable their features when starting
210 * up. See irqsoff tracer for an example (start_irqsoff_tracer).
211 */
212int tracing_is_enabled(void)
213{
214 return tracer_enabled;
215}
216
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200217/*
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400218 * trace_buf_size is the size in bytes that is allocated
219 * for a buffer. Note, the number of bytes is always rounded
220 * to page size.
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400221 *
222 * This number is purposely set to a low number of 16384.
223 * If the dump on oops happens, it will be much appreciated
224 * to not have to wait for all that output. Anyway this can be
225 * boot time and run time configurable.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200226 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400227#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400228
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400229static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200230
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200231/* trace_types holds a link list of available tracers. */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200232static struct tracer *trace_types __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200233
234/* current_trace points to the tracer that is currently active */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200235static struct tracer *current_trace __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200236
237/*
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200238 * trace_types_lock is used to protect the trace_types list.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200239 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200240static DEFINE_MUTEX(trace_types_lock);
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200241
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800242/*
243 * serialize the access of the ring buffer
244 *
245 * ring buffer serializes readers, but it is low level protection.
246 * The validity of the events (which returns by ring_buffer_peek() ..etc)
247 * are not protected by ring buffer.
248 *
249 * The content of events may become garbage if we allow other process consumes
250 * these events concurrently:
251 * A) the page of the consumed events may become a normal page
252 * (not reader page) in ring buffer, and this page will be rewrited
253 * by events producer.
254 * B) The page of the consumed events may become a page for splice_read,
255 * and this page will be returned to system.
256 *
257 * These primitives allow multi process access to different cpu ring buffer
258 * concurrently.
259 *
260 * These primitives don't distinguish read-only and read-consume access.
261 * Multi read-only access are also serialized.
262 */
263
264#ifdef CONFIG_SMP
265static DECLARE_RWSEM(all_cpu_access_lock);
266static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
267
268static inline void trace_access_lock(int cpu)
269{
270 if (cpu == TRACE_PIPE_ALL_CPU) {
271 /* gain it for accessing the whole ring buffer. */
272 down_write(&all_cpu_access_lock);
273 } else {
274 /* gain it for accessing a cpu ring buffer. */
275
276 /* Firstly block other trace_access_lock(TRACE_PIPE_ALL_CPU). */
277 down_read(&all_cpu_access_lock);
278
279 /* Secondly block other access to this @cpu ring buffer. */
280 mutex_lock(&per_cpu(cpu_access_lock, cpu));
281 }
282}
283
284static inline void trace_access_unlock(int cpu)
285{
286 if (cpu == TRACE_PIPE_ALL_CPU) {
287 up_write(&all_cpu_access_lock);
288 } else {
289 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
290 up_read(&all_cpu_access_lock);
291 }
292}
293
294static inline void trace_access_lock_init(void)
295{
296 int cpu;
297
298 for_each_possible_cpu(cpu)
299 mutex_init(&per_cpu(cpu_access_lock, cpu));
300}
301
302#else
303
304static DEFINE_MUTEX(access_lock);
305
306static inline void trace_access_lock(int cpu)
307{
308 (void)cpu;
309 mutex_lock(&access_lock);
310}
311
312static inline void trace_access_unlock(int cpu)
313{
314 (void)cpu;
315 mutex_unlock(&access_lock);
316}
317
318static inline void trace_access_lock_init(void)
319{
320}
321
322#endif
323
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200324/* trace_wait is a waitqueue for tasks blocked on trace_poll */
Ingo Molnar4e655512008-05-12 21:20:52 +0200325static DECLARE_WAIT_QUEUE_HEAD(trace_wait);
326
Steven Rostedtee6bce52008-11-12 17:52:37 -0500327/* trace_flags holds trace_options default values */
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500328unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
Steven Rostedta2a16d62009-03-24 23:17:58 -0400329 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
Steven Rostedt77271ce2011-11-17 09:34:33 -0500330 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
331 TRACE_ITER_IRQ_INFO;
Ingo Molnar4e655512008-05-12 21:20:52 +0200332
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400333static int trace_stop_count;
Thomas Gleixner5389f6f2009-07-25 17:13:33 +0200334static DEFINE_RAW_SPINLOCK(tracing_start_lock);
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400335
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700336static void wakeup_work_handler(struct work_struct *work)
337{
338 wake_up(&trace_wait);
339}
340
341static DECLARE_DELAYED_WORK(wakeup_work, wakeup_work_handler);
342
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200343/**
Steven Rostedt499e5472012-02-22 15:50:28 -0500344 * tracing_on - enable tracing buffers
345 *
346 * This function enables tracing buffers that may have been
347 * disabled with tracing_off.
348 */
349void tracing_on(void)
350{
351 if (global_trace.buffer)
352 ring_buffer_record_on(global_trace.buffer);
353 /*
354 * This flag is only looked at when buffers haven't been
355 * allocated yet. We don't really care about the race
356 * between setting this flag and actually turning
357 * on the buffer.
358 */
359 global_trace.buffer_disabled = 0;
360}
361EXPORT_SYMBOL_GPL(tracing_on);
362
363/**
364 * tracing_off - turn off tracing buffers
365 *
366 * This function stops the tracing buffers from recording data.
367 * It does not disable any overhead the tracers themselves may
368 * be causing. This function simply causes all recording to
369 * the ring buffers to fail.
370 */
371void tracing_off(void)
372{
373 if (global_trace.buffer)
Steven Rostedtf2bf1f62012-06-06 19:50:40 -0400374 ring_buffer_record_off(global_trace.buffer);
Steven Rostedt499e5472012-02-22 15:50:28 -0500375 /*
376 * This flag is only looked at when buffers haven't been
377 * allocated yet. We don't really care about the race
378 * between setting this flag and actually turning
379 * on the buffer.
380 */
381 global_trace.buffer_disabled = 1;
382}
383EXPORT_SYMBOL_GPL(tracing_off);
384
385/**
386 * tracing_is_on - show state of ring buffers enabled
387 */
388int tracing_is_on(void)
389{
390 if (global_trace.buffer)
391 return ring_buffer_record_is_on(global_trace.buffer);
392 return !global_trace.buffer_disabled;
393}
394EXPORT_SYMBOL_GPL(tracing_is_on);
395
396/**
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200397 * trace_wake_up - wake up tasks waiting for trace input
398 *
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700399 * Schedules a delayed work to wake up any task that is blocked on the
400 * trace_wait queue. These is used with trace_poll for tasks polling the
401 * trace.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200402 */
Ingo Molnar4e655512008-05-12 21:20:52 +0200403void trace_wake_up(void)
404{
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700405 const unsigned long delay = msecs_to_jiffies(2);
Andrew Morton89f19f02009-09-19 11:55:44 -0700406
407 if (trace_flags & TRACE_ITER_BLOCK)
408 return;
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700409 schedule_delayed_work(&wakeup_work, delay);
Ingo Molnar4e655512008-05-12 21:20:52 +0200410}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200411
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400412static int __init set_buf_size(char *str)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200413{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400414 unsigned long buf_size;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200415
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200416 if (!str)
417 return 0;
Li Zefan9d612be2009-06-24 17:33:15 +0800418 buf_size = memparse(str, &str);
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200419 /* nr_entries can not be zero */
Li Zefan9d612be2009-06-24 17:33:15 +0800420 if (buf_size == 0)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200421 return 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400422 trace_buf_size = buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200423 return 1;
424}
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400425__setup("trace_buf_size=", set_buf_size);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200426
Tim Bird0e950172010-02-25 15:36:43 -0800427static int __init set_tracing_thresh(char *str)
428{
429 unsigned long threshhold;
430 int ret;
431
432 if (!str)
433 return 0;
434 ret = strict_strtoul(str, 0, &threshhold);
435 if (ret < 0)
436 return 0;
437 tracing_thresh = threshhold * 1000;
438 return 1;
439}
440__setup("tracing_thresh=", set_tracing_thresh);
441
Steven Rostedt57f50be2008-05-12 21:20:44 +0200442unsigned long nsecs_to_usecs(unsigned long nsecs)
443{
444 return nsecs / 1000;
445}
446
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200447/* These must match the bit postions in trace_iterator_flags */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200448static const char *trace_options[] = {
449 "print-parent",
450 "sym-offset",
451 "sym-addr",
452 "verbose",
Ingo Molnarf9896bf2008-05-12 21:20:47 +0200453 "raw",
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +0200454 "hex",
Ingo Molnarcb0f12a2008-05-12 21:20:47 +0200455 "bin",
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +0200456 "block",
Ingo Molnar86387f72008-05-12 21:20:51 +0200457 "stacktrace",
Ingo Molnar5e1607a2009-03-05 10:24:48 +0100458 "trace_printk",
Steven Rostedtb2a866f2008-11-03 23:15:57 -0500459 "ftrace_preempt",
Steven Rostedt9f029e82008-11-12 15:24:24 -0500460 "branch",
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500461 "annotate",
Török Edwin02b67512008-11-22 13:28:47 +0200462 "userstacktrace",
Török Edwinb54d3de2008-11-22 13:28:48 +0200463 "sym-userobj",
Frederic Weisbecker66896a82008-12-13 20:18:13 +0100464 "printk-msg-only",
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -0200465 "context-info",
Steven Rostedtc032ef642009-03-04 20:34:24 -0500466 "latency-format",
Steven Rostedtbe6f1642009-03-24 11:06:24 -0400467 "sleep-time",
Steven Rostedta2a16d62009-03-24 23:17:58 -0400468 "graph-time",
Li Zefane870e9a2010-07-02 11:07:32 +0800469 "record-cmd",
David Sharp750912f2010-12-08 13:46:47 -0800470 "overwrite",
Steven Rostedtcf30cf62011-06-14 22:44:07 -0400471 "disable_on_free",
Steven Rostedt77271ce2011-11-17 09:34:33 -0500472 "irq-info",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200473 NULL
474};
475
Zhaolei5079f322009-08-25 16:12:56 +0800476static struct {
477 u64 (*func)(void);
478 const char *name;
479} trace_clocks[] = {
480 { trace_clock_local, "local" },
481 { trace_clock_global, "global" },
Steven Rostedt62496872011-09-19 11:35:58 -0400482 { trace_clock_counter, "counter" },
Zhaolei5079f322009-08-25 16:12:56 +0800483};
484
485int trace_clock_id;
486
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200487/*
488 * trace_parser_get_init - gets the buffer for trace parser
489 */
490int trace_parser_get_init(struct trace_parser *parser, int size)
491{
492 memset(parser, 0, sizeof(*parser));
493
494 parser->buffer = kmalloc(size, GFP_KERNEL);
495 if (!parser->buffer)
496 return 1;
497
498 parser->size = size;
499 return 0;
500}
501
502/*
503 * trace_parser_put - frees the buffer for trace parser
504 */
505void trace_parser_put(struct trace_parser *parser)
506{
507 kfree(parser->buffer);
508}
509
510/*
511 * trace_get_user - reads the user input string separated by space
512 * (matched by isspace(ch))
513 *
514 * For each string found the 'struct trace_parser' is updated,
515 * and the function returns.
516 *
517 * Returns number of bytes read.
518 *
519 * See kernel/trace/trace.h for 'struct trace_parser' details.
520 */
521int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
522 size_t cnt, loff_t *ppos)
523{
524 char ch;
525 size_t read = 0;
526 ssize_t ret;
527
528 if (!*ppos)
529 trace_parser_clear(parser);
530
531 ret = get_user(ch, ubuf++);
532 if (ret)
533 goto out;
534
535 read++;
536 cnt--;
537
538 /*
539 * The parser is not finished with the last write,
540 * continue reading the user input without skipping spaces.
541 */
542 if (!parser->cont) {
543 /* skip white space */
544 while (cnt && isspace(ch)) {
545 ret = get_user(ch, ubuf++);
546 if (ret)
547 goto out;
548 read++;
549 cnt--;
550 }
551
552 /* only spaces were written */
553 if (isspace(ch)) {
554 *ppos += read;
555 ret = read;
556 goto out;
557 }
558
559 parser->idx = 0;
560 }
561
562 /* read the non-space input */
563 while (cnt && !isspace(ch)) {
Li Zefan3c235a32009-09-22 13:51:54 +0800564 if (parser->idx < parser->size - 1)
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200565 parser->buffer[parser->idx++] = ch;
566 else {
567 ret = -EINVAL;
568 goto out;
569 }
570 ret = get_user(ch, ubuf++);
571 if (ret)
572 goto out;
573 read++;
574 cnt--;
575 }
576
577 /* We either got finished input or we have to wait for another call. */
578 if (isspace(ch)) {
579 parser->buffer[parser->idx] = 0;
580 parser->cont = false;
581 } else {
582 parser->cont = true;
583 parser->buffer[parser->idx++] = ch;
584 }
585
586 *ppos += read;
587 ret = read;
588
589out:
590 return ret;
591}
592
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200593ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
594{
595 int len;
596 int ret;
597
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500598 if (!cnt)
599 return 0;
600
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200601 if (s->len <= s->readpos)
602 return -EBUSY;
603
604 len = s->len - s->readpos;
605 if (cnt > len)
606 cnt = len;
607 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500608 if (ret == cnt)
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200609 return -EFAULT;
610
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500611 cnt -= ret;
612
Steven Rostedte74da522009-03-04 20:31:11 -0500613 s->readpos += cnt;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200614 return cnt;
Steven Rostedt214023c2008-05-12 21:20:46 +0200615}
616
Dmitri Vorobievb8b94262009-03-22 19:11:11 +0200617static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200618{
619 int len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200620
621 if (s->len <= s->readpos)
622 return -EBUSY;
623
624 len = s->len - s->readpos;
625 if (cnt > len)
626 cnt = len;
Dan Carpenter5a26c8f2012-04-20 09:31:45 +0300627 memcpy(buf, s->buffer + s->readpos, cnt);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200628
Steven Rostedte74da522009-03-04 20:31:11 -0500629 s->readpos += cnt;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200630 return cnt;
631}
632
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400633/*
634 * ftrace_max_lock is used to protect the swapping of buffers
635 * when taking a max snapshot. The buffers themselves are
636 * protected by per_cpu spinlocks. But the action of the swap
637 * needs its own lock.
638 *
Thomas Gleixner445c8952009-12-02 19:49:50 +0100639 * This is defined as a arch_spinlock_t in order to help
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400640 * with performance when lockdep debugging is enabled.
641 *
642 * It is also used in other places outside the update_max_tr
643 * so it needs to be defined outside of the
644 * CONFIG_TRACER_MAX_TRACE.
645 */
Thomas Gleixner445c8952009-12-02 19:49:50 +0100646static arch_spinlock_t ftrace_max_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100647 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400648
Tim Bird0e950172010-02-25 15:36:43 -0800649unsigned long __read_mostly tracing_thresh;
650
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400651#ifdef CONFIG_TRACER_MAX_TRACE
652unsigned long __read_mostly tracing_max_latency;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400653
654/*
655 * Copy the new maximum trace into the separate maximum-trace
656 * structure. (this way the maximum trace is permanently saved,
657 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
658 */
659static void
660__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
661{
662 struct trace_array_cpu *data = tr->data[cpu];
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300663 struct trace_array_cpu *max_data;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400664
665 max_tr.cpu = cpu;
666 max_tr.time_start = data->preempt_timestamp;
667
Steven Rostedt8248ac02009-09-02 12:27:41 -0400668 max_data = max_tr.data[cpu];
669 max_data->saved_latency = tracing_max_latency;
670 max_data->critical_start = data->critical_start;
671 max_data->critical_end = data->critical_end;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400672
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300673 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
Steven Rostedt8248ac02009-09-02 12:27:41 -0400674 max_data->pid = tsk->pid;
675 max_data->uid = task_uid(tsk);
676 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
677 max_data->policy = tsk->policy;
678 max_data->rt_priority = tsk->rt_priority;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400679
680 /* record this tasks comm */
681 tracing_record_cmdline(tsk);
682}
683
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200684/**
685 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
686 * @tr: tracer
687 * @tsk: the task with the latency
688 * @cpu: The cpu that initiated the trace.
689 *
690 * Flip the buffers between the @tr and the max_tr and record information
691 * about which task was the cause of this latency.
692 */
Ingo Molnare309b412008-05-12 21:20:51 +0200693void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200694update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
695{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400696 struct ring_buffer *buf = tr->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200697
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400698 if (trace_stop_count)
699 return;
700
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200701 WARN_ON_ONCE(!irqs_disabled());
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900702 if (!current_trace->use_max_tr) {
703 WARN_ON_ONCE(1);
704 return;
705 }
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100706 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400707
708 tr->buffer = max_tr.buffer;
709 max_tr.buffer = buf;
710
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200711 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100712 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200713}
714
715/**
716 * update_max_tr_single - only copy one trace over, and reset the rest
717 * @tr - tracer
718 * @tsk - task with the latency
719 * @cpu - the cpu of the buffer to copy.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200720 *
721 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200722 */
Ingo Molnare309b412008-05-12 21:20:51 +0200723void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200724update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
725{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400726 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200727
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400728 if (trace_stop_count)
729 return;
730
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200731 WARN_ON_ONCE(!irqs_disabled());
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900732 if (!current_trace->use_max_tr) {
733 WARN_ON_ONCE(1);
734 return;
735 }
736
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100737 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200738
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400739 ret = ring_buffer_swap_cpu(max_tr.buffer, tr->buffer, cpu);
740
Steven Rostedte8165db2009-09-03 19:13:05 -0400741 if (ret == -EBUSY) {
742 /*
743 * We failed to swap the buffer due to a commit taking
744 * place on this CPU. We fail to record, but we reset
745 * the max trace buffer (no one writes directly to it)
746 * and flag that it failed.
747 */
748 trace_array_printk(&max_tr, _THIS_IP_,
749 "Failed to swap buffers due to commit in progress\n");
750 }
751
Steven Rostedte8165db2009-09-03 19:13:05 -0400752 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200753
754 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100755 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200756}
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400757#endif /* CONFIG_TRACER_MAX_TRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200758
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200759/**
760 * register_tracer - register a tracer with the ftrace system.
761 * @type - the plugin for the tracer
762 *
763 * Register a new plugin tracer.
764 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200765int register_tracer(struct tracer *type)
766{
767 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200768 int ret = 0;
769
770 if (!type->name) {
771 pr_info("Tracer must have a name\n");
772 return -1;
773 }
774
Dan Carpenter24a461d2010-07-10 12:06:44 +0200775 if (strlen(type->name) >= MAX_TRACER_SIZE) {
Li Zefanee6c2c12009-09-18 14:06:47 +0800776 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
777 return -1;
778 }
779
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200780 mutex_lock(&trace_types_lock);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100781
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +0100782 tracing_selftest_running = true;
783
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200784 for (t = trace_types; t; t = t->next) {
785 if (strcmp(type->name, t->name) == 0) {
786 /* already found */
Li Zefanee6c2c12009-09-18 14:06:47 +0800787 pr_info("Tracer %s already registered\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200788 type->name);
789 ret = -1;
790 goto out;
791 }
792 }
793
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +0100794 if (!type->set_flag)
795 type->set_flag = &dummy_set_flag;
796 if (!type->flags)
797 type->flags = &dummy_tracer_flags;
798 else
799 if (!type->flags->opts)
800 type->flags->opts = dummy_tracer_opt;
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +0100801 if (!type->wait_pipe)
802 type->wait_pipe = default_wait_pipe;
803
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +0100804
Steven Rostedt60a11772008-05-12 21:20:44 +0200805#ifdef CONFIG_FTRACE_STARTUP_TEST
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500806 if (type->selftest && !tracing_selftest_disabled) {
Steven Rostedt60a11772008-05-12 21:20:44 +0200807 struct tracer *saved_tracer = current_trace;
Steven Rostedt60a11772008-05-12 21:20:44 +0200808 struct trace_array *tr = &global_trace;
Frederic Weisbeckerff325042008-12-04 23:47:35 +0100809
Steven Rostedt60a11772008-05-12 21:20:44 +0200810 /*
811 * Run a selftest on this tracer.
812 * Here we reset the trace buffer, and set the current
813 * tracer to be this tracer. The tracer can then run some
814 * internal tracing to verify that everything is in order.
815 * If we fail, we do not register this tracer.
816 */
Steven Rostedt76f0d072009-09-04 12:12:39 -0400817 tracing_reset_online_cpus(tr);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100818
Steven Rostedt60a11772008-05-12 21:20:44 +0200819 current_trace = type;
Steven Rostedt4a0b1662011-03-09 20:09:26 -0500820
821 /* If we expanded the buffers, make sure the max is expanded too */
822 if (ring_buffer_expanded && type->use_max_tr)
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -0800823 ring_buffer_resize(max_tr.buffer, trace_buf_size,
824 RING_BUFFER_ALL_CPUS);
Steven Rostedt4a0b1662011-03-09 20:09:26 -0500825
Steven Rostedt60a11772008-05-12 21:20:44 +0200826 /* the test is responsible for initializing and enabling */
827 pr_info("Testing tracer %s: ", type->name);
828 ret = type->selftest(type, tr);
829 /* the test is responsible for resetting too */
830 current_trace = saved_tracer;
Steven Rostedt60a11772008-05-12 21:20:44 +0200831 if (ret) {
832 printk(KERN_CONT "FAILED!\n");
Steven Rostedt0be61eb2012-06-18 09:28:16 -0400833 /* Add the warning after printing 'FAILED' */
834 WARN_ON(1);
Steven Rostedt60a11772008-05-12 21:20:44 +0200835 goto out;
836 }
Steven Rostedt1d4db002008-05-12 21:20:45 +0200837 /* Only reset on passing, to avoid touching corrupted buffers */
Steven Rostedt76f0d072009-09-04 12:12:39 -0400838 tracing_reset_online_cpus(tr);
Ingo Molnar86fa2f62008-11-19 10:00:15 +0100839
Steven Rostedt4a0b1662011-03-09 20:09:26 -0500840 /* Shrink the max buffer again */
841 if (ring_buffer_expanded && type->use_max_tr)
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -0800842 ring_buffer_resize(max_tr.buffer, 1,
843 RING_BUFFER_ALL_CPUS);
Steven Rostedt4a0b1662011-03-09 20:09:26 -0500844
Steven Rostedt60a11772008-05-12 21:20:44 +0200845 printk(KERN_CONT "PASSED\n");
846 }
847#endif
848
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200849 type->next = trace_types;
850 trace_types = type;
Steven Rostedt60a11772008-05-12 21:20:44 +0200851
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200852 out:
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +0100853 tracing_selftest_running = false;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200854 mutex_unlock(&trace_types_lock);
855
Steven Rostedtdac74942009-02-05 01:13:38 -0500856 if (ret || !default_bootup_tracer)
857 goto out_unlock;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500858
Li Zefanee6c2c12009-09-18 14:06:47 +0800859 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
Steven Rostedtdac74942009-02-05 01:13:38 -0500860 goto out_unlock;
861
862 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
863 /* Do we want this tracer to start on bootup? */
864 tracing_set_tracer(type->name);
865 default_bootup_tracer = NULL;
866 /* disable other selftests, since this will break it. */
867 tracing_selftest_disabled = 1;
868#ifdef CONFIG_FTRACE_STARTUP_TEST
869 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
870 type->name);
871#endif
872
873 out_unlock:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200874 return ret;
875}
876
877void unregister_tracer(struct tracer *type)
878{
879 struct tracer **t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200880
881 mutex_lock(&trace_types_lock);
882 for (t = &trace_types; *t; t = &(*t)->next) {
883 if (*t == type)
884 goto found;
885 }
Li Zefanee6c2c12009-09-18 14:06:47 +0800886 pr_info("Tracer %s not registered\n", type->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200887 goto out;
888
889 found:
890 *t = (*t)->next;
Arnaldo Carvalho de Melob5db03c2009-02-07 18:52:59 -0200891
892 if (type == current_trace && tracer_enabled) {
893 tracer_enabled = 0;
894 tracing_stop();
895 if (current_trace->stop)
896 current_trace->stop(&global_trace);
897 current_trace = &nop_trace;
898 }
Li Zefanee6c2c12009-09-18 14:06:47 +0800899out:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200900 mutex_unlock(&trace_types_lock);
901}
902
Steven Rostedtf6339032009-09-04 12:35:16 -0400903void tracing_reset(struct trace_array *tr, int cpu)
904{
905 struct ring_buffer *buffer = tr->buffer;
906
907 ring_buffer_record_disable(buffer);
908
909 /* Make sure all commits have finished */
910 synchronize_sched();
Steven Rostedt68179682012-05-08 20:57:53 -0400911 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedtf6339032009-09-04 12:35:16 -0400912
913 ring_buffer_record_enable(buffer);
914}
915
Pekka J Enberg213cc062008-12-19 12:08:39 +0200916void tracing_reset_online_cpus(struct trace_array *tr)
917{
Steven Rostedt621968c2009-09-04 12:02:35 -0400918 struct ring_buffer *buffer = tr->buffer;
Pekka J Enberg213cc062008-12-19 12:08:39 +0200919 int cpu;
920
Steven Rostedt621968c2009-09-04 12:02:35 -0400921 ring_buffer_record_disable(buffer);
922
923 /* Make sure all commits have finished */
924 synchronize_sched();
925
Pekka J Enberg213cc062008-12-19 12:08:39 +0200926 tr->time_start = ftrace_now(tr->cpu);
927
928 for_each_online_cpu(cpu)
Steven Rostedt68179682012-05-08 20:57:53 -0400929 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedt621968c2009-09-04 12:02:35 -0400930
931 ring_buffer_record_enable(buffer);
Pekka J Enberg213cc062008-12-19 12:08:39 +0200932}
933
Steven Rostedt9456f0f2009-05-06 21:54:09 -0400934void tracing_reset_current(int cpu)
935{
936 tracing_reset(&global_trace, cpu);
937}
938
939void tracing_reset_current_online_cpus(void)
940{
941 tracing_reset_online_cpus(&global_trace);
942}
943
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200944#define SAVED_CMDLINES 128
Thomas Gleixner2c7eea42009-03-18 09:03:19 +0100945#define NO_CMDLINE_MAP UINT_MAX
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200946static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
947static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
948static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
949static int cmdline_idx;
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100950static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt25b0b442008-05-12 21:21:00 +0200951
Steven Rostedt25b0b442008-05-12 21:21:00 +0200952/* temporary disable recording */
Hannes Eder4fd27352009-02-10 19:44:12 +0100953static atomic_t trace_record_cmdline_disabled __read_mostly;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200954
955static void trace_init_cmdlines(void)
956{
Thomas Gleixner2c7eea42009-03-18 09:03:19 +0100957 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
958 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200959 cmdline_idx = 0;
960}
961
Carsten Emdeb5130b12009-09-13 01:43:07 +0200962int is_tracing_stopped(void)
963{
964 return trace_stop_count;
965}
966
Steven Rostedt0f048702008-11-05 16:05:44 -0500967/**
Steven Rostedt69bb54e2008-11-21 12:59:38 -0500968 * ftrace_off_permanent - disable all ftrace code permanently
969 *
970 * This should only be called when a serious anomally has
971 * been detected. This will turn off the function tracing,
972 * ring buffers, and other tracing utilites. It takes no
973 * locks and can be called from any context.
974 */
975void ftrace_off_permanent(void)
976{
977 tracing_disabled = 1;
978 ftrace_stop();
979 tracing_off_permanent();
980}
981
982/**
Steven Rostedt0f048702008-11-05 16:05:44 -0500983 * tracing_start - quick start of the tracer
984 *
985 * If tracing is enabled but was stopped by tracing_stop,
986 * this will start the tracer back up.
987 */
988void tracing_start(void)
989{
990 struct ring_buffer *buffer;
991 unsigned long flags;
992
993 if (tracing_disabled)
994 return;
995
Thomas Gleixner5389f6f2009-07-25 17:13:33 +0200996 raw_spin_lock_irqsave(&tracing_start_lock, flags);
Steven Rostedtb06a8302009-01-22 14:26:15 -0500997 if (--trace_stop_count) {
998 if (trace_stop_count < 0) {
999 /* Someone screwed up their debugging */
1000 WARN_ON_ONCE(1);
1001 trace_stop_count = 0;
1002 }
Steven Rostedt0f048702008-11-05 16:05:44 -05001003 goto out;
1004 }
1005
Steven Rostedta2f80712010-03-12 19:56:00 -05001006 /* Prevent the buffers from switching */
1007 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt0f048702008-11-05 16:05:44 -05001008
1009 buffer = global_trace.buffer;
1010 if (buffer)
1011 ring_buffer_record_enable(buffer);
1012
1013 buffer = max_tr.buffer;
1014 if (buffer)
1015 ring_buffer_record_enable(buffer);
1016
Steven Rostedta2f80712010-03-12 19:56:00 -05001017 arch_spin_unlock(&ftrace_max_lock);
1018
Steven Rostedt0f048702008-11-05 16:05:44 -05001019 ftrace_start();
1020 out:
Thomas Gleixner5389f6f2009-07-25 17:13:33 +02001021 raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001022}
1023
1024/**
1025 * tracing_stop - quick stop of the tracer
1026 *
1027 * Light weight way to stop tracing. Use in conjunction with
1028 * tracing_start.
1029 */
1030void tracing_stop(void)
1031{
1032 struct ring_buffer *buffer;
1033 unsigned long flags;
1034
1035 ftrace_stop();
Thomas Gleixner5389f6f2009-07-25 17:13:33 +02001036 raw_spin_lock_irqsave(&tracing_start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001037 if (trace_stop_count++)
1038 goto out;
1039
Steven Rostedta2f80712010-03-12 19:56:00 -05001040 /* Prevent the buffers from switching */
1041 arch_spin_lock(&ftrace_max_lock);
1042
Steven Rostedt0f048702008-11-05 16:05:44 -05001043 buffer = global_trace.buffer;
1044 if (buffer)
1045 ring_buffer_record_disable(buffer);
1046
1047 buffer = max_tr.buffer;
1048 if (buffer)
1049 ring_buffer_record_disable(buffer);
1050
Steven Rostedta2f80712010-03-12 19:56:00 -05001051 arch_spin_unlock(&ftrace_max_lock);
1052
Steven Rostedt0f048702008-11-05 16:05:44 -05001053 out:
Thomas Gleixner5389f6f2009-07-25 17:13:33 +02001054 raw_spin_unlock_irqrestore(&tracing_start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001055}
1056
Ingo Molnare309b412008-05-12 21:20:51 +02001057void trace_stop_cmdline_recording(void);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001058
Ingo Molnare309b412008-05-12 21:20:51 +02001059static void trace_save_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001060{
Carsten Emdea635cf02009-03-18 09:00:41 +01001061 unsigned pid, idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001062
1063 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1064 return;
1065
1066 /*
1067 * It's not the end of the world if we don't get
1068 * the lock, but we also don't want to spin
1069 * nor do we want to disable interrupts,
1070 * so if we miss here, then better luck next time.
1071 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001072 if (!arch_spin_trylock(&trace_cmdline_lock))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001073 return;
1074
1075 idx = map_pid_to_cmdline[tsk->pid];
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001076 if (idx == NO_CMDLINE_MAP) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001077 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1078
Carsten Emdea635cf02009-03-18 09:00:41 +01001079 /*
1080 * Check whether the cmdline buffer at idx has a pid
1081 * mapped. We are going to overwrite that entry so we
1082 * need to clear the map_pid_to_cmdline. Otherwise we
1083 * would read the new comm for the old pid.
1084 */
1085 pid = map_cmdline_to_pid[idx];
1086 if (pid != NO_CMDLINE_MAP)
1087 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001088
Carsten Emdea635cf02009-03-18 09:00:41 +01001089 map_cmdline_to_pid[idx] = tsk->pid;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001090 map_pid_to_cmdline[tsk->pid] = idx;
1091
1092 cmdline_idx = idx;
1093 }
1094
1095 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1096
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001097 arch_spin_unlock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001098}
1099
Steven Rostedt4ca53082009-03-16 19:20:15 -04001100void trace_find_cmdline(int pid, char comm[])
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001101{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001102 unsigned map;
1103
Steven Rostedt4ca53082009-03-16 19:20:15 -04001104 if (!pid) {
1105 strcpy(comm, "<idle>");
1106 return;
1107 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001108
Steven Rostedt74bf4072010-01-25 15:11:53 -05001109 if (WARN_ON_ONCE(pid < 0)) {
1110 strcpy(comm, "<XXX>");
1111 return;
1112 }
1113
Steven Rostedt4ca53082009-03-16 19:20:15 -04001114 if (pid > PID_MAX_DEFAULT) {
1115 strcpy(comm, "<...>");
1116 return;
1117 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001118
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001119 preempt_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001120 arch_spin_lock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001121 map = map_pid_to_cmdline[pid];
Thomas Gleixner50d88752009-03-18 08:58:44 +01001122 if (map != NO_CMDLINE_MAP)
1123 strcpy(comm, saved_cmdlines[map]);
1124 else
1125 strcpy(comm, "<...>");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001126
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001127 arch_spin_unlock(&trace_cmdline_lock);
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001128 preempt_enable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001129}
1130
Ingo Molnare309b412008-05-12 21:20:51 +02001131void tracing_record_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001132{
Thomas Gleixner18aecd32009-03-18 08:56:58 +01001133 if (atomic_read(&trace_record_cmdline_disabled) || !tracer_enabled ||
1134 !tracing_is_on())
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001135 return;
1136
1137 trace_save_cmdline(tsk);
1138}
1139
Pekka Paalanen45dcd8b2008-09-16 21:56:41 +03001140void
Steven Rostedt38697052008-10-01 13:14:09 -04001141tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1142 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001143{
1144 struct task_struct *tsk = current;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001145
Steven Rostedt777e2082008-09-29 23:02:42 -04001146 entry->preempt_count = pc & 0xff;
1147 entry->pid = (tsk) ? tsk->pid : 0;
Arjan van de Vena3a4a5a2011-05-05 23:55:18 -04001148 entry->padding = 0;
Steven Rostedt777e2082008-09-29 23:02:42 -04001149 entry->flags =
Steven Rostedt92444892008-10-24 09:42:59 -04001150#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
Steven Rostedt2e2ca152008-08-01 12:26:40 -04001151 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
Steven Rostedt92444892008-10-24 09:42:59 -04001152#else
1153 TRACE_FLAG_IRQS_NOSUPPORT |
1154#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001155 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1156 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1157 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1158}
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +02001159EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001160
Steven Rostedte77405a2009-09-02 14:17:06 -04001161struct ring_buffer_event *
1162trace_buffer_lock_reserve(struct ring_buffer *buffer,
1163 int type,
1164 unsigned long len,
1165 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001166{
1167 struct ring_buffer_event *event;
1168
Steven Rostedte77405a2009-09-02 14:17:06 -04001169 event = ring_buffer_lock_reserve(buffer, len);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001170 if (event != NULL) {
1171 struct trace_entry *ent = ring_buffer_event_data(event);
1172
1173 tracing_generic_entry_update(ent, flags, pc);
1174 ent->type = type;
1175 }
1176
1177 return event;
1178}
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001179
Steven Rostedte77405a2009-09-02 14:17:06 -04001180static inline void
1181__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1182 struct ring_buffer_event *event,
1183 unsigned long flags, int pc,
1184 int wake)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001185{
Steven Rostedte77405a2009-09-02 14:17:06 -04001186 ring_buffer_unlock_commit(buffer, event);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001187
Steven Rostedte77405a2009-09-02 14:17:06 -04001188 ftrace_trace_stack(buffer, flags, 6, pc);
1189 ftrace_trace_userstack(buffer, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001190
1191 if (wake)
1192 trace_wake_up();
1193}
1194
Steven Rostedte77405a2009-09-02 14:17:06 -04001195void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1196 struct ring_buffer_event *event,
1197 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001198{
Steven Rostedte77405a2009-09-02 14:17:06 -04001199 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001200}
1201
Steven Rostedtef5580d2009-02-27 19:38:04 -05001202struct ring_buffer_event *
Steven Rostedte77405a2009-09-02 14:17:06 -04001203trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1204 int type, unsigned long len,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001205 unsigned long flags, int pc)
1206{
Steven Rostedte77405a2009-09-02 14:17:06 -04001207 *current_rb = global_trace.buffer;
1208 return trace_buffer_lock_reserve(*current_rb,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001209 type, len, flags, pc);
1210}
Steven Rostedt94487d62009-05-05 19:22:53 -04001211EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001212
Steven Rostedte77405a2009-09-02 14:17:06 -04001213void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1214 struct ring_buffer_event *event,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001215 unsigned long flags, int pc)
1216{
Steven Rostedte77405a2009-09-02 14:17:06 -04001217 __trace_buffer_unlock_commit(buffer, event, flags, pc, 1);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001218}
Steven Rostedt94487d62009-05-05 19:22:53 -04001219EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001220
Steven Rostedte77405a2009-09-02 14:17:06 -04001221void trace_nowake_buffer_unlock_commit(struct ring_buffer *buffer,
1222 struct ring_buffer_event *event,
1223 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001224{
Steven Rostedte77405a2009-09-02 14:17:06 -04001225 __trace_buffer_unlock_commit(buffer, event, flags, pc, 0);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001226}
Steven Rostedt94487d62009-05-05 19:22:53 -04001227EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit);
Steven Rostedt77d9f462009-04-02 01:16:59 -04001228
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001229void trace_nowake_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1230 struct ring_buffer_event *event,
1231 unsigned long flags, int pc,
1232 struct pt_regs *regs)
1233{
1234 ring_buffer_unlock_commit(buffer, event);
1235
1236 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1237 ftrace_trace_userstack(buffer, flags, pc);
1238}
1239EXPORT_SYMBOL_GPL(trace_nowake_buffer_unlock_commit_regs);
1240
Steven Rostedte77405a2009-09-02 14:17:06 -04001241void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1242 struct ring_buffer_event *event)
Steven Rostedt77d9f462009-04-02 01:16:59 -04001243{
Steven Rostedte77405a2009-09-02 14:17:06 -04001244 ring_buffer_discard_commit(buffer, event);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001245}
Steven Rostedt12acd472009-04-17 16:01:56 -04001246EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001247
Ingo Molnare309b412008-05-12 21:20:51 +02001248void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001249trace_function(struct trace_array *tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001250 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1251 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001252{
Tom Zanussie1112b42009-03-31 00:48:49 -05001253 struct ftrace_event_call *call = &event_function;
Steven Rostedte77405a2009-09-02 14:17:06 -04001254 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001255 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001256 struct ftrace_entry *entry;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001257
Steven Rostedtd7690412008-10-01 00:29:53 -04001258 /* If we are reading the ring buffer, don't trace */
Rusty Russelldd17c8f2009-10-29 22:34:15 +09001259 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
Steven Rostedtd7690412008-10-01 00:29:53 -04001260 return;
1261
Steven Rostedte77405a2009-09-02 14:17:06 -04001262 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001263 flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001264 if (!event)
1265 return;
1266 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001267 entry->ip = ip;
1268 entry->parent_ip = parent_ip;
Tom Zanussie1112b42009-03-31 00:48:49 -05001269
Steven Rostedte77405a2009-09-02 14:17:06 -04001270 if (!filter_check_discard(call, entry, buffer, event))
1271 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001272}
1273
Ingo Molnare309b412008-05-12 21:20:51 +02001274void
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001275ftrace(struct trace_array *tr, struct trace_array_cpu *data,
Steven Rostedt38697052008-10-01 13:14:09 -04001276 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1277 int pc)
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001278{
1279 if (likely(!atomic_read(&data->disabled)))
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001280 trace_function(tr, ip, parent_ip, flags, pc);
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001281}
1282
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001283#ifdef CONFIG_STACKTRACE
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001284
1285#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1286struct ftrace_stack {
1287 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1288};
1289
1290static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1291static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1292
Steven Rostedte77405a2009-09-02 14:17:06 -04001293static void __ftrace_trace_stack(struct ring_buffer *buffer,
Steven Rostedt53614992009-01-15 19:12:40 -05001294 unsigned long flags,
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001295 int skip, int pc, struct pt_regs *regs)
Ingo Molnar86387f72008-05-12 21:20:51 +02001296{
Tom Zanussie1112b42009-03-31 00:48:49 -05001297 struct ftrace_event_call *call = &event_kernel_stack;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001298 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001299 struct stack_entry *entry;
Ingo Molnar86387f72008-05-12 21:20:51 +02001300 struct stack_trace trace;
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001301 int use_stack;
1302 int size = FTRACE_STACK_ENTRIES;
Ingo Molnar86387f72008-05-12 21:20:51 +02001303
1304 trace.nr_entries = 0;
Ingo Molnar86387f72008-05-12 21:20:51 +02001305 trace.skip = skip;
Ingo Molnar86387f72008-05-12 21:20:51 +02001306
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001307 /*
1308 * Since events can happen in NMIs there's no safe way to
1309 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1310 * or NMI comes in, it will just have to use the default
1311 * FTRACE_STACK_SIZE.
1312 */
1313 preempt_disable_notrace();
1314
1315 use_stack = ++__get_cpu_var(ftrace_stack_reserve);
1316 /*
1317 * We don't need any atomic variables, just a barrier.
1318 * If an interrupt comes in, we don't care, because it would
1319 * have exited and put the counter back to what we want.
1320 * We just need a barrier to keep gcc from moving things
1321 * around.
1322 */
1323 barrier();
1324 if (use_stack == 1) {
1325 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1326 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1327
1328 if (regs)
1329 save_stack_trace_regs(regs, &trace);
1330 else
1331 save_stack_trace(&trace);
1332
1333 if (trace.nr_entries > size)
1334 size = trace.nr_entries;
1335 } else
1336 /* From now on, use_stack is a boolean */
1337 use_stack = 0;
1338
1339 size *= sizeof(unsigned long);
1340
1341 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1342 sizeof(*entry) + size, flags, pc);
1343 if (!event)
1344 goto out;
1345 entry = ring_buffer_event_data(event);
1346
1347 memset(&entry->caller, 0, size);
1348
1349 if (use_stack)
1350 memcpy(&entry->caller, trace.entries,
1351 trace.nr_entries * sizeof(unsigned long));
1352 else {
1353 trace.max_entries = FTRACE_STACK_ENTRIES;
1354 trace.entries = entry->caller;
1355 if (regs)
1356 save_stack_trace_regs(regs, &trace);
1357 else
1358 save_stack_trace(&trace);
1359 }
1360
1361 entry->size = trace.nr_entries;
1362
Steven Rostedte77405a2009-09-02 14:17:06 -04001363 if (!filter_check_discard(call, entry, buffer, event))
1364 ring_buffer_unlock_commit(buffer, event);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001365
1366 out:
1367 /* Again, don't let gcc optimize things here */
1368 barrier();
1369 __get_cpu_var(ftrace_stack_reserve)--;
1370 preempt_enable_notrace();
1371
Ingo Molnarf0a920d2008-05-12 21:20:47 +02001372}
1373
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001374void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1375 int skip, int pc, struct pt_regs *regs)
1376{
1377 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1378 return;
1379
1380 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1381}
1382
Steven Rostedte77405a2009-09-02 14:17:06 -04001383void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1384 int skip, int pc)
Steven Rostedt53614992009-01-15 19:12:40 -05001385{
1386 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1387 return;
1388
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001389 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
Steven Rostedt53614992009-01-15 19:12:40 -05001390}
1391
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001392void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1393 int pc)
Steven Rostedt38697052008-10-01 13:14:09 -04001394{
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001395 __ftrace_trace_stack(tr->buffer, flags, skip, pc, NULL);
Steven Rostedt38697052008-10-01 13:14:09 -04001396}
1397
Steven Rostedt03889382009-12-11 09:48:22 -05001398/**
1399 * trace_dump_stack - record a stack back trace in the trace buffer
1400 */
1401void trace_dump_stack(void)
1402{
1403 unsigned long flags;
1404
1405 if (tracing_disabled || tracing_selftest_running)
Steven Rostedte36c5452009-12-14 15:58:33 -05001406 return;
Steven Rostedt03889382009-12-11 09:48:22 -05001407
1408 local_save_flags(flags);
1409
1410 /* skipping 3 traces, seems to get us at the caller of this function */
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001411 __ftrace_trace_stack(global_trace.buffer, flags, 3, preempt_count(), NULL);
Steven Rostedt03889382009-12-11 09:48:22 -05001412}
1413
Steven Rostedt91e86e52010-11-10 12:56:12 +01001414static DEFINE_PER_CPU(int, user_stack_count);
1415
Steven Rostedte77405a2009-09-02 14:17:06 -04001416void
1417ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
Török Edwin02b67512008-11-22 13:28:47 +02001418{
Tom Zanussie1112b42009-03-31 00:48:49 -05001419 struct ftrace_event_call *call = &event_user_stack;
Török Edwin8d7c6a92008-11-23 12:39:06 +02001420 struct ring_buffer_event *event;
Török Edwin02b67512008-11-22 13:28:47 +02001421 struct userstack_entry *entry;
1422 struct stack_trace trace;
Török Edwin02b67512008-11-22 13:28:47 +02001423
1424 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1425 return;
1426
Steven Rostedtb6345872010-03-12 20:03:30 -05001427 /*
1428 * NMIs can not handle page faults, even with fix ups.
1429 * The save user stack can (and often does) fault.
1430 */
1431 if (unlikely(in_nmi()))
1432 return;
1433
Steven Rostedt91e86e52010-11-10 12:56:12 +01001434 /*
1435 * prevent recursion, since the user stack tracing may
1436 * trigger other kernel events.
1437 */
1438 preempt_disable();
1439 if (__this_cpu_read(user_stack_count))
1440 goto out;
1441
1442 __this_cpu_inc(user_stack_count);
1443
Steven Rostedte77405a2009-09-02 14:17:06 -04001444 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001445 sizeof(*entry), flags, pc);
Török Edwin02b67512008-11-22 13:28:47 +02001446 if (!event)
Li Zefan1dbd1952010-12-09 15:47:56 +08001447 goto out_drop_count;
Török Edwin02b67512008-11-22 13:28:47 +02001448 entry = ring_buffer_event_data(event);
Török Edwin02b67512008-11-22 13:28:47 +02001449
Steven Rostedt48659d32009-09-11 11:36:23 -04001450 entry->tgid = current->tgid;
Török Edwin02b67512008-11-22 13:28:47 +02001451 memset(&entry->caller, 0, sizeof(entry->caller));
1452
1453 trace.nr_entries = 0;
1454 trace.max_entries = FTRACE_STACK_ENTRIES;
1455 trace.skip = 0;
1456 trace.entries = entry->caller;
1457
1458 save_stack_trace_user(&trace);
Steven Rostedte77405a2009-09-02 14:17:06 -04001459 if (!filter_check_discard(call, entry, buffer, event))
1460 ring_buffer_unlock_commit(buffer, event);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001461
Li Zefan1dbd1952010-12-09 15:47:56 +08001462 out_drop_count:
Steven Rostedt91e86e52010-11-10 12:56:12 +01001463 __this_cpu_dec(user_stack_count);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001464 out:
1465 preempt_enable();
Török Edwin02b67512008-11-22 13:28:47 +02001466}
1467
Hannes Eder4fd27352009-02-10 19:44:12 +01001468#ifdef UNUSED
1469static void __trace_userstack(struct trace_array *tr, unsigned long flags)
Török Edwin02b67512008-11-22 13:28:47 +02001470{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001471 ftrace_trace_userstack(tr, flags, preempt_count());
Török Edwin02b67512008-11-22 13:28:47 +02001472}
Hannes Eder4fd27352009-02-10 19:44:12 +01001473#endif /* UNUSED */
Török Edwin02b67512008-11-22 13:28:47 +02001474
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001475#endif /* CONFIG_STACKTRACE */
1476
Steven Rostedt07d777f2011-09-22 14:01:55 -04001477/* created for use with alloc_percpu */
1478struct trace_buffer_struct {
1479 char buffer[TRACE_BUF_SIZE];
1480};
1481
1482static struct trace_buffer_struct *trace_percpu_buffer;
1483static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1484static struct trace_buffer_struct *trace_percpu_irq_buffer;
1485static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1486
1487/*
1488 * The buffer used is dependent on the context. There is a per cpu
1489 * buffer for normal context, softirq contex, hard irq context and
1490 * for NMI context. Thise allows for lockless recording.
1491 *
1492 * Note, if the buffers failed to be allocated, then this returns NULL
1493 */
1494static char *get_trace_buf(void)
1495{
1496 struct trace_buffer_struct *percpu_buffer;
1497 struct trace_buffer_struct *buffer;
1498
1499 /*
1500 * If we have allocated per cpu buffers, then we do not
1501 * need to do any locking.
1502 */
1503 if (in_nmi())
1504 percpu_buffer = trace_percpu_nmi_buffer;
1505 else if (in_irq())
1506 percpu_buffer = trace_percpu_irq_buffer;
1507 else if (in_softirq())
1508 percpu_buffer = trace_percpu_sirq_buffer;
1509 else
1510 percpu_buffer = trace_percpu_buffer;
1511
1512 if (!percpu_buffer)
1513 return NULL;
1514
1515 buffer = per_cpu_ptr(percpu_buffer, smp_processor_id());
1516
1517 return buffer->buffer;
1518}
1519
1520static int alloc_percpu_trace_buffer(void)
1521{
1522 struct trace_buffer_struct *buffers;
1523 struct trace_buffer_struct *sirq_buffers;
1524 struct trace_buffer_struct *irq_buffers;
1525 struct trace_buffer_struct *nmi_buffers;
1526
1527 buffers = alloc_percpu(struct trace_buffer_struct);
1528 if (!buffers)
1529 goto err_warn;
1530
1531 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1532 if (!sirq_buffers)
1533 goto err_sirq;
1534
1535 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1536 if (!irq_buffers)
1537 goto err_irq;
1538
1539 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1540 if (!nmi_buffers)
1541 goto err_nmi;
1542
1543 trace_percpu_buffer = buffers;
1544 trace_percpu_sirq_buffer = sirq_buffers;
1545 trace_percpu_irq_buffer = irq_buffers;
1546 trace_percpu_nmi_buffer = nmi_buffers;
1547
1548 return 0;
1549
1550 err_nmi:
1551 free_percpu(irq_buffers);
1552 err_irq:
1553 free_percpu(sirq_buffers);
1554 err_sirq:
1555 free_percpu(buffers);
1556 err_warn:
1557 WARN(1, "Could not allocate percpu trace_printk buffer");
1558 return -ENOMEM;
1559}
1560
1561void trace_printk_init_buffers(void)
1562{
1563 static int buffers_allocated;
1564
1565 if (buffers_allocated)
1566 return;
1567
1568 if (alloc_percpu_trace_buffer())
1569 return;
1570
1571 pr_info("ftrace: Allocated trace_printk buffers\n");
1572
1573 buffers_allocated = 1;
1574}
1575
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001576/**
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001577 * trace_vbprintk - write binary msg to tracing buffer
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001578 *
1579 */
Steven Rostedt40ce74f2009-03-19 14:03:53 -04001580int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001581{
Tom Zanussie1112b42009-03-31 00:48:49 -05001582 struct ftrace_event_call *call = &event_bprint;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001583 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001584 struct ring_buffer *buffer;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001585 struct trace_array *tr = &global_trace;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001586 struct bprint_entry *entry;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001587 unsigned long flags;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001588 char *tbuffer;
1589 int len = 0, size, pc;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001590
1591 if (unlikely(tracing_selftest_running || tracing_disabled))
1592 return 0;
1593
1594 /* Don't pollute graph traces with trace_vprintk internals */
1595 pause_graph_tracing();
1596
1597 pc = preempt_count();
Steven Rostedt5168ae52010-06-03 09:36:50 -04001598 preempt_disable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001599
Steven Rostedt07d777f2011-09-22 14:01:55 -04001600 tbuffer = get_trace_buf();
1601 if (!tbuffer) {
1602 len = 0;
1603 goto out;
1604 }
1605
1606 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1607
1608 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001609 goto out;
1610
Steven Rostedt07d777f2011-09-22 14:01:55 -04001611 local_save_flags(flags);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001612 size = sizeof(*entry) + sizeof(u32) * len;
Steven Rostedte77405a2009-09-02 14:17:06 -04001613 buffer = tr->buffer;
1614 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1615 flags, pc);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001616 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04001617 goto out;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001618 entry = ring_buffer_event_data(event);
1619 entry->ip = ip;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001620 entry->fmt = fmt;
1621
Steven Rostedt07d777f2011-09-22 14:01:55 -04001622 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
Steven Rostedtd9313692010-01-06 17:27:11 -05001623 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedte77405a2009-09-02 14:17:06 -04001624 ring_buffer_unlock_commit(buffer, event);
Steven Rostedtd9313692010-01-06 17:27:11 -05001625 ftrace_trace_stack(buffer, flags, 6, pc);
1626 }
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001627
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001628out:
Steven Rostedt5168ae52010-06-03 09:36:50 -04001629 preempt_enable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001630 unpause_graph_tracing();
1631
1632 return len;
1633}
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001634EXPORT_SYMBOL_GPL(trace_vbprintk);
1635
Steven Rostedt659372d2009-09-03 19:11:07 -04001636int trace_array_printk(struct trace_array *tr,
1637 unsigned long ip, const char *fmt, ...)
1638{
1639 int ret;
1640 va_list ap;
1641
1642 if (!(trace_flags & TRACE_ITER_PRINTK))
1643 return 0;
1644
1645 va_start(ap, fmt);
1646 ret = trace_array_vprintk(tr, ip, fmt, ap);
1647 va_end(ap);
1648 return ret;
1649}
1650
1651int trace_array_vprintk(struct trace_array *tr,
1652 unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001653{
Tom Zanussie1112b42009-03-31 00:48:49 -05001654 struct ftrace_event_call *call = &event_print;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001655 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001656 struct ring_buffer *buffer;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001657 int len = 0, size, pc;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001658 struct print_entry *entry;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001659 unsigned long flags;
1660 char *tbuffer;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001661
1662 if (tracing_disabled || tracing_selftest_running)
1663 return 0;
1664
Steven Rostedt07d777f2011-09-22 14:01:55 -04001665 /* Don't pollute graph traces with trace_vprintk internals */
1666 pause_graph_tracing();
1667
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001668 pc = preempt_count();
1669 preempt_disable_notrace();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001670
Steven Rostedt07d777f2011-09-22 14:01:55 -04001671
1672 tbuffer = get_trace_buf();
1673 if (!tbuffer) {
1674 len = 0;
1675 goto out;
1676 }
1677
1678 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1679 if (len > TRACE_BUF_SIZE)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001680 goto out;
1681
Steven Rostedt07d777f2011-09-22 14:01:55 -04001682 local_save_flags(flags);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001683 size = sizeof(*entry) + len + 1;
Steven Rostedte77405a2009-09-02 14:17:06 -04001684 buffer = tr->buffer;
1685 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
Steven Rostedt07d777f2011-09-22 14:01:55 -04001686 flags, pc);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001687 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04001688 goto out;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001689 entry = ring_buffer_event_data(event);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001690 entry->ip = ip;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001691
Steven Rostedt07d777f2011-09-22 14:01:55 -04001692 memcpy(&entry->buf, tbuffer, len);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001693 entry->buf[len] = '\0';
Steven Rostedtd9313692010-01-06 17:27:11 -05001694 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedte77405a2009-09-02 14:17:06 -04001695 ring_buffer_unlock_commit(buffer, event);
Steven Rostedt07d777f2011-09-22 14:01:55 -04001696 ftrace_trace_stack(buffer, flags, 6, pc);
Steven Rostedtd9313692010-01-06 17:27:11 -05001697 }
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001698 out:
1699 preempt_enable_notrace();
Steven Rostedt07d777f2011-09-22 14:01:55 -04001700 unpause_graph_tracing();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001701
1702 return len;
1703}
Steven Rostedt659372d2009-09-03 19:11:07 -04001704
1705int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
1706{
Steven Rostedta813a152009-10-09 01:41:35 -04001707 return trace_array_vprintk(&global_trace, ip, fmt, args);
Steven Rostedt659372d2009-09-03 19:11:07 -04001708}
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001709EXPORT_SYMBOL_GPL(trace_vprintk);
1710
Robert Richtere2ac8ef2008-11-12 12:59:32 +01001711static void trace_iterator_increment(struct trace_iterator *iter)
Steven Rostedt5a90f572008-09-03 17:42:51 -04001712{
Steven Rostedt6d158a82012-06-27 20:46:14 -04001713 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
1714
Steven Rostedt5a90f572008-09-03 17:42:51 -04001715 iter->idx++;
Steven Rostedt6d158a82012-06-27 20:46:14 -04001716 if (buf_iter)
1717 ring_buffer_read(buf_iter, NULL);
Steven Rostedt5a90f572008-09-03 17:42:51 -04001718}
1719
Ingo Molnare309b412008-05-12 21:20:51 +02001720static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04001721peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
1722 unsigned long *lost_events)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001723{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001724 struct ring_buffer_event *event;
Steven Rostedt6d158a82012-06-27 20:46:14 -04001725 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001726
Steven Rostedtd7690412008-10-01 00:29:53 -04001727 if (buf_iter)
1728 event = ring_buffer_iter_peek(buf_iter, ts);
1729 else
Steven Rostedtbc21b472010-03-31 19:49:26 -04001730 event = ring_buffer_peek(iter->tr->buffer, cpu, ts,
1731 lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04001732
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001733 if (event) {
1734 iter->ent_size = ring_buffer_event_length(event);
1735 return ring_buffer_event_data(event);
1736 }
1737 iter->ent_size = 0;
1738 return NULL;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001739}
Steven Rostedtd7690412008-10-01 00:29:53 -04001740
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001741static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04001742__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
1743 unsigned long *missing_events, u64 *ent_ts)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001744{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001745 struct ring_buffer *buffer = iter->tr->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001746 struct trace_entry *ent, *next = NULL;
Lai Jiangshanaa274972010-04-05 17:11:05 +08001747 unsigned long lost_events = 0, next_lost = 0;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001748 int cpu_file = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001749 u64 next_ts = 0, ts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001750 int next_cpu = -1;
Steven Rostedt12b5da32012-03-27 10:43:28 -04001751 int next_size = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001752 int cpu;
1753
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001754 /*
1755 * If we are in a per_cpu trace file, don't bother by iterating over
1756 * all cpu and peek directly.
1757 */
1758 if (cpu_file > TRACE_PIPE_ALL_CPU) {
1759 if (ring_buffer_empty_cpu(buffer, cpu_file))
1760 return NULL;
Steven Rostedtbc21b472010-03-31 19:49:26 -04001761 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001762 if (ent_cpu)
1763 *ent_cpu = cpu_file;
1764
1765 return ent;
1766 }
1767
Steven Rostedtab464282008-05-12 21:21:00 +02001768 for_each_tracing_cpu(cpu) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001769
1770 if (ring_buffer_empty_cpu(buffer, cpu))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001771 continue;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001772
Steven Rostedtbc21b472010-03-31 19:49:26 -04001773 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001774
Ingo Molnarcdd31cd2008-05-12 21:20:46 +02001775 /*
1776 * Pick the entry with the smallest timestamp:
1777 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001778 if (ent && (!next || ts < next_ts)) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001779 next = ent;
1780 next_cpu = cpu;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001781 next_ts = ts;
Steven Rostedtbc21b472010-03-31 19:49:26 -04001782 next_lost = lost_events;
Steven Rostedt12b5da32012-03-27 10:43:28 -04001783 next_size = iter->ent_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001784 }
1785 }
1786
Steven Rostedt12b5da32012-03-27 10:43:28 -04001787 iter->ent_size = next_size;
1788
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001789 if (ent_cpu)
1790 *ent_cpu = next_cpu;
1791
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001792 if (ent_ts)
1793 *ent_ts = next_ts;
1794
Steven Rostedtbc21b472010-03-31 19:49:26 -04001795 if (missing_events)
1796 *missing_events = next_lost;
1797
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001798 return next;
1799}
1800
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001801/* Find the next real entry, without updating the iterator itself */
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02001802struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
1803 int *ent_cpu, u64 *ent_ts)
Steven Rostedtb3806b42008-05-12 21:20:46 +02001804{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001805 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001806}
Ingo Molnar8c523a92008-05-12 21:20:46 +02001807
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001808/* Find the next real entry, and increment the iterator to the next entry */
Jason Wessel955b61e2010-08-05 09:22:23 -05001809void *trace_find_next_entry_inc(struct trace_iterator *iter)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001810{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001811 iter->ent = __find_next_entry(iter, &iter->cpu,
1812 &iter->lost_events, &iter->ts);
Steven Rostedtb3806b42008-05-12 21:20:46 +02001813
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001814 if (iter->ent)
Robert Richtere2ac8ef2008-11-12 12:59:32 +01001815 trace_iterator_increment(iter);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04001816
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001817 return iter->ent ? iter : NULL;
Steven Rostedtb3806b42008-05-12 21:20:46 +02001818}
1819
Ingo Molnare309b412008-05-12 21:20:51 +02001820static void trace_consume(struct trace_iterator *iter)
Steven Rostedtb3806b42008-05-12 21:20:46 +02001821{
Steven Rostedtbc21b472010-03-31 19:49:26 -04001822 ring_buffer_consume(iter->tr->buffer, iter->cpu, &iter->ts,
1823 &iter->lost_events);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001824}
1825
Ingo Molnare309b412008-05-12 21:20:51 +02001826static void *s_next(struct seq_file *m, void *v, loff_t *pos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001827{
1828 struct trace_iterator *iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001829 int i = (int)*pos;
Ingo Molnar4e3c3332008-05-12 21:20:45 +02001830 void *ent;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001831
Steven Rostedta63ce5b2009-12-07 09:11:39 -05001832 WARN_ON_ONCE(iter->leftover);
1833
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001834 (*pos)++;
1835
1836 /* can't go backwards */
1837 if (iter->idx > i)
1838 return NULL;
1839
1840 if (iter->idx < 0)
Jason Wessel955b61e2010-08-05 09:22:23 -05001841 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001842 else
1843 ent = iter;
1844
1845 while (ent && iter->idx < i)
Jason Wessel955b61e2010-08-05 09:22:23 -05001846 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001847
1848 iter->pos = *pos;
1849
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001850 return ent;
1851}
1852
Jason Wessel955b61e2010-08-05 09:22:23 -05001853void tracing_iter_reset(struct trace_iterator *iter, int cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001854{
1855 struct trace_array *tr = iter->tr;
1856 struct ring_buffer_event *event;
1857 struct ring_buffer_iter *buf_iter;
1858 unsigned long entries = 0;
1859 u64 ts;
1860
1861 tr->data[cpu]->skipped_entries = 0;
1862
Steven Rostedt6d158a82012-06-27 20:46:14 -04001863 buf_iter = trace_buffer_iter(iter, cpu);
1864 if (!buf_iter)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001865 return;
1866
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001867 ring_buffer_iter_reset(buf_iter);
1868
1869 /*
1870 * We could have the case with the max latency tracers
1871 * that a reset never took place on a cpu. This is evident
1872 * by the timestamp being before the start of the buffer.
1873 */
1874 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
1875 if (ts >= iter->tr->time_start)
1876 break;
1877 entries++;
1878 ring_buffer_read(buf_iter, NULL);
1879 }
1880
1881 tr->data[cpu]->skipped_entries = entries;
1882}
1883
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001884/*
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001885 * The current tracer is copied to avoid a global locking
1886 * all around.
1887 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001888static void *s_start(struct seq_file *m, loff_t *pos)
1889{
1890 struct trace_iterator *iter = m->private;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001891 static struct tracer *old_tracer;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001892 int cpu_file = iter->cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001893 void *p = NULL;
1894 loff_t l = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001895 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001896
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001897 /* copy the tracer to avoid using a global lock all around */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001898 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001899 if (unlikely(old_tracer != current_trace && current_trace)) {
1900 old_tracer = current_trace;
1901 *iter->trace = *current_trace;
Steven Rostedtd15f57f2008-05-12 21:20:56 +02001902 }
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01001903 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001904
1905 atomic_inc(&trace_record_cmdline_disabled);
1906
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001907 if (*pos != iter->pos) {
1908 iter->ent = NULL;
1909 iter->cpu = 0;
1910 iter->idx = -1;
1911
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001912 if (cpu_file == TRACE_PIPE_ALL_CPU) {
1913 for_each_tracing_cpu(cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001914 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01001915 } else
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04001916 tracing_iter_reset(iter, cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001917
Lai Jiangshanac91d852010-03-02 17:54:50 +08001918 iter->leftover = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001919 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
1920 ;
1921
1922 } else {
Steven Rostedta63ce5b2009-12-07 09:11:39 -05001923 /*
1924 * If we overflowed the seq_file before, then we want
1925 * to just reuse the trace_seq buffer again.
1926 */
1927 if (iter->leftover)
1928 p = iter;
1929 else {
1930 l = *pos - 1;
1931 p = s_next(m, p, &l);
1932 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001933 }
1934
Lai Jiangshan4f535962009-05-18 19:35:34 +08001935 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001936 trace_access_lock(cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001937 return p;
1938}
1939
1940static void s_stop(struct seq_file *m, void *p)
1941{
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001942 struct trace_iterator *iter = m->private;
1943
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001944 atomic_dec(&trace_record_cmdline_disabled);
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08001945 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08001946 trace_event_read_unlock();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001947}
1948
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05001949static void
1950get_total_entries(struct trace_array *tr, unsigned long *total, unsigned long *entries)
1951{
1952 unsigned long count;
1953 int cpu;
1954
1955 *total = 0;
1956 *entries = 0;
1957
1958 for_each_tracing_cpu(cpu) {
1959 count = ring_buffer_entries_cpu(tr->buffer, cpu);
1960 /*
1961 * If this buffer has skipped entries, then we hold all
1962 * entries for the trace and we need to ignore the
1963 * ones before the time stamp.
1964 */
1965 if (tr->data[cpu]->skipped_entries) {
1966 count -= tr->data[cpu]->skipped_entries;
1967 /* total is the same as the entries */
1968 *total += count;
1969 } else
1970 *total += count +
1971 ring_buffer_overrun_cpu(tr->buffer, cpu);
1972 *entries += count;
1973 }
1974}
1975
Ingo Molnare309b412008-05-12 21:20:51 +02001976static void print_lat_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001977{
Michael Ellermana6168352008-08-20 16:36:11 -07001978 seq_puts(m, "# _------=> CPU# \n");
1979 seq_puts(m, "# / _-----=> irqs-off \n");
1980 seq_puts(m, "# | / _----=> need-resched \n");
1981 seq_puts(m, "# || / _---=> hardirq/softirq \n");
1982 seq_puts(m, "# ||| / _--=> preempt-depth \n");
Steven Rostedte6e1e252011-03-09 10:41:56 -05001983 seq_puts(m, "# |||| / delay \n");
1984 seq_puts(m, "# cmd pid ||||| time | caller \n");
1985 seq_puts(m, "# \\ / ||||| \\ | / \n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001986}
1987
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05001988static void print_event_info(struct trace_array *tr, struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001989{
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05001990 unsigned long total;
1991 unsigned long entries;
1992
1993 get_total_entries(tr, &total, &entries);
1994 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
1995 entries, total, num_online_cpus());
1996 seq_puts(m, "#\n");
1997}
1998
1999static void print_func_help_header(struct trace_array *tr, struct seq_file *m)
2000{
2001 print_event_info(tr, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002002 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
Michael Ellermana6168352008-08-20 16:36:11 -07002003 seq_puts(m, "# | | | | |\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002004}
2005
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002006static void print_func_help_header_irq(struct trace_array *tr, struct seq_file *m)
Steven Rostedt77271ce2011-11-17 09:34:33 -05002007{
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002008 print_event_info(tr, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002009 seq_puts(m, "# _-----=> irqs-off\n");
2010 seq_puts(m, "# / _----=> need-resched\n");
2011 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2012 seq_puts(m, "# || / _--=> preempt-depth\n");
2013 seq_puts(m, "# ||| / delay\n");
2014 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2015 seq_puts(m, "# | | | |||| | |\n");
2016}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002017
Jiri Olsa62b915f2010-04-02 19:01:22 +02002018void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002019print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2020{
2021 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2022 struct trace_array *tr = iter->tr;
2023 struct trace_array_cpu *data = tr->data[tr->cpu];
2024 struct tracer *type = current_trace;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002025 unsigned long entries;
2026 unsigned long total;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002027 const char *name = "preemption";
2028
2029 if (type)
2030 name = type->name;
2031
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002032 get_total_entries(tr, &total, &entries);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002033
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002034 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002035 name, UTS_RELEASE);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002036 seq_puts(m, "# -----------------------------------"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002037 "---------------------------------\n");
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002038 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002039 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
Steven Rostedt57f50be2008-05-12 21:20:44 +02002040 nsecs_to_usecs(data->saved_latency),
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002041 entries,
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02002042 total,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002043 tr->cpu,
2044#if defined(CONFIG_PREEMPT_NONE)
2045 "server",
2046#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2047 "desktop",
Steven Rostedtb5c21b42008-07-10 20:58:12 -04002048#elif defined(CONFIG_PREEMPT)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002049 "preempt",
2050#else
2051 "unknown",
2052#endif
2053 /* These are reserved for later use */
2054 0, 0, 0, 0);
2055#ifdef CONFIG_SMP
2056 seq_printf(m, " #P:%d)\n", num_online_cpus());
2057#else
2058 seq_puts(m, ")\n");
2059#endif
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002060 seq_puts(m, "# -----------------\n");
2061 seq_printf(m, "# | task: %.16s-%d "
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002062 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2063 data->comm, data->pid, data->uid, data->nice,
2064 data->policy, data->rt_priority);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002065 seq_puts(m, "# -----------------\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002066
2067 if (data->critical_start) {
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002068 seq_puts(m, "# => started at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002069 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2070 trace_print_seq(m, &iter->seq);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002071 seq_puts(m, "\n# => ended at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002072 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2073 trace_print_seq(m, &iter->seq);
Steven Rostedt8248ac02009-09-02 12:27:41 -04002074 seq_puts(m, "\n#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002075 }
2076
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002077 seq_puts(m, "#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002078}
2079
Steven Rostedta3097202008-11-07 22:36:02 -05002080static void test_cpu_buff_start(struct trace_iterator *iter)
2081{
2082 struct trace_seq *s = &iter->seq;
2083
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002084 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2085 return;
2086
2087 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2088 return;
2089
Rusty Russell44623442009-01-01 10:12:23 +10302090 if (cpumask_test_cpu(iter->cpu, iter->started))
Steven Rostedta3097202008-11-07 22:36:02 -05002091 return;
2092
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002093 if (iter->tr->data[iter->cpu]->skipped_entries)
2094 return;
2095
Rusty Russell44623442009-01-01 10:12:23 +10302096 cpumask_set_cpu(iter->cpu, iter->started);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002097
2098 /* Don't print started cpu buffer for the first entry of the trace */
2099 if (iter->idx > 1)
2100 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2101 iter->cpu);
Steven Rostedta3097202008-11-07 22:36:02 -05002102}
2103
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002104static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002105{
Steven Rostedt214023c2008-05-12 21:20:46 +02002106 struct trace_seq *s = &iter->seq;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002107 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002108 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002109 struct trace_event *event;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002110
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002111 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002112
Steven Rostedta3097202008-11-07 22:36:02 -05002113 test_cpu_buff_start(iter);
2114
Steven Rostedtf633cef2008-12-23 23:24:13 -05002115 event = ftrace_find_event(entry->type);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002116
2117 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt27d48be2009-03-04 21:57:29 -05002118 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2119 if (!trace_print_lat_context(iter))
2120 goto partial;
2121 } else {
2122 if (!trace_print_context(iter))
2123 goto partial;
2124 }
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002125 }
2126
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002127 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002128 return event->funcs->trace(iter, sym_flags, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002129
2130 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2131 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04002132
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002133 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002134partial:
2135 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002136}
2137
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002138static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002139{
2140 struct trace_seq *s = &iter->seq;
2141 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002142 struct trace_event *event;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002143
2144 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002145
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002146 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002147 if (!trace_seq_printf(s, "%d %d %llu ",
2148 entry->pid, iter->cpu, iter->ts))
2149 goto partial;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002150 }
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002151
Steven Rostedtf633cef2008-12-23 23:24:13 -05002152 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002153 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002154 return event->funcs->raw(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002155
2156 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2157 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04002158
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002159 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002160partial:
2161 return TRACE_TYPE_PARTIAL_LINE;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002162}
2163
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002164static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002165{
2166 struct trace_seq *s = &iter->seq;
2167 unsigned char newline = '\n';
2168 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002169 struct trace_event *event;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002170
2171 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002172
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002173 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2174 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2175 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2176 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2177 }
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002178
Steven Rostedtf633cef2008-12-23 23:24:13 -05002179 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002180 if (event) {
Steven Rostedta9a57762010-04-22 18:46:14 -04002181 enum print_line_t ret = event->funcs->hex(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002182 if (ret != TRACE_TYPE_HANDLED)
2183 return ret;
2184 }
Steven Rostedt7104f302008-10-01 10:52:51 -04002185
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002186 SEQ_PUT_FIELD_RET(s, newline);
2187
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002188 return TRACE_TYPE_HANDLED;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002189}
2190
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002191static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002192{
2193 struct trace_seq *s = &iter->seq;
2194 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002195 struct trace_event *event;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002196
2197 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002198
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002199 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2200 SEQ_PUT_FIELD_RET(s, entry->pid);
Steven Rostedt1830b52d2009-02-07 19:38:43 -05002201 SEQ_PUT_FIELD_RET(s, iter->cpu);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002202 SEQ_PUT_FIELD_RET(s, iter->ts);
2203 }
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002204
Steven Rostedtf633cef2008-12-23 23:24:13 -05002205 event = ftrace_find_event(entry->type);
Steven Rostedta9a57762010-04-22 18:46:14 -04002206 return event ? event->funcs->binary(iter, 0, event) :
2207 TRACE_TYPE_HANDLED;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002208}
2209
Jiri Olsa62b915f2010-04-02 19:01:22 +02002210int trace_empty(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002211{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002212 struct ring_buffer_iter *buf_iter;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002213 int cpu;
2214
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002215 /* If we are looking at one CPU buffer, only check that one */
2216 if (iter->cpu_file != TRACE_PIPE_ALL_CPU) {
2217 cpu = iter->cpu_file;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002218 buf_iter = trace_buffer_iter(iter, cpu);
2219 if (buf_iter) {
2220 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002221 return 0;
2222 } else {
2223 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2224 return 0;
2225 }
2226 return 1;
2227 }
2228
Steven Rostedtab464282008-05-12 21:21:00 +02002229 for_each_tracing_cpu(cpu) {
Steven Rostedt6d158a82012-06-27 20:46:14 -04002230 buf_iter = trace_buffer_iter(iter, cpu);
2231 if (buf_iter) {
2232 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedtd7690412008-10-01 00:29:53 -04002233 return 0;
2234 } else {
2235 if (!ring_buffer_empty_cpu(iter->tr->buffer, cpu))
2236 return 0;
2237 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002238 }
Steven Rostedtd7690412008-10-01 00:29:53 -04002239
Frederic Weisbecker797d3712008-09-30 18:13:45 +02002240 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002241}
2242
Lai Jiangshan4f535962009-05-18 19:35:34 +08002243/* Called with trace_event_read_lock() held. */
Jason Wessel955b61e2010-08-05 09:22:23 -05002244enum print_line_t print_trace_line(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002245{
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002246 enum print_line_t ret;
2247
Jiri Olsaee5e51f2011-03-25 12:05:18 +01002248 if (iter->lost_events &&
2249 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2250 iter->cpu, iter->lost_events))
2251 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002252
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002253 if (iter->trace && iter->trace->print_line) {
2254 ret = iter->trace->print_line(iter);
2255 if (ret != TRACE_TYPE_UNHANDLED)
2256 return ret;
2257 }
Thomas Gleixner72829bc2008-05-23 21:37:28 +02002258
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002259 if (iter->ent->type == TRACE_BPRINT &&
2260 trace_flags & TRACE_ITER_PRINTK &&
2261 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002262 return trace_print_bprintk_msg_only(iter);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002263
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002264 if (iter->ent->type == TRACE_PRINT &&
2265 trace_flags & TRACE_ITER_PRINTK &&
2266 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002267 return trace_print_printk_msg_only(iter);
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002268
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002269 if (trace_flags & TRACE_ITER_BIN)
2270 return print_bin_fmt(iter);
2271
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002272 if (trace_flags & TRACE_ITER_HEX)
2273 return print_hex_fmt(iter);
2274
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002275 if (trace_flags & TRACE_ITER_RAW)
2276 return print_raw_fmt(iter);
2277
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002278 return print_trace_fmt(iter);
2279}
2280
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002281void trace_latency_header(struct seq_file *m)
2282{
2283 struct trace_iterator *iter = m->private;
2284
2285 /* print nothing if the buffers are empty */
2286 if (trace_empty(iter))
2287 return;
2288
2289 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2290 print_trace_header(m, iter);
2291
2292 if (!(trace_flags & TRACE_ITER_VERBOSE))
2293 print_lat_help_header(m);
2294}
2295
Jiri Olsa62b915f2010-04-02 19:01:22 +02002296void trace_default_header(struct seq_file *m)
2297{
2298 struct trace_iterator *iter = m->private;
2299
Jiri Olsaf56e7f82011-06-03 16:58:49 +02002300 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2301 return;
2302
Jiri Olsa62b915f2010-04-02 19:01:22 +02002303 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2304 /* print nothing if the buffers are empty */
2305 if (trace_empty(iter))
2306 return;
2307 print_trace_header(m, iter);
2308 if (!(trace_flags & TRACE_ITER_VERBOSE))
2309 print_lat_help_header(m);
2310 } else {
Steven Rostedt77271ce2011-11-17 09:34:33 -05002311 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2312 if (trace_flags & TRACE_ITER_IRQ_INFO)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002313 print_func_help_header_irq(iter->tr, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002314 else
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002315 print_func_help_header(iter->tr, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002316 }
Jiri Olsa62b915f2010-04-02 19:01:22 +02002317 }
2318}
2319
Steven Rostedte0a413f2011-09-29 21:26:16 -04002320static void test_ftrace_alive(struct seq_file *m)
2321{
2322 if (!ftrace_is_dead())
2323 return;
2324 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2325 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2326}
2327
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002328static int s_show(struct seq_file *m, void *v)
2329{
2330 struct trace_iterator *iter = v;
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002331 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002332
2333 if (iter->ent == NULL) {
2334 if (iter->tr) {
2335 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2336 seq_puts(m, "#\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002337 test_ftrace_alive(m);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002338 }
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002339 if (iter->trace && iter->trace->print_header)
2340 iter->trace->print_header(m);
Jiri Olsa62b915f2010-04-02 19:01:22 +02002341 else
2342 trace_default_header(m);
2343
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002344 } else if (iter->leftover) {
2345 /*
2346 * If we filled the seq_file buffer earlier, we
2347 * want to just show it now.
2348 */
2349 ret = trace_print_seq(m, &iter->seq);
2350
2351 /* ret should this time be zero, but you never know */
2352 iter->leftover = ret;
2353
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002354 } else {
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002355 print_trace_line(iter);
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002356 ret = trace_print_seq(m, &iter->seq);
2357 /*
2358 * If we overflow the seq_file buffer, then it will
2359 * ask us for this data again at start up.
2360 * Use that instead.
2361 * ret is 0 if seq_file write succeeded.
2362 * -1 otherwise.
2363 */
2364 iter->leftover = ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002365 }
2366
2367 return 0;
2368}
2369
James Morris88e9d342009-09-22 16:43:43 -07002370static const struct seq_operations tracer_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002371 .start = s_start,
2372 .next = s_next,
2373 .stop = s_stop,
2374 .show = s_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002375};
2376
Ingo Molnare309b412008-05-12 21:20:51 +02002377static struct trace_iterator *
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002378__tracing_open(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002379{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002380 long cpu_file = (long) inode->i_private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002381 struct trace_iterator *iter;
Jiri Olsa50e18b92012-04-25 10:23:39 +02002382 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002383
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002384 if (tracing_disabled)
2385 return ERR_PTR(-ENODEV);
Steven Rostedt60a11772008-05-12 21:20:44 +02002386
Jiri Olsa50e18b92012-04-25 10:23:39 +02002387 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002388 if (!iter)
2389 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002390
Steven Rostedt6d158a82012-06-27 20:46:14 -04002391 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2392 GFP_KERNEL);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002393 /*
2394 * We make a copy of the current tracer to avoid concurrent
2395 * changes on it while we are reading.
2396 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002397 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002398 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002399 if (!iter->trace)
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002400 goto fail;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002401
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002402 if (current_trace)
2403 *iter->trace = *current_trace;
2404
Li Zefan79f55992009-06-15 14:58:26 +08002405 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002406 goto fail;
2407
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002408 if (current_trace && current_trace->print_max)
2409 iter->tr = &max_tr;
2410 else
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002411 iter->tr = &global_trace;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002412 iter->pos = -1;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002413 mutex_init(&iter->mutex);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002414 iter->cpu_file = cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002415
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002416 /* Notify the tracer early; before we stop tracing. */
2417 if (iter->trace && iter->trace->open)
Markus Metzgera93751c2008-12-11 13:53:26 +01002418 iter->trace->open(iter);
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002419
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002420 /* Annotate start of buffers if we had overruns */
2421 if (ring_buffer_overruns(iter->tr->buffer))
2422 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2423
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002424 /* stop the trace while dumping */
2425 tracing_stop();
2426
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002427 if (iter->cpu_file == TRACE_PIPE_ALL_CPU) {
2428 for_each_tracing_cpu(cpu) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002429 iter->buffer_iter[cpu] =
David Miller72c9ddf2010-04-20 15:47:11 -07002430 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2431 }
2432 ring_buffer_read_prepare_sync();
2433 for_each_tracing_cpu(cpu) {
2434 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002435 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002436 }
2437 } else {
2438 cpu = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002439 iter->buffer_iter[cpu] =
David Miller72c9ddf2010-04-20 15:47:11 -07002440 ring_buffer_read_prepare(iter->tr->buffer, cpu);
2441 ring_buffer_read_prepare_sync();
2442 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002443 tracing_iter_reset(iter, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002444 }
2445
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002446 mutex_unlock(&trace_types_lock);
2447
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002448 return iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002449
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002450 fail:
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002451 mutex_unlock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002452 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04002453 kfree(iter->buffer_iter);
Jiri Olsa50e18b92012-04-25 10:23:39 +02002454 seq_release_private(inode, file);
2455 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002456}
2457
2458int tracing_open_generic(struct inode *inode, struct file *filp)
2459{
Steven Rostedt60a11772008-05-12 21:20:44 +02002460 if (tracing_disabled)
2461 return -ENODEV;
2462
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002463 filp->private_data = inode->i_private;
2464 return 0;
2465}
2466
Hannes Eder4fd27352009-02-10 19:44:12 +01002467static int tracing_release(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002468{
matt mooney907f2782010-09-27 19:04:53 -07002469 struct seq_file *m = file->private_data;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002470 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002471 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002472
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002473 if (!(file->f_mode & FMODE_READ))
2474 return 0;
2475
2476 iter = m->private;
2477
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002478 mutex_lock(&trace_types_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002479 for_each_tracing_cpu(cpu) {
2480 if (iter->buffer_iter[cpu])
2481 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2482 }
2483
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002484 if (iter->trace && iter->trace->close)
2485 iter->trace->close(iter);
2486
2487 /* reenable tracing if it was previously enabled */
Steven Rostedt90369902008-11-05 16:05:44 -05002488 tracing_start();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002489 mutex_unlock(&trace_types_lock);
2490
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002491 mutex_destroy(&iter->mutex);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002492 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002493 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04002494 kfree(iter->buffer_iter);
Jiri Olsa50e18b92012-04-25 10:23:39 +02002495 seq_release_private(inode, file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002496 return 0;
2497}
2498
2499static int tracing_open(struct inode *inode, struct file *file)
2500{
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002501 struct trace_iterator *iter;
2502 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002503
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002504 /* If this file was open for write, then erase contents */
2505 if ((file->f_mode & FMODE_WRITE) &&
Steven Rostedt8650ae32009-07-22 23:29:30 -04002506 (file->f_flags & O_TRUNC)) {
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002507 long cpu = (long) inode->i_private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002508
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002509 if (cpu == TRACE_PIPE_ALL_CPU)
2510 tracing_reset_online_cpus(&global_trace);
2511 else
2512 tracing_reset(&global_trace, cpu);
2513 }
2514
2515 if (file->f_mode & FMODE_READ) {
2516 iter = __tracing_open(inode, file);
2517 if (IS_ERR(iter))
2518 ret = PTR_ERR(iter);
2519 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2520 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2521 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002522 return ret;
2523}
2524
Ingo Molnare309b412008-05-12 21:20:51 +02002525static void *
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002526t_next(struct seq_file *m, void *v, loff_t *pos)
2527{
Li Zefanf129e962009-06-24 09:53:44 +08002528 struct tracer *t = v;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002529
2530 (*pos)++;
2531
2532 if (t)
2533 t = t->next;
2534
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002535 return t;
2536}
2537
2538static void *t_start(struct seq_file *m, loff_t *pos)
2539{
Li Zefanf129e962009-06-24 09:53:44 +08002540 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002541 loff_t l = 0;
2542
2543 mutex_lock(&trace_types_lock);
Li Zefanf129e962009-06-24 09:53:44 +08002544 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002545 ;
2546
2547 return t;
2548}
2549
2550static void t_stop(struct seq_file *m, void *p)
2551{
2552 mutex_unlock(&trace_types_lock);
2553}
2554
2555static int t_show(struct seq_file *m, void *v)
2556{
2557 struct tracer *t = v;
2558
2559 if (!t)
2560 return 0;
2561
2562 seq_printf(m, "%s", t->name);
2563 if (t->next)
2564 seq_putc(m, ' ');
2565 else
2566 seq_putc(m, '\n');
2567
2568 return 0;
2569}
2570
James Morris88e9d342009-09-22 16:43:43 -07002571static const struct seq_operations show_traces_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002572 .start = t_start,
2573 .next = t_next,
2574 .stop = t_stop,
2575 .show = t_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002576};
2577
2578static int show_traces_open(struct inode *inode, struct file *file)
2579{
Steven Rostedt60a11772008-05-12 21:20:44 +02002580 if (tracing_disabled)
2581 return -ENODEV;
2582
Li Zefanf129e962009-06-24 09:53:44 +08002583 return seq_open(file, &show_traces_seq_ops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002584}
2585
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002586static ssize_t
2587tracing_write_stub(struct file *filp, const char __user *ubuf,
2588 size_t count, loff_t *ppos)
2589{
2590 return count;
2591}
2592
Slava Pestov364829b2010-11-24 15:13:16 -08002593static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
2594{
2595 if (file->f_mode & FMODE_READ)
2596 return seq_lseek(file, offset, origin);
2597 else
2598 return 0;
2599}
2600
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002601static const struct file_operations tracing_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002602 .open = tracing_open,
2603 .read = seq_read,
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002604 .write = tracing_write_stub,
Slava Pestov364829b2010-11-24 15:13:16 -08002605 .llseek = tracing_seek,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002606 .release = tracing_release,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002607};
2608
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002609static const struct file_operations show_traces_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002610 .open = show_traces_open,
2611 .read = seq_read,
2612 .release = seq_release,
Arnd Bergmannb4447862010-07-07 23:40:11 +02002613 .llseek = seq_lseek,
Ingo Molnarc7078de2008-05-12 21:20:52 +02002614};
2615
Ingo Molnar36dfe922008-05-12 21:20:52 +02002616/*
2617 * Only trace on a CPU if the bitmask is set:
2618 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302619static cpumask_var_t tracing_cpumask;
Ingo Molnar36dfe922008-05-12 21:20:52 +02002620
2621/*
2622 * The tracer itself will not take this lock, but still we want
2623 * to provide a consistent cpumask to user-space:
2624 */
2625static DEFINE_MUTEX(tracing_cpumask_update_lock);
2626
2627/*
2628 * Temporary storage for the character representation of the
2629 * CPU bitmask (and one more byte for the newline):
2630 */
2631static char mask_str[NR_CPUS + 1];
2632
Ingo Molnarc7078de2008-05-12 21:20:52 +02002633static ssize_t
2634tracing_cpumask_read(struct file *filp, char __user *ubuf,
2635 size_t count, loff_t *ppos)
2636{
Ingo Molnar36dfe922008-05-12 21:20:52 +02002637 int len;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002638
2639 mutex_lock(&tracing_cpumask_update_lock);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002640
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302641 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002642 if (count - len < 2) {
2643 count = -EINVAL;
2644 goto out_err;
2645 }
2646 len += sprintf(mask_str + len, "\n");
2647 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
2648
2649out_err:
Ingo Molnarc7078de2008-05-12 21:20:52 +02002650 mutex_unlock(&tracing_cpumask_update_lock);
2651
2652 return count;
2653}
2654
2655static ssize_t
2656tracing_cpumask_write(struct file *filp, const char __user *ubuf,
2657 size_t count, loff_t *ppos)
2658{
Ingo Molnar36dfe922008-05-12 21:20:52 +02002659 int err, cpu;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302660 cpumask_var_t tracing_cpumask_new;
2661
2662 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
2663 return -ENOMEM;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002664
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302665 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002666 if (err)
2667 goto err_unlock;
2668
Li Zefan215368e2009-06-15 10:56:42 +08002669 mutex_lock(&tracing_cpumask_update_lock);
2670
Steven Rostedta5e25882008-12-02 15:34:05 -05002671 local_irq_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01002672 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtab464282008-05-12 21:21:00 +02002673 for_each_tracing_cpu(cpu) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002674 /*
2675 * Increase/decrease the disabled counter if we are
2676 * about to flip a bit in the cpumask:
2677 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302678 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
2679 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002680 atomic_inc(&global_trace.data[cpu]->disabled);
Vaibhav Nagarnaik71babb22012-05-03 18:59:52 -07002681 ring_buffer_record_disable_cpu(global_trace.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002682 }
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302683 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
2684 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02002685 atomic_dec(&global_trace.data[cpu]->disabled);
Vaibhav Nagarnaik71babb22012-05-03 18:59:52 -07002686 ring_buffer_record_enable_cpu(global_trace.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002687 }
2688 }
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01002689 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedta5e25882008-12-02 15:34:05 -05002690 local_irq_enable();
Ingo Molnar36dfe922008-05-12 21:20:52 +02002691
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302692 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002693
Ingo Molnarc7078de2008-05-12 21:20:52 +02002694 mutex_unlock(&tracing_cpumask_update_lock);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10302695 free_cpumask_var(tracing_cpumask_new);
Ingo Molnarc7078de2008-05-12 21:20:52 +02002696
Ingo Molnarc7078de2008-05-12 21:20:52 +02002697 return count;
Ingo Molnar36dfe922008-05-12 21:20:52 +02002698
2699err_unlock:
Li Zefan215368e2009-06-15 10:56:42 +08002700 free_cpumask_var(tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02002701
2702 return err;
Ingo Molnarc7078de2008-05-12 21:20:52 +02002703}
2704
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002705static const struct file_operations tracing_cpumask_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002706 .open = tracing_open_generic,
2707 .read = tracing_cpumask_read,
2708 .write = tracing_cpumask_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02002709 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002710};
2711
Li Zefanfdb372e2009-12-08 11:15:59 +08002712static int tracing_trace_options_show(struct seq_file *m, void *v)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002713{
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002714 struct tracer_opt *trace_opts;
2715 u32 tracer_flags;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002716 int i;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002717
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002718 mutex_lock(&trace_types_lock);
2719 tracer_flags = current_trace->flags->val;
2720 trace_opts = current_trace->flags->opts;
2721
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002722 for (i = 0; trace_options[i]; i++) {
2723 if (trace_flags & (1 << i))
Li Zefanfdb372e2009-12-08 11:15:59 +08002724 seq_printf(m, "%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002725 else
Li Zefanfdb372e2009-12-08 11:15:59 +08002726 seq_printf(m, "no%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002727 }
2728
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002729 for (i = 0; trace_opts[i].name; i++) {
2730 if (tracer_flags & trace_opts[i].bit)
Li Zefanfdb372e2009-12-08 11:15:59 +08002731 seq_printf(m, "%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002732 else
Li Zefanfdb372e2009-12-08 11:15:59 +08002733 seq_printf(m, "no%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002734 }
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002735 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002736
Li Zefanfdb372e2009-12-08 11:15:59 +08002737 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002738}
2739
Li Zefan8d18eaa2009-12-08 11:17:06 +08002740static int __set_tracer_option(struct tracer *trace,
2741 struct tracer_flags *tracer_flags,
2742 struct tracer_opt *opts, int neg)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002743{
Li Zefan8d18eaa2009-12-08 11:17:06 +08002744 int ret;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002745
Li Zefan8d18eaa2009-12-08 11:17:06 +08002746 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002747 if (ret)
2748 return ret;
2749
2750 if (neg)
Zhaolei77708412009-08-07 18:53:21 +08002751 tracer_flags->val &= ~opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002752 else
Zhaolei77708412009-08-07 18:53:21 +08002753 tracer_flags->val |= opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002754 return 0;
2755}
2756
Li Zefan8d18eaa2009-12-08 11:17:06 +08002757/* Try to assign a tracer specific option */
2758static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
2759{
2760 struct tracer_flags *tracer_flags = trace->flags;
2761 struct tracer_opt *opts = NULL;
2762 int i;
2763
2764 for (i = 0; tracer_flags->opts[i].name; i++) {
2765 opts = &tracer_flags->opts[i];
2766
2767 if (strcmp(cmp, opts->name) == 0)
2768 return __set_tracer_option(trace, trace->flags,
2769 opts, neg);
2770 }
2771
2772 return -EINVAL;
2773}
2774
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002775static void set_tracer_flags(unsigned int mask, int enabled)
2776{
2777 /* do nothing if flag is already set */
2778 if (!!(trace_flags & mask) == !!enabled)
2779 return;
2780
2781 if (enabled)
2782 trace_flags |= mask;
2783 else
2784 trace_flags &= ~mask;
Li Zefane870e9a2010-07-02 11:07:32 +08002785
2786 if (mask == TRACE_ITER_RECORD_CMD)
2787 trace_event_enable_cmd_record(enabled);
David Sharp750912f2010-12-08 13:46:47 -08002788
2789 if (mask == TRACE_ITER_OVERWRITE)
2790 ring_buffer_change_overwrite(global_trace.buffer, enabled);
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002791}
2792
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002793static ssize_t
Steven Rostedtee6bce52008-11-12 17:52:37 -05002794tracing_trace_options_write(struct file *filp, const char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002795 size_t cnt, loff_t *ppos)
2796{
2797 char buf[64];
Li Zefan8d18eaa2009-12-08 11:17:06 +08002798 char *cmp;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002799 int neg = 0;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002800 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002801 int i;
2802
Steven Rostedtcffae432008-05-12 21:21:00 +02002803 if (cnt >= sizeof(buf))
2804 return -EINVAL;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002805
2806 if (copy_from_user(&buf, ubuf, cnt))
2807 return -EFAULT;
2808
2809 buf[cnt] = 0;
Li Zefan8d18eaa2009-12-08 11:17:06 +08002810 cmp = strstrip(buf);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002811
Li Zefan8d18eaa2009-12-08 11:17:06 +08002812 if (strncmp(cmp, "no", 2) == 0) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002813 neg = 1;
2814 cmp += 2;
2815 }
2816
2817 for (i = 0; trace_options[i]; i++) {
Li Zefan8d18eaa2009-12-08 11:17:06 +08002818 if (strcmp(cmp, trace_options[i]) == 0) {
Steven Rostedtaf4617b2009-03-17 18:09:55 -04002819 set_tracer_flags(1 << i, !neg);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002820 break;
2821 }
2822 }
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002823
2824 /* If no option could be set, test the specific tracer options */
2825 if (!trace_options[i]) {
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002826 mutex_lock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002827 ret = set_tracer_option(current_trace, cmp, neg);
Steven Rostedtd8e83d22009-02-26 23:55:58 -05002828 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01002829 if (ret)
2830 return ret;
2831 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002832
Jiri Olsacf8517c2009-10-23 19:36:16 -04002833 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002834
2835 return cnt;
2836}
2837
Li Zefanfdb372e2009-12-08 11:15:59 +08002838static int tracing_trace_options_open(struct inode *inode, struct file *file)
2839{
2840 if (tracing_disabled)
2841 return -ENODEV;
2842 return single_open(file, tracing_trace_options_show, NULL);
2843}
2844
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002845static const struct file_operations tracing_iter_fops = {
Li Zefanfdb372e2009-12-08 11:15:59 +08002846 .open = tracing_trace_options_open,
2847 .read = seq_read,
2848 .llseek = seq_lseek,
2849 .release = single_release,
Steven Rostedtee6bce52008-11-12 17:52:37 -05002850 .write = tracing_trace_options_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002851};
2852
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002853static const char readme_msg[] =
2854 "tracing mini-HOWTO:\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002855 "# mount -t debugfs nodev /sys/kernel/debug\n\n"
2856 "# cat /sys/kernel/debug/tracing/available_tracers\n"
Geunsik Lim1e42e832012-02-08 19:05:36 +09002857 "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002858 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Nikanth Karthikesanbc2b6872009-03-23 11:58:31 +05302859 "nop\n"
Geunsik Lim1e42e832012-02-08 19:05:36 +09002860 "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002861 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Geunsik Lim1e42e832012-02-08 19:05:36 +09002862 "wakeup\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002863 "# cat /sys/kernel/debug/tracing/trace_options\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002864 "noprint-parent nosym-offset nosym-addr noverbose\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002865 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
Geunsik Lim9b5f8b32011-08-12 14:30:22 +09002866 "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09002867 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
Geunsik Lim9b5f8b32011-08-12 14:30:22 +09002868 "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002869;
2870
2871static ssize_t
2872tracing_readme_read(struct file *filp, char __user *ubuf,
2873 size_t cnt, loff_t *ppos)
2874{
2875 return simple_read_from_buffer(ubuf, cnt, ppos,
2876 readme_msg, strlen(readme_msg));
2877}
2878
Steven Rostedt5e2336a2009-03-05 21:44:55 -05002879static const struct file_operations tracing_readme_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02002880 .open = tracing_open_generic,
2881 .read = tracing_readme_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02002882 .llseek = generic_file_llseek,
Ingo Molnar7bd2f242008-05-12 21:20:45 +02002883};
2884
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002885static ssize_t
Avadh Patel69abe6a2009-04-10 16:04:48 -04002886tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
2887 size_t cnt, loff_t *ppos)
2888{
2889 char *buf_comm;
2890 char *file_buf;
2891 char *buf;
2892 int len = 0;
2893 int pid;
2894 int i;
2895
2896 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
2897 if (!file_buf)
2898 return -ENOMEM;
2899
2900 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
2901 if (!buf_comm) {
2902 kfree(file_buf);
2903 return -ENOMEM;
2904 }
2905
2906 buf = file_buf;
2907
2908 for (i = 0; i < SAVED_CMDLINES; i++) {
2909 int r;
2910
2911 pid = map_cmdline_to_pid[i];
2912 if (pid == -1 || pid == NO_CMDLINE_MAP)
2913 continue;
2914
2915 trace_find_cmdline(pid, buf_comm);
2916 r = sprintf(buf, "%d %s\n", pid, buf_comm);
2917 buf += r;
2918 len += r;
2919 }
2920
2921 len = simple_read_from_buffer(ubuf, cnt, ppos,
2922 file_buf, len);
2923
2924 kfree(file_buf);
2925 kfree(buf_comm);
2926
2927 return len;
2928}
2929
2930static const struct file_operations tracing_saved_cmdlines_fops = {
2931 .open = tracing_open_generic,
2932 .read = tracing_saved_cmdlines_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02002933 .llseek = generic_file_llseek,
Avadh Patel69abe6a2009-04-10 16:04:48 -04002934};
2935
2936static ssize_t
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002937tracing_ctrl_read(struct file *filp, char __user *ubuf,
2938 size_t cnt, loff_t *ppos)
2939{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002940 char buf[64];
2941 int r;
2942
Steven Rostedt90369902008-11-05 16:05:44 -05002943 r = sprintf(buf, "%u\n", tracer_enabled);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002944 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002945}
2946
2947static ssize_t
2948tracing_ctrl_write(struct file *filp, const char __user *ubuf,
2949 size_t cnt, loff_t *ppos)
2950{
2951 struct trace_array *tr = filp->private_data;
Hannes Eder5e398412009-02-10 19:44:34 +01002952 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002953 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002954
Peter Huewe22fe9b52011-06-07 21:58:27 +02002955 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
2956 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02002957 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002958
2959 val = !!val;
2960
2961 mutex_lock(&trace_types_lock);
Steven Rostedt90369902008-11-05 16:05:44 -05002962 if (tracer_enabled ^ val) {
Steven Rostedt6752ab42011-02-08 13:54:06 -05002963
2964 /* Only need to warn if this is used to change the state */
2965 WARN_ONCE(1, "tracing_enabled is deprecated. Use tracing_on");
2966
Steven Rostedt90369902008-11-05 16:05:44 -05002967 if (val) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002968 tracer_enabled = 1;
Steven Rostedt90369902008-11-05 16:05:44 -05002969 if (current_trace->start)
2970 current_trace->start(tr);
2971 tracing_start();
2972 } else {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002973 tracer_enabled = 0;
Steven Rostedt90369902008-11-05 16:05:44 -05002974 tracing_stop();
2975 if (current_trace->stop)
2976 current_trace->stop(tr);
2977 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002978 }
2979 mutex_unlock(&trace_types_lock);
2980
Jiri Olsacf8517c2009-10-23 19:36:16 -04002981 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002982
2983 return cnt;
2984}
2985
2986static ssize_t
2987tracing_set_trace_read(struct file *filp, char __user *ubuf,
2988 size_t cnt, loff_t *ppos)
2989{
Li Zefanee6c2c12009-09-18 14:06:47 +08002990 char buf[MAX_TRACER_SIZE+2];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002991 int r;
2992
2993 mutex_lock(&trace_types_lock);
2994 if (current_trace)
2995 r = sprintf(buf, "%s\n", current_trace->name);
2996 else
2997 r = sprintf(buf, "\n");
2998 mutex_unlock(&trace_types_lock);
2999
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003000 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003001}
3002
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003003int tracer_init(struct tracer *t, struct trace_array *tr)
3004{
3005 tracing_reset_online_cpus(tr);
3006 return t->init(tr);
3007}
3008
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003009static void set_buffer_entries(struct trace_array *tr, unsigned long val)
3010{
3011 int cpu;
3012 for_each_tracing_cpu(cpu)
3013 tr->data[cpu]->entries = val;
3014}
3015
3016static int __tracing_resize_ring_buffer(unsigned long size, int cpu)
Steven Rostedt73c51622009-03-11 13:42:01 -04003017{
3018 int ret;
3019
3020 /*
3021 * If kernel or user changes the size of the ring buffer
Steven Rostedta123c522009-03-12 11:21:08 -04003022 * we use the size that was given, and we can forget about
3023 * expanding it later.
Steven Rostedt73c51622009-03-11 13:42:01 -04003024 */
3025 ring_buffer_expanded = 1;
3026
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003027 ret = ring_buffer_resize(global_trace.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003028 if (ret < 0)
3029 return ret;
3030
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003031 if (!current_trace->use_max_tr)
3032 goto out;
3033
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003034 ret = ring_buffer_resize(max_tr.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003035 if (ret < 0) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003036 int r = 0;
Steven Rostedt73c51622009-03-11 13:42:01 -04003037
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003038 if (cpu == RING_BUFFER_ALL_CPUS) {
3039 int i;
3040 for_each_tracing_cpu(i) {
3041 r = ring_buffer_resize(global_trace.buffer,
3042 global_trace.data[i]->entries,
3043 i);
3044 if (r < 0)
3045 break;
3046 }
3047 } else {
3048 r = ring_buffer_resize(global_trace.buffer,
3049 global_trace.data[cpu]->entries,
3050 cpu);
3051 }
3052
Steven Rostedt73c51622009-03-11 13:42:01 -04003053 if (r < 0) {
Steven Rostedta123c522009-03-12 11:21:08 -04003054 /*
3055 * AARGH! We are left with different
3056 * size max buffer!!!!
3057 * The max buffer is our "snapshot" buffer.
3058 * When a tracer needs a snapshot (one of the
3059 * latency tracers), it swaps the max buffer
3060 * with the saved snap shot. We succeeded to
3061 * update the size of the main buffer, but failed to
3062 * update the size of the max buffer. But when we tried
3063 * to reset the main buffer to the original size, we
3064 * failed there too. This is very unlikely to
3065 * happen, but if it does, warn and kill all
3066 * tracing.
3067 */
Steven Rostedt73c51622009-03-11 13:42:01 -04003068 WARN_ON(1);
3069 tracing_disabled = 1;
3070 }
3071 return ret;
3072 }
3073
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003074 if (cpu == RING_BUFFER_ALL_CPUS)
3075 set_buffer_entries(&max_tr, size);
3076 else
3077 max_tr.data[cpu]->entries = size;
3078
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003079 out:
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003080 if (cpu == RING_BUFFER_ALL_CPUS)
3081 set_buffer_entries(&global_trace, size);
3082 else
3083 global_trace.data[cpu]->entries = size;
Steven Rostedt73c51622009-03-11 13:42:01 -04003084
3085 return ret;
3086}
3087
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003088static ssize_t tracing_resize_ring_buffer(unsigned long size, int cpu_id)
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003089{
Vaibhav Nagarnaik83f40312012-05-03 18:59:50 -07003090 int ret = size;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003091
3092 mutex_lock(&trace_types_lock);
3093
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003094 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3095 /* make sure, this cpu is enabled in the mask */
3096 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3097 ret = -EINVAL;
3098 goto out;
3099 }
3100 }
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003101
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003102 ret = __tracing_resize_ring_buffer(size, cpu_id);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003103 if (ret < 0)
3104 ret = -ENOMEM;
3105
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003106out:
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003107 mutex_unlock(&trace_types_lock);
3108
3109 return ret;
3110}
3111
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003112
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003113/**
3114 * tracing_update_buffers - used by tracing facility to expand ring buffers
3115 *
3116 * To save on memory when the tracing is never used on a system with it
3117 * configured in. The ring buffers are set to a minimum size. But once
3118 * a user starts to use the tracing facility, then they need to grow
3119 * to their default size.
3120 *
3121 * This function is to be called when a tracer is about to be used.
3122 */
3123int tracing_update_buffers(void)
3124{
3125 int ret = 0;
3126
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003127 mutex_lock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003128 if (!ring_buffer_expanded)
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003129 ret = __tracing_resize_ring_buffer(trace_buf_size,
3130 RING_BUFFER_ALL_CPUS);
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003131 mutex_unlock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003132
3133 return ret;
3134}
3135
Steven Rostedt577b7852009-02-26 23:43:05 -05003136struct trace_option_dentry;
3137
3138static struct trace_option_dentry *
3139create_trace_option_files(struct tracer *tracer);
3140
3141static void
3142destroy_trace_option_files(struct trace_option_dentry *topts);
3143
Steven Rostedtb2821ae2009-02-02 21:38:32 -05003144static int tracing_set_tracer(const char *buf)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003145{
Steven Rostedt577b7852009-02-26 23:43:05 -05003146 static struct trace_option_dentry *topts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003147 struct trace_array *tr = &global_trace;
3148 struct tracer *t;
Peter Zijlstrad9e54072008-11-01 19:57:37 +01003149 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003150
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003151 mutex_lock(&trace_types_lock);
3152
Steven Rostedt73c51622009-03-11 13:42:01 -04003153 if (!ring_buffer_expanded) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003154 ret = __tracing_resize_ring_buffer(trace_buf_size,
3155 RING_BUFFER_ALL_CPUS);
Steven Rostedt73c51622009-03-11 13:42:01 -04003156 if (ret < 0)
Frederic Weisbecker59f586d2009-03-15 22:10:39 +01003157 goto out;
Steven Rostedt73c51622009-03-11 13:42:01 -04003158 ret = 0;
3159 }
3160
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003161 for (t = trace_types; t; t = t->next) {
3162 if (strcmp(t->name, buf) == 0)
3163 break;
3164 }
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02003165 if (!t) {
3166 ret = -EINVAL;
3167 goto out;
3168 }
3169 if (t == current_trace)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003170 goto out;
3171
Steven Rostedt9f029e82008-11-12 15:24:24 -05003172 trace_branch_disable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003173 if (current_trace && current_trace->reset)
3174 current_trace->reset(tr);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003175 if (current_trace && current_trace->use_max_tr) {
3176 /*
3177 * We don't free the ring buffer. instead, resize it because
3178 * The max_tr ring buffer has some state (e.g. ring->clock) and
3179 * we want preserve it.
3180 */
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003181 ring_buffer_resize(max_tr.buffer, 1, RING_BUFFER_ALL_CPUS);
3182 set_buffer_entries(&max_tr, 1);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003183 }
Steven Rostedt577b7852009-02-26 23:43:05 -05003184 destroy_trace_option_files(topts);
3185
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003186 current_trace = t;
Steven Rostedt577b7852009-02-26 23:43:05 -05003187
3188 topts = create_trace_option_files(current_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003189 if (current_trace->use_max_tr) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003190 int cpu;
3191 /* we need to make per cpu buffer sizes equivalent */
3192 for_each_tracing_cpu(cpu) {
3193 ret = ring_buffer_resize(max_tr.buffer,
3194 global_trace.data[cpu]->entries,
3195 cpu);
3196 if (ret < 0)
3197 goto out;
3198 max_tr.data[cpu]->entries =
3199 global_trace.data[cpu]->entries;
3200 }
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003201 }
Steven Rostedt577b7852009-02-26 23:43:05 -05003202
Frederic Weisbecker1c800252008-11-16 05:57:26 +01003203 if (t->init) {
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003204 ret = tracer_init(t, tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +01003205 if (ret)
3206 goto out;
3207 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003208
Steven Rostedt9f029e82008-11-12 15:24:24 -05003209 trace_branch_enable(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003210 out:
3211 mutex_unlock(&trace_types_lock);
3212
Peter Zijlstrad9e54072008-11-01 19:57:37 +01003213 return ret;
3214}
3215
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003216static ssize_t
3217tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3218 size_t cnt, loff_t *ppos)
3219{
Li Zefanee6c2c12009-09-18 14:06:47 +08003220 char buf[MAX_TRACER_SIZE+1];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003221 int i;
3222 size_t ret;
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01003223 int err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003224
Steven Rostedt60063a62008-10-28 10:44:24 -04003225 ret = cnt;
3226
Li Zefanee6c2c12009-09-18 14:06:47 +08003227 if (cnt > MAX_TRACER_SIZE)
3228 cnt = MAX_TRACER_SIZE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003229
3230 if (copy_from_user(&buf, ubuf, cnt))
3231 return -EFAULT;
3232
3233 buf[cnt] = 0;
3234
3235 /* strip ending whitespace. */
3236 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3237 buf[i] = 0;
3238
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01003239 err = tracing_set_tracer(buf);
3240 if (err)
3241 return err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003242
Jiri Olsacf8517c2009-10-23 19:36:16 -04003243 *ppos += ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003244
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02003245 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003246}
3247
3248static ssize_t
3249tracing_max_lat_read(struct file *filp, char __user *ubuf,
3250 size_t cnt, loff_t *ppos)
3251{
3252 unsigned long *ptr = filp->private_data;
3253 char buf[64];
3254 int r;
3255
Steven Rostedtcffae432008-05-12 21:21:00 +02003256 r = snprintf(buf, sizeof(buf), "%ld\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003257 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
Steven Rostedtcffae432008-05-12 21:21:00 +02003258 if (r > sizeof(buf))
3259 r = sizeof(buf);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003260 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003261}
3262
3263static ssize_t
3264tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3265 size_t cnt, loff_t *ppos)
3266{
Hannes Eder5e398412009-02-10 19:44:34 +01003267 unsigned long *ptr = filp->private_data;
Hannes Eder5e398412009-02-10 19:44:34 +01003268 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003269 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003270
Peter Huewe22fe9b52011-06-07 21:58:27 +02003271 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3272 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003273 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003274
3275 *ptr = val * 1000;
3276
3277 return cnt;
3278}
3279
Steven Rostedtb3806b42008-05-12 21:20:46 +02003280static int tracing_open_pipe(struct inode *inode, struct file *filp)
3281{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003282 long cpu_file = (long) inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003283 struct trace_iterator *iter;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003284 int ret = 0;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003285
3286 if (tracing_disabled)
3287 return -ENODEV;
3288
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003289 mutex_lock(&trace_types_lock);
3290
Steven Rostedtb3806b42008-05-12 21:20:46 +02003291 /* create a buffer to store the information to pass to userspace */
3292 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003293 if (!iter) {
3294 ret = -ENOMEM;
3295 goto out;
3296 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003297
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003298 /*
3299 * We make a copy of the current tracer to avoid concurrent
3300 * changes on it while we are reading.
3301 */
3302 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3303 if (!iter->trace) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003304 ret = -ENOMEM;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003305 goto fail;
3306 }
3307 if (current_trace)
3308 *iter->trace = *current_trace;
3309
3310 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3311 ret = -ENOMEM;
3312 goto fail;
Rusty Russell44623442009-01-01 10:12:23 +10303313 }
3314
Steven Rostedta3097202008-11-07 22:36:02 -05003315 /* trace pipe does not show start of buffer */
Rusty Russell44623442009-01-01 10:12:23 +10303316 cpumask_setall(iter->started);
Steven Rostedta3097202008-11-07 22:36:02 -05003317
Steven Rostedt112f38a72009-06-01 15:16:05 -04003318 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3319 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3320
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003321 iter->cpu_file = cpu_file;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003322 iter->tr = &global_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003323 mutex_init(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003324 filp->private_data = iter;
3325
Steven Rostedt107bad82008-05-12 21:21:01 +02003326 if (iter->trace->pipe_open)
3327 iter->trace->pipe_open(iter);
Steven Rostedt107bad82008-05-12 21:21:01 +02003328
Arnd Bergmannb4447862010-07-07 23:40:11 +02003329 nonseekable_open(inode, filp);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003330out:
3331 mutex_unlock(&trace_types_lock);
3332 return ret;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003333
3334fail:
3335 kfree(iter->trace);
3336 kfree(iter);
3337 mutex_unlock(&trace_types_lock);
3338 return ret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003339}
3340
3341static int tracing_release_pipe(struct inode *inode, struct file *file)
3342{
3343 struct trace_iterator *iter = file->private_data;
3344
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003345 mutex_lock(&trace_types_lock);
3346
Steven Rostedt29bf4a52009-12-09 12:37:43 -05003347 if (iter->trace->pipe_close)
Steven Rostedtc521efd2009-12-07 09:06:24 -05003348 iter->trace->pipe_close(iter);
3349
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003350 mutex_unlock(&trace_types_lock);
3351
Rusty Russell44623442009-01-01 10:12:23 +10303352 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003353 mutex_destroy(&iter->mutex);
3354 kfree(iter->trace);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003355 kfree(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003356
3357 return 0;
3358}
3359
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003360static unsigned int
3361tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3362{
3363 struct trace_iterator *iter = filp->private_data;
3364
3365 if (trace_flags & TRACE_ITER_BLOCK) {
3366 /*
3367 * Always select as readable when in blocking mode
3368 */
3369 return POLLIN | POLLRDNORM;
Ingo Molnarafc2abc2008-05-12 21:21:00 +02003370 } else {
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003371 if (!trace_empty(iter))
3372 return POLLIN | POLLRDNORM;
3373 poll_wait(filp, &trace_wait, poll_table);
3374 if (!trace_empty(iter))
3375 return POLLIN | POLLRDNORM;
3376
3377 return 0;
3378 }
3379}
3380
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003381
3382void default_wait_pipe(struct trace_iterator *iter)
3383{
3384 DEFINE_WAIT(wait);
3385
3386 prepare_to_wait(&trace_wait, &wait, TASK_INTERRUPTIBLE);
3387
3388 if (trace_empty(iter))
3389 schedule();
3390
3391 finish_wait(&trace_wait, &wait);
3392}
3393
3394/*
3395 * This is a make-shift waitqueue.
3396 * A tracer might use this callback on some rare cases:
3397 *
3398 * 1) the current tracer might hold the runqueue lock when it wakes up
3399 * a reader, hence a deadlock (sched, function, and function graph tracers)
3400 * 2) the function tracers, trace all functions, we don't want
3401 * the overhead of calling wake_up and friends
3402 * (and tracing them too)
3403 *
3404 * Anyway, this is really very primitive wakeup.
3405 */
3406void poll_wait_pipe(struct trace_iterator *iter)
3407{
3408 set_current_state(TASK_INTERRUPTIBLE);
3409 /* sleep for 100 msecs, and try again. */
3410 schedule_timeout(HZ / 10);
3411}
3412
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003413/* Must be called with trace_types_lock mutex held. */
3414static int tracing_wait_pipe(struct file *filp)
3415{
3416 struct trace_iterator *iter = filp->private_data;
3417
3418 while (trace_empty(iter)) {
3419
3420 if ((filp->f_flags & O_NONBLOCK)) {
3421 return -EAGAIN;
3422 }
3423
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003424 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003425
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003426 iter->trace->wait_pipe(iter);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003427
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003428 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003429
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003430 if (signal_pending(current))
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003431 return -EINTR;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003432
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003433 /*
3434 * We block until we read something and tracing is disabled.
3435 * We still block if tracing is disabled, but we have never
3436 * read anything. This allows a user to cat this file, and
3437 * then enable tracing. But after we have read something,
3438 * we give an EOF when tracing is again disabled.
3439 *
3440 * iter->pos will be 0 if we haven't read anything.
3441 */
3442 if (!tracer_enabled && iter->pos)
3443 break;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003444 }
3445
3446 return 1;
3447}
3448
Steven Rostedtb3806b42008-05-12 21:20:46 +02003449/*
3450 * Consumer reader.
3451 */
3452static ssize_t
3453tracing_read_pipe(struct file *filp, char __user *ubuf,
3454 size_t cnt, loff_t *ppos)
3455{
3456 struct trace_iterator *iter = filp->private_data;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003457 static struct tracer *old_tracer;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003458 ssize_t sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003459
3460 /* return any leftover data */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003461 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3462 if (sret != -EBUSY)
3463 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003464
Steven Rostedtf9520752009-03-02 14:04:40 -05003465 trace_seq_init(&iter->seq);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003466
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003467 /* copy the tracer to avoid using a global lock all around */
Steven Rostedt107bad82008-05-12 21:21:01 +02003468 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003469 if (unlikely(old_tracer != current_trace && current_trace)) {
3470 old_tracer = current_trace;
3471 *iter->trace = *current_trace;
3472 }
3473 mutex_unlock(&trace_types_lock);
3474
3475 /*
3476 * Avoid more than one consumer on a single file descriptor
3477 * This is just a matter of traces coherency, the ring buffer itself
3478 * is protected.
3479 */
3480 mutex_lock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003481 if (iter->trace->read) {
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003482 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3483 if (sret)
Steven Rostedt107bad82008-05-12 21:21:01 +02003484 goto out;
Steven Rostedt107bad82008-05-12 21:21:01 +02003485 }
3486
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003487waitagain:
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003488 sret = tracing_wait_pipe(filp);
3489 if (sret <= 0)
3490 goto out;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003491
3492 /* stop when tracing is finished */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003493 if (trace_empty(iter)) {
3494 sret = 0;
Steven Rostedt107bad82008-05-12 21:21:01 +02003495 goto out;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003496 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003497
3498 if (cnt >= PAGE_SIZE)
3499 cnt = PAGE_SIZE - 1;
3500
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003501 /* reset all but tr, trace, and overruns */
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003502 memset(&iter->seq, 0,
3503 sizeof(struct trace_iterator) -
3504 offsetof(struct trace_iterator, seq));
Steven Rostedt4823ed72008-05-12 21:21:01 +02003505 iter->pos = -1;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003506
Lai Jiangshan4f535962009-05-18 19:35:34 +08003507 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003508 trace_access_lock(iter->cpu_file);
Jason Wessel955b61e2010-08-05 09:22:23 -05003509 while (trace_find_next_entry_inc(iter) != NULL) {
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003510 enum print_line_t ret;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003511 int len = iter->seq.len;
3512
Ingo Molnarf9896bf2008-05-12 21:20:47 +02003513 ret = print_trace_line(iter);
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003514 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt088b1e422008-05-12 21:20:48 +02003515 /* don't print partial lines */
3516 iter->seq.len = len;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003517 break;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003518 }
Frederic Weisbeckerb91facc2009-02-06 18:30:44 +01003519 if (ret != TRACE_TYPE_NO_CONSUME)
3520 trace_consume(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003521
3522 if (iter->seq.len >= cnt)
3523 break;
Jiri Olsaee5e51f2011-03-25 12:05:18 +01003524
3525 /*
3526 * Setting the full flag means we reached the trace_seq buffer
3527 * size and we should leave by partial output condition above.
3528 * One of the trace_seq_* functions is not used properly.
3529 */
3530 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3531 iter->ent->type);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003532 }
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003533 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003534 trace_event_read_unlock();
Steven Rostedtb3806b42008-05-12 21:20:46 +02003535
Steven Rostedtb3806b42008-05-12 21:20:46 +02003536 /* Now copy what we have to the user */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003537 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3538 if (iter->seq.readpos >= iter->seq.len)
Steven Rostedtf9520752009-03-02 14:04:40 -05003539 trace_seq_init(&iter->seq);
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003540
3541 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003542 * If there was nothing to send to user, in spite of consuming trace
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003543 * entries, go back to wait for more entries.
3544 */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003545 if (sret == -EBUSY)
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003546 goto waitagain;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003547
Steven Rostedt107bad82008-05-12 21:21:01 +02003548out:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003549 mutex_unlock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003550
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003551 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003552}
3553
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003554static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3555 struct pipe_buffer *buf)
3556{
3557 __free_page(buf->page);
3558}
3559
3560static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3561 unsigned int idx)
3562{
3563 __free_page(spd->pages[idx]);
3564}
3565
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08003566static const struct pipe_buf_operations tracing_pipe_buf_ops = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003567 .can_merge = 0,
3568 .map = generic_pipe_buf_map,
3569 .unmap = generic_pipe_buf_unmap,
3570 .confirm = generic_pipe_buf_confirm,
3571 .release = tracing_pipe_buf_release,
3572 .steal = generic_pipe_buf_steal,
3573 .get = generic_pipe_buf_get,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003574};
3575
Steven Rostedt34cd4992009-02-09 12:06:29 -05003576static size_t
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01003577tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003578{
3579 size_t count;
3580 int ret;
3581
3582 /* Seq buffer is page-sized, exactly what we need. */
3583 for (;;) {
3584 count = iter->seq.len;
3585 ret = print_trace_line(iter);
3586 count = iter->seq.len - count;
3587 if (rem < count) {
3588 rem = 0;
3589 iter->seq.len -= count;
3590 break;
3591 }
3592 if (ret == TRACE_TYPE_PARTIAL_LINE) {
3593 iter->seq.len -= count;
3594 break;
3595 }
3596
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08003597 if (ret != TRACE_TYPE_NO_CONSUME)
3598 trace_consume(iter);
Steven Rostedt34cd4992009-02-09 12:06:29 -05003599 rem -= count;
Jason Wessel955b61e2010-08-05 09:22:23 -05003600 if (!trace_find_next_entry_inc(iter)) {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003601 rem = 0;
3602 iter->ent = NULL;
3603 break;
3604 }
3605 }
3606
3607 return rem;
3608}
3609
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003610static ssize_t tracing_splice_read_pipe(struct file *filp,
3611 loff_t *ppos,
3612 struct pipe_inode_info *pipe,
3613 size_t len,
3614 unsigned int flags)
3615{
Jens Axboe35f3d142010-05-20 10:43:18 +02003616 struct page *pages_def[PIPE_DEF_BUFFERS];
3617 struct partial_page partial_def[PIPE_DEF_BUFFERS];
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003618 struct trace_iterator *iter = filp->private_data;
3619 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02003620 .pages = pages_def,
3621 .partial = partial_def,
Steven Rostedt34cd4992009-02-09 12:06:29 -05003622 .nr_pages = 0, /* This gets updated below. */
3623 .flags = flags,
3624 .ops = &tracing_pipe_buf_ops,
3625 .spd_release = tracing_spd_release_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003626 };
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003627 static struct tracer *old_tracer;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003628 ssize_t ret;
Steven Rostedt34cd4992009-02-09 12:06:29 -05003629 size_t rem;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003630 unsigned int i;
3631
Jens Axboe35f3d142010-05-20 10:43:18 +02003632 if (splice_grow_spd(pipe, &spd))
3633 return -ENOMEM;
3634
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003635 /* copy the tracer to avoid using a global lock all around */
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003636 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003637 if (unlikely(old_tracer != current_trace && current_trace)) {
3638 old_tracer = current_trace;
3639 *iter->trace = *current_trace;
3640 }
3641 mutex_unlock(&trace_types_lock);
3642
3643 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003644
3645 if (iter->trace->splice_read) {
3646 ret = iter->trace->splice_read(iter, filp,
3647 ppos, pipe, len, flags);
3648 if (ret)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003649 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003650 }
3651
3652 ret = tracing_wait_pipe(filp);
3653 if (ret <= 0)
Steven Rostedt34cd4992009-02-09 12:06:29 -05003654 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003655
Jason Wessel955b61e2010-08-05 09:22:23 -05003656 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003657 ret = -EFAULT;
Steven Rostedt34cd4992009-02-09 12:06:29 -05003658 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003659 }
3660
Lai Jiangshan4f535962009-05-18 19:35:34 +08003661 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003662 trace_access_lock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003663
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003664 /* Fill as many pages as possible. */
Jens Axboe35f3d142010-05-20 10:43:18 +02003665 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
3666 spd.pages[i] = alloc_page(GFP_KERNEL);
3667 if (!spd.pages[i])
Steven Rostedt34cd4992009-02-09 12:06:29 -05003668 break;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003669
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01003670 rem = tracing_fill_pipe_page(rem, iter);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003671
3672 /* Copy the data into the page, so we can start over. */
3673 ret = trace_seq_to_buffer(&iter->seq,
Jens Axboe35f3d142010-05-20 10:43:18 +02003674 page_address(spd.pages[i]),
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003675 iter->seq.len);
3676 if (ret < 0) {
Jens Axboe35f3d142010-05-20 10:43:18 +02003677 __free_page(spd.pages[i]);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003678 break;
3679 }
Jens Axboe35f3d142010-05-20 10:43:18 +02003680 spd.partial[i].offset = 0;
3681 spd.partial[i].len = iter->seq.len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003682
Steven Rostedtf9520752009-03-02 14:04:40 -05003683 trace_seq_init(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003684 }
3685
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003686 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003687 trace_event_read_unlock();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003688 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003689
3690 spd.nr_pages = i;
3691
Jens Axboe35f3d142010-05-20 10:43:18 +02003692 ret = splice_to_pipe(pipe, &spd);
3693out:
3694 splice_shrink_spd(pipe, &spd);
3695 return ret;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003696
Steven Rostedt34cd4992009-02-09 12:06:29 -05003697out_err:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003698 mutex_unlock(&iter->mutex);
Jens Axboe35f3d142010-05-20 10:43:18 +02003699 goto out;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003700}
3701
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003702struct ftrace_entries_info {
3703 struct trace_array *tr;
3704 int cpu;
3705};
3706
3707static int tracing_entries_open(struct inode *inode, struct file *filp)
3708{
3709 struct ftrace_entries_info *info;
3710
3711 if (tracing_disabled)
3712 return -ENODEV;
3713
3714 info = kzalloc(sizeof(*info), GFP_KERNEL);
3715 if (!info)
3716 return -ENOMEM;
3717
3718 info->tr = &global_trace;
3719 info->cpu = (unsigned long)inode->i_private;
3720
3721 filp->private_data = info;
3722
3723 return 0;
3724}
3725
Steven Rostedta98a3c32008-05-12 21:20:59 +02003726static ssize_t
3727tracing_entries_read(struct file *filp, char __user *ubuf,
3728 size_t cnt, loff_t *ppos)
3729{
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003730 struct ftrace_entries_info *info = filp->private_data;
3731 struct trace_array *tr = info->tr;
3732 char buf[64];
3733 int r = 0;
3734 ssize_t ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003735
Steven Rostedtdb526ca2009-03-12 13:53:25 -04003736 mutex_lock(&trace_types_lock);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003737
3738 if (info->cpu == RING_BUFFER_ALL_CPUS) {
3739 int cpu, buf_size_same;
3740 unsigned long size;
3741
3742 size = 0;
3743 buf_size_same = 1;
3744 /* check if all cpu sizes are same */
3745 for_each_tracing_cpu(cpu) {
3746 /* fill in the size from first enabled cpu */
3747 if (size == 0)
3748 size = tr->data[cpu]->entries;
3749 if (size != tr->data[cpu]->entries) {
3750 buf_size_same = 0;
3751 break;
3752 }
3753 }
3754
3755 if (buf_size_same) {
3756 if (!ring_buffer_expanded)
3757 r = sprintf(buf, "%lu (expanded: %lu)\n",
3758 size >> 10,
3759 trace_buf_size >> 10);
3760 else
3761 r = sprintf(buf, "%lu\n", size >> 10);
3762 } else
3763 r = sprintf(buf, "X\n");
3764 } else
3765 r = sprintf(buf, "%lu\n", tr->data[info->cpu]->entries >> 10);
3766
Steven Rostedtdb526ca2009-03-12 13:53:25 -04003767 mutex_unlock(&trace_types_lock);
3768
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003769 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3770 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003771}
3772
3773static ssize_t
3774tracing_entries_write(struct file *filp, const char __user *ubuf,
3775 size_t cnt, loff_t *ppos)
3776{
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003777 struct ftrace_entries_info *info = filp->private_data;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003778 unsigned long val;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003779 int ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003780
Peter Huewe22fe9b52011-06-07 21:58:27 +02003781 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3782 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003783 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003784
3785 /* must have at least 1 entry */
3786 if (!val)
3787 return -EINVAL;
3788
Steven Rostedt1696b2b2008-11-13 00:09:35 -05003789 /* value is in KB */
3790 val <<= 10;
3791
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003792 ret = tracing_resize_ring_buffer(val, info->cpu);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003793 if (ret < 0)
3794 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003795
Jiri Olsacf8517c2009-10-23 19:36:16 -04003796 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003797
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003798 return cnt;
3799}
Steven Rostedtbf5e6512008-11-10 21:46:00 -05003800
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003801static int
3802tracing_entries_release(struct inode *inode, struct file *filp)
3803{
3804 struct ftrace_entries_info *info = filp->private_data;
3805
3806 kfree(info);
3807
3808 return 0;
3809}
3810
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003811static ssize_t
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07003812tracing_total_entries_read(struct file *filp, char __user *ubuf,
3813 size_t cnt, loff_t *ppos)
3814{
3815 struct trace_array *tr = filp->private_data;
3816 char buf[64];
3817 int r, cpu;
3818 unsigned long size = 0, expanded_size = 0;
3819
3820 mutex_lock(&trace_types_lock);
3821 for_each_tracing_cpu(cpu) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003822 size += tr->data[cpu]->entries >> 10;
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07003823 if (!ring_buffer_expanded)
3824 expanded_size += trace_buf_size >> 10;
3825 }
3826 if (ring_buffer_expanded)
3827 r = sprintf(buf, "%lu\n", size);
3828 else
3829 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
3830 mutex_unlock(&trace_types_lock);
3831
3832 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3833}
3834
3835static ssize_t
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003836tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
3837 size_t cnt, loff_t *ppos)
3838{
3839 /*
3840 * There is no need to read what the user has written, this function
3841 * is just to make sure that there is no error when "echo" is used
3842 */
3843
3844 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02003845
3846 return cnt;
3847}
3848
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003849static int
3850tracing_free_buffer_release(struct inode *inode, struct file *filp)
3851{
Steven Rostedtcf30cf62011-06-14 22:44:07 -04003852 /* disable tracing ? */
3853 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
3854 tracing_off();
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003855 /* resize the ring buffer to 0 */
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003856 tracing_resize_ring_buffer(0, RING_BUFFER_ALL_CPUS);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003857
3858 return 0;
3859}
3860
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003861static ssize_t
3862tracing_mark_write(struct file *filp, const char __user *ubuf,
3863 size_t cnt, loff_t *fpos)
3864{
Steven Rostedtd696b582011-09-22 11:50:27 -04003865 unsigned long addr = (unsigned long)ubuf;
3866 struct ring_buffer_event *event;
3867 struct ring_buffer *buffer;
3868 struct print_entry *entry;
3869 unsigned long irq_flags;
3870 struct page *pages[2];
Steven Rostedt6edb2a82012-05-11 23:28:49 -04003871 void *map_page[2];
Steven Rostedtd696b582011-09-22 11:50:27 -04003872 int nr_pages = 1;
3873 ssize_t written;
Steven Rostedtd696b582011-09-22 11:50:27 -04003874 int offset;
3875 int size;
3876 int len;
3877 int ret;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04003878 int i;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003879
Steven Rostedtc76f0692008-11-07 22:36:02 -05003880 if (tracing_disabled)
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003881 return -EINVAL;
3882
3883 if (cnt > TRACE_BUF_SIZE)
3884 cnt = TRACE_BUF_SIZE;
3885
Steven Rostedtd696b582011-09-22 11:50:27 -04003886 /*
3887 * Userspace is injecting traces into the kernel trace buffer.
3888 * We want to be as non intrusive as possible.
3889 * To do so, we do not want to allocate any special buffers
3890 * or take any locks, but instead write the userspace data
3891 * straight into the ring buffer.
3892 *
3893 * First we need to pin the userspace buffer into memory,
3894 * which, most likely it is, because it just referenced it.
3895 * But there's no guarantee that it is. By using get_user_pages_fast()
3896 * and kmap_atomic/kunmap_atomic() we can get access to the
3897 * pages directly. We then write the data directly into the
3898 * ring buffer.
3899 */
3900 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003901
Steven Rostedtd696b582011-09-22 11:50:27 -04003902 /* check if we cross pages */
3903 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
3904 nr_pages = 2;
3905
3906 offset = addr & (PAGE_SIZE - 1);
3907 addr &= PAGE_MASK;
3908
3909 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
3910 if (ret < nr_pages) {
3911 while (--ret >= 0)
3912 put_page(pages[ret]);
3913 written = -EFAULT;
3914 goto out;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003915 }
3916
Steven Rostedt6edb2a82012-05-11 23:28:49 -04003917 for (i = 0; i < nr_pages; i++)
3918 map_page[i] = kmap_atomic(pages[i]);
Steven Rostedtd696b582011-09-22 11:50:27 -04003919
3920 local_save_flags(irq_flags);
3921 size = sizeof(*entry) + cnt + 2; /* possible \n added */
3922 buffer = global_trace.buffer;
3923 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3924 irq_flags, preempt_count());
3925 if (!event) {
3926 /* Ring buffer disabled, return as if not open for write */
3927 written = -EBADF;
3928 goto out_unlock;
3929 }
3930
3931 entry = ring_buffer_event_data(event);
3932 entry->ip = _THIS_IP_;
3933
3934 if (nr_pages == 2) {
3935 len = PAGE_SIZE - offset;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04003936 memcpy(&entry->buf, map_page[0] + offset, len);
3937 memcpy(&entry->buf[len], map_page[1], cnt - len);
Steven Rostedtd696b582011-09-22 11:50:27 -04003938 } else
Steven Rostedt6edb2a82012-05-11 23:28:49 -04003939 memcpy(&entry->buf, map_page[0] + offset, cnt);
Steven Rostedtd696b582011-09-22 11:50:27 -04003940
3941 if (entry->buf[cnt - 1] != '\n') {
3942 entry->buf[cnt] = '\n';
3943 entry->buf[cnt + 1] = '\0';
3944 } else
3945 entry->buf[cnt] = '\0';
3946
3947 ring_buffer_unlock_commit(buffer, event);
3948
3949 written = cnt;
3950
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02003951 *fpos += written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003952
Steven Rostedtd696b582011-09-22 11:50:27 -04003953 out_unlock:
Steven Rostedt6edb2a82012-05-11 23:28:49 -04003954 for (i = 0; i < nr_pages; i++){
3955 kunmap_atomic(map_page[i]);
3956 put_page(pages[i]);
3957 }
Steven Rostedtd696b582011-09-22 11:50:27 -04003958 out:
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02003959 return written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03003960}
3961
Li Zefan13f16d22009-12-08 11:16:11 +08003962static int tracing_clock_show(struct seq_file *m, void *v)
Zhaolei5079f322009-08-25 16:12:56 +08003963{
Zhaolei5079f322009-08-25 16:12:56 +08003964 int i;
3965
3966 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
Li Zefan13f16d22009-12-08 11:16:11 +08003967 seq_printf(m,
Zhaolei5079f322009-08-25 16:12:56 +08003968 "%s%s%s%s", i ? " " : "",
3969 i == trace_clock_id ? "[" : "", trace_clocks[i].name,
3970 i == trace_clock_id ? "]" : "");
Li Zefan13f16d22009-12-08 11:16:11 +08003971 seq_putc(m, '\n');
Zhaolei5079f322009-08-25 16:12:56 +08003972
Li Zefan13f16d22009-12-08 11:16:11 +08003973 return 0;
Zhaolei5079f322009-08-25 16:12:56 +08003974}
3975
3976static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
3977 size_t cnt, loff_t *fpos)
3978{
3979 char buf[64];
3980 const char *clockstr;
3981 int i;
3982
3983 if (cnt >= sizeof(buf))
3984 return -EINVAL;
3985
3986 if (copy_from_user(&buf, ubuf, cnt))
3987 return -EFAULT;
3988
3989 buf[cnt] = 0;
3990
3991 clockstr = strstrip(buf);
3992
3993 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
3994 if (strcmp(trace_clocks[i].name, clockstr) == 0)
3995 break;
3996 }
3997 if (i == ARRAY_SIZE(trace_clocks))
3998 return -EINVAL;
3999
4000 trace_clock_id = i;
4001
4002 mutex_lock(&trace_types_lock);
4003
4004 ring_buffer_set_clock(global_trace.buffer, trace_clocks[i].func);
4005 if (max_tr.buffer)
4006 ring_buffer_set_clock(max_tr.buffer, trace_clocks[i].func);
4007
4008 mutex_unlock(&trace_types_lock);
4009
4010 *fpos += cnt;
4011
4012 return cnt;
4013}
4014
Li Zefan13f16d22009-12-08 11:16:11 +08004015static int tracing_clock_open(struct inode *inode, struct file *file)
4016{
4017 if (tracing_disabled)
4018 return -ENODEV;
4019 return single_open(file, tracing_clock_show, NULL);
4020}
4021
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004022static const struct file_operations tracing_max_lat_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004023 .open = tracing_open_generic,
4024 .read = tracing_max_lat_read,
4025 .write = tracing_max_lat_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004026 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004027};
4028
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004029static const struct file_operations tracing_ctrl_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004030 .open = tracing_open_generic,
4031 .read = tracing_ctrl_read,
4032 .write = tracing_ctrl_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004033 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004034};
4035
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004036static const struct file_operations set_tracer_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004037 .open = tracing_open_generic,
4038 .read = tracing_set_trace_read,
4039 .write = tracing_set_trace_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004040 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004041};
4042
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004043static const struct file_operations tracing_pipe_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004044 .open = tracing_open_pipe,
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004045 .poll = tracing_poll_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004046 .read = tracing_read_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004047 .splice_read = tracing_splice_read_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004048 .release = tracing_release_pipe,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004049 .llseek = no_llseek,
Steven Rostedtb3806b42008-05-12 21:20:46 +02004050};
4051
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004052static const struct file_operations tracing_entries_fops = {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004053 .open = tracing_entries_open,
Steven Rostedta98a3c32008-05-12 21:20:59 +02004054 .read = tracing_entries_read,
4055 .write = tracing_entries_write,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004056 .release = tracing_entries_release,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004057 .llseek = generic_file_llseek,
Steven Rostedta98a3c32008-05-12 21:20:59 +02004058};
4059
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004060static const struct file_operations tracing_total_entries_fops = {
4061 .open = tracing_open_generic,
4062 .read = tracing_total_entries_read,
4063 .llseek = generic_file_llseek,
4064};
4065
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004066static const struct file_operations tracing_free_buffer_fops = {
4067 .write = tracing_free_buffer_write,
4068 .release = tracing_free_buffer_release,
4069};
4070
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004071static const struct file_operations tracing_mark_fops = {
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02004072 .open = tracing_open_generic,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004073 .write = tracing_mark_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004074 .llseek = generic_file_llseek,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004075};
4076
Zhaolei5079f322009-08-25 16:12:56 +08004077static const struct file_operations trace_clock_fops = {
Li Zefan13f16d22009-12-08 11:16:11 +08004078 .open = tracing_clock_open,
4079 .read = seq_read,
4080 .llseek = seq_lseek,
4081 .release = single_release,
Zhaolei5079f322009-08-25 16:12:56 +08004082 .write = tracing_clock_write,
4083};
4084
Steven Rostedt2cadf912008-12-01 22:20:19 -05004085struct ftrace_buffer_info {
4086 struct trace_array *tr;
4087 void *spare;
4088 int cpu;
4089 unsigned int read;
4090};
4091
4092static int tracing_buffers_open(struct inode *inode, struct file *filp)
4093{
4094 int cpu = (int)(long)inode->i_private;
4095 struct ftrace_buffer_info *info;
4096
4097 if (tracing_disabled)
4098 return -ENODEV;
4099
4100 info = kzalloc(sizeof(*info), GFP_KERNEL);
4101 if (!info)
4102 return -ENOMEM;
4103
4104 info->tr = &global_trace;
4105 info->cpu = cpu;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004106 info->spare = NULL;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004107 /* Force reading ring buffer for first read */
4108 info->read = (unsigned int)-1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004109
4110 filp->private_data = info;
4111
Lai Jiangshand1e7e022009-04-02 15:16:56 +08004112 return nonseekable_open(inode, filp);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004113}
4114
4115static ssize_t
4116tracing_buffers_read(struct file *filp, char __user *ubuf,
4117 size_t count, loff_t *ppos)
4118{
4119 struct ftrace_buffer_info *info = filp->private_data;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004120 ssize_t ret;
4121 size_t size;
4122
Steven Rostedt2dc5d122009-03-04 19:10:05 -05004123 if (!count)
4124 return 0;
4125
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004126 if (!info->spare)
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07004127 info->spare = ring_buffer_alloc_read_page(info->tr->buffer, info->cpu);
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004128 if (!info->spare)
4129 return -ENOMEM;
4130
Steven Rostedt2cadf912008-12-01 22:20:19 -05004131 /* Do we have previous read data to read? */
4132 if (info->read < PAGE_SIZE)
4133 goto read;
4134
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004135 trace_access_lock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004136 ret = ring_buffer_read_page(info->tr->buffer,
4137 &info->spare,
4138 count,
4139 info->cpu, 0);
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004140 trace_access_unlock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004141 if (ret < 0)
4142 return 0;
4143
Steven Rostedt436fc282011-10-14 10:44:25 -04004144 info->read = 0;
4145
Steven Rostedt2cadf912008-12-01 22:20:19 -05004146read:
4147 size = PAGE_SIZE - info->read;
4148 if (size > count)
4149 size = count;
4150
4151 ret = copy_to_user(ubuf, info->spare + info->read, size);
Steven Rostedt2dc5d122009-03-04 19:10:05 -05004152 if (ret == size)
Steven Rostedt2cadf912008-12-01 22:20:19 -05004153 return -EFAULT;
Steven Rostedt2dc5d122009-03-04 19:10:05 -05004154 size -= ret;
4155
Steven Rostedt2cadf912008-12-01 22:20:19 -05004156 *ppos += size;
4157 info->read += size;
4158
4159 return size;
4160}
4161
4162static int tracing_buffers_release(struct inode *inode, struct file *file)
4163{
4164 struct ftrace_buffer_info *info = file->private_data;
4165
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004166 if (info->spare)
4167 ring_buffer_free_read_page(info->tr->buffer, info->spare);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004168 kfree(info);
4169
4170 return 0;
4171}
4172
4173struct buffer_ref {
4174 struct ring_buffer *buffer;
4175 void *page;
4176 int ref;
4177};
4178
4179static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4180 struct pipe_buffer *buf)
4181{
4182 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4183
4184 if (--ref->ref)
4185 return;
4186
4187 ring_buffer_free_read_page(ref->buffer, ref->page);
4188 kfree(ref);
4189 buf->private = 0;
4190}
4191
4192static int buffer_pipe_buf_steal(struct pipe_inode_info *pipe,
4193 struct pipe_buffer *buf)
4194{
4195 return 1;
4196}
4197
4198static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4199 struct pipe_buffer *buf)
4200{
4201 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4202
4203 ref->ref++;
4204}
4205
4206/* Pipe buffer operations for a buffer. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08004207static const struct pipe_buf_operations buffer_pipe_buf_ops = {
Steven Rostedt2cadf912008-12-01 22:20:19 -05004208 .can_merge = 0,
4209 .map = generic_pipe_buf_map,
4210 .unmap = generic_pipe_buf_unmap,
4211 .confirm = generic_pipe_buf_confirm,
4212 .release = buffer_pipe_buf_release,
4213 .steal = buffer_pipe_buf_steal,
4214 .get = buffer_pipe_buf_get,
4215};
4216
4217/*
4218 * Callback from splice_to_pipe(), if we need to release some pages
4219 * at the end of the spd in case we error'ed out in filling the pipe.
4220 */
4221static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4222{
4223 struct buffer_ref *ref =
4224 (struct buffer_ref *)spd->partial[i].private;
4225
4226 if (--ref->ref)
4227 return;
4228
4229 ring_buffer_free_read_page(ref->buffer, ref->page);
4230 kfree(ref);
4231 spd->partial[i].private = 0;
4232}
4233
4234static ssize_t
4235tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4236 struct pipe_inode_info *pipe, size_t len,
4237 unsigned int flags)
4238{
4239 struct ftrace_buffer_info *info = file->private_data;
Jens Axboe35f3d142010-05-20 10:43:18 +02004240 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4241 struct page *pages_def[PIPE_DEF_BUFFERS];
Steven Rostedt2cadf912008-12-01 22:20:19 -05004242 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02004243 .pages = pages_def,
4244 .partial = partial_def,
Steven Rostedt2cadf912008-12-01 22:20:19 -05004245 .flags = flags,
4246 .ops = &buffer_pipe_buf_ops,
4247 .spd_release = buffer_spd_release,
4248 };
4249 struct buffer_ref *ref;
Steven Rostedt93459c62009-04-29 00:23:13 -04004250 int entries, size, i;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004251 size_t ret;
4252
Jens Axboe35f3d142010-05-20 10:43:18 +02004253 if (splice_grow_spd(pipe, &spd))
4254 return -ENOMEM;
4255
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004256 if (*ppos & (PAGE_SIZE - 1)) {
4257 WARN_ONCE(1, "Ftrace: previous read must page-align\n");
Jens Axboe35f3d142010-05-20 10:43:18 +02004258 ret = -EINVAL;
4259 goto out;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004260 }
4261
4262 if (len & (PAGE_SIZE - 1)) {
4263 WARN_ONCE(1, "Ftrace: splice_read should page-align\n");
Jens Axboe35f3d142010-05-20 10:43:18 +02004264 if (len < PAGE_SIZE) {
4265 ret = -EINVAL;
4266 goto out;
4267 }
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004268 len &= PAGE_MASK;
4269 }
4270
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004271 trace_access_lock(info->cpu);
Steven Rostedt93459c62009-04-29 00:23:13 -04004272 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
4273
Jens Axboe35f3d142010-05-20 10:43:18 +02004274 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05004275 struct page *page;
4276 int r;
4277
4278 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4279 if (!ref)
4280 break;
4281
Steven Rostedt7267fa62009-04-29 00:16:21 -04004282 ref->ref = 1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004283 ref->buffer = info->tr->buffer;
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07004284 ref->page = ring_buffer_alloc_read_page(ref->buffer, info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004285 if (!ref->page) {
4286 kfree(ref);
4287 break;
4288 }
4289
4290 r = ring_buffer_read_page(ref->buffer, &ref->page,
Steven Rostedtf2957f12009-04-29 00:26:30 -04004291 len, info->cpu, 1);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004292 if (r < 0) {
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07004293 ring_buffer_free_read_page(ref->buffer, ref->page);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004294 kfree(ref);
4295 break;
4296 }
4297
4298 /*
4299 * zero out any left over data, this is going to
4300 * user land.
4301 */
4302 size = ring_buffer_page_len(ref->page);
4303 if (size < PAGE_SIZE)
4304 memset(ref->page + size, 0, PAGE_SIZE - size);
4305
4306 page = virt_to_page(ref->page);
4307
4308 spd.pages[i] = page;
4309 spd.partial[i].len = PAGE_SIZE;
4310 spd.partial[i].offset = 0;
4311 spd.partial[i].private = (unsigned long)ref;
4312 spd.nr_pages++;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004313 *ppos += PAGE_SIZE;
Steven Rostedt93459c62009-04-29 00:23:13 -04004314
4315 entries = ring_buffer_entries_cpu(info->tr->buffer, info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004316 }
4317
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004318 trace_access_unlock(info->cpu);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004319 spd.nr_pages = i;
4320
4321 /* did we read anything? */
4322 if (!spd.nr_pages) {
4323 if (flags & SPLICE_F_NONBLOCK)
4324 ret = -EAGAIN;
4325 else
4326 ret = 0;
4327 /* TODO: block */
Jens Axboe35f3d142010-05-20 10:43:18 +02004328 goto out;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004329 }
4330
4331 ret = splice_to_pipe(pipe, &spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02004332 splice_shrink_spd(pipe, &spd);
4333out:
Steven Rostedt2cadf912008-12-01 22:20:19 -05004334 return ret;
4335}
4336
4337static const struct file_operations tracing_buffers_fops = {
4338 .open = tracing_buffers_open,
4339 .read = tracing_buffers_read,
4340 .release = tracing_buffers_release,
4341 .splice_read = tracing_buffers_splice_read,
4342 .llseek = no_llseek,
4343};
4344
Steven Rostedtc8d77182009-04-29 18:03:45 -04004345static ssize_t
4346tracing_stats_read(struct file *filp, char __user *ubuf,
4347 size_t count, loff_t *ppos)
4348{
4349 unsigned long cpu = (unsigned long)filp->private_data;
4350 struct trace_array *tr = &global_trace;
4351 struct trace_seq *s;
4352 unsigned long cnt;
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07004353 unsigned long long t;
4354 unsigned long usec_rem;
Steven Rostedtc8d77182009-04-29 18:03:45 -04004355
Li Zefane4f2d102009-06-15 10:57:28 +08004356 s = kmalloc(sizeof(*s), GFP_KERNEL);
Steven Rostedtc8d77182009-04-29 18:03:45 -04004357 if (!s)
Roel Kluina6463652009-11-11 22:26:35 +01004358 return -ENOMEM;
Steven Rostedtc8d77182009-04-29 18:03:45 -04004359
4360 trace_seq_init(s);
4361
4362 cnt = ring_buffer_entries_cpu(tr->buffer, cpu);
4363 trace_seq_printf(s, "entries: %ld\n", cnt);
4364
4365 cnt = ring_buffer_overrun_cpu(tr->buffer, cpu);
4366 trace_seq_printf(s, "overrun: %ld\n", cnt);
4367
4368 cnt = ring_buffer_commit_overrun_cpu(tr->buffer, cpu);
4369 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
4370
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07004371 cnt = ring_buffer_bytes_cpu(tr->buffer, cpu);
4372 trace_seq_printf(s, "bytes: %ld\n", cnt);
4373
4374 t = ns2usecs(ring_buffer_oldest_event_ts(tr->buffer, cpu));
4375 usec_rem = do_div(t, USEC_PER_SEC);
4376 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n", t, usec_rem);
4377
4378 t = ns2usecs(ring_buffer_time_stamp(tr->buffer, cpu));
4379 usec_rem = do_div(t, USEC_PER_SEC);
4380 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
4381
Steven Rostedtc8d77182009-04-29 18:03:45 -04004382 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
4383
4384 kfree(s);
4385
4386 return count;
4387}
4388
4389static const struct file_operations tracing_stats_fops = {
4390 .open = tracing_open_generic,
4391 .read = tracing_stats_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004392 .llseek = generic_file_llseek,
Steven Rostedtc8d77182009-04-29 18:03:45 -04004393};
4394
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004395#ifdef CONFIG_DYNAMIC_FTRACE
4396
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004397int __weak ftrace_arch_read_dyn_info(char *buf, int size)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004398{
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004399 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004400}
4401
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004402static ssize_t
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004403tracing_read_dyn_info(struct file *filp, char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004404 size_t cnt, loff_t *ppos)
4405{
Steven Rostedta26a2a22008-10-31 00:03:22 -04004406 static char ftrace_dyn_info_buffer[1024];
4407 static DEFINE_MUTEX(dyn_info_mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004408 unsigned long *p = filp->private_data;
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004409 char *buf = ftrace_dyn_info_buffer;
Steven Rostedta26a2a22008-10-31 00:03:22 -04004410 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004411 int r;
4412
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004413 mutex_lock(&dyn_info_mutex);
4414 r = sprintf(buf, "%ld ", *p);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004415
Steven Rostedta26a2a22008-10-31 00:03:22 -04004416 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004417 buf[r++] = '\n';
4418
4419 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4420
4421 mutex_unlock(&dyn_info_mutex);
4422
4423 return r;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004424}
4425
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004426static const struct file_operations tracing_dyn_info_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004427 .open = tracing_open_generic,
Steven Rostedtb807c3d2008-10-30 16:08:33 -04004428 .read = tracing_read_dyn_info,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004429 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004430};
4431#endif
4432
4433static struct dentry *d_tracer;
4434
4435struct dentry *tracing_init_dentry(void)
4436{
4437 static int once;
4438
4439 if (d_tracer)
4440 return d_tracer;
4441
Frederic Weisbecker3e1f60b2009-03-22 23:10:45 +01004442 if (!debugfs_initialized())
4443 return NULL;
4444
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004445 d_tracer = debugfs_create_dir("tracing", NULL);
4446
4447 if (!d_tracer && !once) {
4448 once = 1;
4449 pr_warning("Could not create debugfs directory 'tracing'\n");
4450 return NULL;
4451 }
4452
4453 return d_tracer;
4454}
4455
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004456static struct dentry *d_percpu;
4457
4458struct dentry *tracing_dentry_percpu(void)
4459{
4460 static int once;
4461 struct dentry *d_tracer;
4462
4463 if (d_percpu)
4464 return d_percpu;
4465
4466 d_tracer = tracing_init_dentry();
4467
4468 if (!d_tracer)
4469 return NULL;
4470
4471 d_percpu = debugfs_create_dir("per_cpu", d_tracer);
4472
4473 if (!d_percpu && !once) {
4474 once = 1;
4475 pr_warning("Could not create debugfs directory 'per_cpu'\n");
4476 return NULL;
4477 }
4478
4479 return d_percpu;
4480}
4481
4482static void tracing_init_debugfs_percpu(long cpu)
4483{
4484 struct dentry *d_percpu = tracing_dentry_percpu();
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004485 struct dentry *d_cpu;
Steven Rostedtdd49a382010-10-20 21:51:26 -04004486 char cpu_dir[30]; /* 30 characters should be more than enough */
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004487
Namhyung Kim0a3d7ce2012-04-23 10:11:57 +09004488 if (!d_percpu)
4489 return;
4490
Steven Rostedtdd49a382010-10-20 21:51:26 -04004491 snprintf(cpu_dir, 30, "cpu%ld", cpu);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004492 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
4493 if (!d_cpu) {
4494 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
4495 return;
4496 }
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004497
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01004498 /* per cpu trace_pipe */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004499 trace_create_file("trace_pipe", 0444, d_cpu,
4500 (void *) cpu, &tracing_pipe_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004501
4502 /* per cpu trace */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004503 trace_create_file("trace", 0644, d_cpu,
4504 (void *) cpu, &tracing_fops);
Steven Rostedt7f96f932009-03-13 00:37:42 -04004505
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004506 trace_create_file("trace_pipe_raw", 0444, d_cpu,
4507 (void *) cpu, &tracing_buffers_fops);
Steven Rostedtc8d77182009-04-29 18:03:45 -04004508
4509 trace_create_file("stats", 0444, d_cpu,
4510 (void *) cpu, &tracing_stats_fops);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004511
4512 trace_create_file("buffer_size_kb", 0444, d_cpu,
4513 (void *) cpu, &tracing_entries_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004514}
4515
Steven Rostedt60a11772008-05-12 21:20:44 +02004516#ifdef CONFIG_FTRACE_SELFTEST
4517/* Let selftest have access to static functions in this file */
4518#include "trace_selftest.c"
4519#endif
4520
Steven Rostedt577b7852009-02-26 23:43:05 -05004521struct trace_option_dentry {
4522 struct tracer_opt *opt;
4523 struct tracer_flags *flags;
4524 struct dentry *entry;
4525};
4526
4527static ssize_t
4528trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
4529 loff_t *ppos)
4530{
4531 struct trace_option_dentry *topt = filp->private_data;
4532 char *buf;
4533
4534 if (topt->flags->val & topt->opt->bit)
4535 buf = "1\n";
4536 else
4537 buf = "0\n";
4538
4539 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4540}
4541
4542static ssize_t
4543trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
4544 loff_t *ppos)
4545{
4546 struct trace_option_dentry *topt = filp->private_data;
4547 unsigned long val;
Steven Rostedt577b7852009-02-26 23:43:05 -05004548 int ret;
4549
Peter Huewe22fe9b52011-06-07 21:58:27 +02004550 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4551 if (ret)
Steven Rostedt577b7852009-02-26 23:43:05 -05004552 return ret;
4553
Li Zefan8d18eaa2009-12-08 11:17:06 +08004554 if (val != 0 && val != 1)
Steven Rostedt577b7852009-02-26 23:43:05 -05004555 return -EINVAL;
Li Zefan8d18eaa2009-12-08 11:17:06 +08004556
4557 if (!!(topt->flags->val & topt->opt->bit) != val) {
4558 mutex_lock(&trace_types_lock);
4559 ret = __set_tracer_option(current_trace, topt->flags,
Steven Rostedtc757bea2009-12-21 22:35:16 -05004560 topt->opt, !val);
Li Zefan8d18eaa2009-12-08 11:17:06 +08004561 mutex_unlock(&trace_types_lock);
4562 if (ret)
4563 return ret;
Steven Rostedt577b7852009-02-26 23:43:05 -05004564 }
4565
4566 *ppos += cnt;
4567
4568 return cnt;
4569}
4570
4571
4572static const struct file_operations trace_options_fops = {
4573 .open = tracing_open_generic,
4574 .read = trace_options_read,
4575 .write = trace_options_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004576 .llseek = generic_file_llseek,
Steven Rostedt577b7852009-02-26 23:43:05 -05004577};
4578
Steven Rostedta8259072009-02-26 22:19:12 -05004579static ssize_t
4580trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
4581 loff_t *ppos)
4582{
4583 long index = (long)filp->private_data;
4584 char *buf;
4585
4586 if (trace_flags & (1 << index))
4587 buf = "1\n";
4588 else
4589 buf = "0\n";
4590
4591 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
4592}
4593
4594static ssize_t
4595trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
4596 loff_t *ppos)
4597{
4598 long index = (long)filp->private_data;
Steven Rostedta8259072009-02-26 22:19:12 -05004599 unsigned long val;
4600 int ret;
4601
Peter Huewe22fe9b52011-06-07 21:58:27 +02004602 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4603 if (ret)
Steven Rostedta8259072009-02-26 22:19:12 -05004604 return ret;
4605
Zhaoleif2d84b62009-08-07 18:55:48 +08004606 if (val != 0 && val != 1)
Steven Rostedta8259072009-02-26 22:19:12 -05004607 return -EINVAL;
Zhaoleif2d84b62009-08-07 18:55:48 +08004608 set_tracer_flags(1 << index, val);
Steven Rostedta8259072009-02-26 22:19:12 -05004609
4610 *ppos += cnt;
4611
4612 return cnt;
4613}
4614
Steven Rostedta8259072009-02-26 22:19:12 -05004615static const struct file_operations trace_options_core_fops = {
4616 .open = tracing_open_generic,
4617 .read = trace_options_core_read,
4618 .write = trace_options_core_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004619 .llseek = generic_file_llseek,
Steven Rostedta8259072009-02-26 22:19:12 -05004620};
4621
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004622struct dentry *trace_create_file(const char *name,
Al Virof4ae40a2011-07-24 04:33:43 -04004623 umode_t mode,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004624 struct dentry *parent,
4625 void *data,
4626 const struct file_operations *fops)
4627{
4628 struct dentry *ret;
4629
4630 ret = debugfs_create_file(name, mode, parent, data, fops);
4631 if (!ret)
4632 pr_warning("Could not create debugfs '%s' entry\n", name);
4633
4634 return ret;
4635}
4636
4637
Steven Rostedta8259072009-02-26 22:19:12 -05004638static struct dentry *trace_options_init_dentry(void)
4639{
4640 struct dentry *d_tracer;
4641 static struct dentry *t_options;
4642
4643 if (t_options)
4644 return t_options;
4645
4646 d_tracer = tracing_init_dentry();
4647 if (!d_tracer)
4648 return NULL;
4649
4650 t_options = debugfs_create_dir("options", d_tracer);
4651 if (!t_options) {
4652 pr_warning("Could not create debugfs directory 'options'\n");
4653 return NULL;
4654 }
4655
4656 return t_options;
4657}
4658
Steven Rostedt577b7852009-02-26 23:43:05 -05004659static void
4660create_trace_option_file(struct trace_option_dentry *topt,
4661 struct tracer_flags *flags,
4662 struct tracer_opt *opt)
4663{
4664 struct dentry *t_options;
Steven Rostedt577b7852009-02-26 23:43:05 -05004665
4666 t_options = trace_options_init_dentry();
4667 if (!t_options)
4668 return;
4669
4670 topt->flags = flags;
4671 topt->opt = opt;
4672
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004673 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05004674 &trace_options_fops);
4675
Steven Rostedt577b7852009-02-26 23:43:05 -05004676}
4677
4678static struct trace_option_dentry *
4679create_trace_option_files(struct tracer *tracer)
4680{
4681 struct trace_option_dentry *topts;
4682 struct tracer_flags *flags;
4683 struct tracer_opt *opts;
4684 int cnt;
4685
4686 if (!tracer)
4687 return NULL;
4688
4689 flags = tracer->flags;
4690
4691 if (!flags || !flags->opts)
4692 return NULL;
4693
4694 opts = flags->opts;
4695
4696 for (cnt = 0; opts[cnt].name; cnt++)
4697 ;
4698
Steven Rostedt0cfe8242009-02-27 10:51:10 -05004699 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
Steven Rostedt577b7852009-02-26 23:43:05 -05004700 if (!topts)
4701 return NULL;
4702
4703 for (cnt = 0; opts[cnt].name; cnt++)
4704 create_trace_option_file(&topts[cnt], flags,
4705 &opts[cnt]);
4706
4707 return topts;
4708}
4709
4710static void
4711destroy_trace_option_files(struct trace_option_dentry *topts)
4712{
4713 int cnt;
4714
4715 if (!topts)
4716 return;
4717
4718 for (cnt = 0; topts[cnt].opt; cnt++) {
4719 if (topts[cnt].entry)
4720 debugfs_remove(topts[cnt].entry);
4721 }
4722
4723 kfree(topts);
4724}
4725
Steven Rostedta8259072009-02-26 22:19:12 -05004726static struct dentry *
4727create_trace_option_core_file(const char *option, long index)
4728{
4729 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05004730
4731 t_options = trace_options_init_dentry();
4732 if (!t_options)
4733 return NULL;
4734
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004735 return trace_create_file(option, 0644, t_options, (void *)index,
Steven Rostedta8259072009-02-26 22:19:12 -05004736 &trace_options_core_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05004737}
4738
4739static __init void create_trace_options_dir(void)
4740{
4741 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05004742 int i;
4743
4744 t_options = trace_options_init_dentry();
4745 if (!t_options)
4746 return;
4747
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004748 for (i = 0; trace_options[i]; i++)
4749 create_trace_option_core_file(trace_options[i], i);
Steven Rostedta8259072009-02-26 22:19:12 -05004750}
4751
Steven Rostedt499e5472012-02-22 15:50:28 -05004752static ssize_t
4753rb_simple_read(struct file *filp, char __user *ubuf,
4754 size_t cnt, loff_t *ppos)
4755{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04004756 struct trace_array *tr = filp->private_data;
4757 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt499e5472012-02-22 15:50:28 -05004758 char buf[64];
4759 int r;
4760
4761 if (buffer)
4762 r = ring_buffer_record_is_on(buffer);
4763 else
4764 r = 0;
4765
4766 r = sprintf(buf, "%d\n", r);
4767
4768 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4769}
4770
4771static ssize_t
4772rb_simple_write(struct file *filp, const char __user *ubuf,
4773 size_t cnt, loff_t *ppos)
4774{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04004775 struct trace_array *tr = filp->private_data;
4776 struct ring_buffer *buffer = tr->buffer;
Steven Rostedt499e5472012-02-22 15:50:28 -05004777 unsigned long val;
4778 int ret;
4779
4780 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4781 if (ret)
4782 return ret;
4783
4784 if (buffer) {
4785 if (val)
4786 ring_buffer_record_on(buffer);
4787 else
4788 ring_buffer_record_off(buffer);
4789 }
4790
4791 (*ppos)++;
4792
4793 return cnt;
4794}
4795
4796static const struct file_operations rb_simple_fops = {
4797 .open = tracing_open_generic,
4798 .read = rb_simple_read,
4799 .write = rb_simple_write,
4800 .llseek = default_llseek,
4801};
4802
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004803static __init int tracer_init_debugfs(void)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004804{
4805 struct dentry *d_tracer;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004806 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004807
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004808 trace_access_lock_init();
4809
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004810 d_tracer = tracing_init_dentry();
4811
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004812 trace_create_file("tracing_enabled", 0644, d_tracer,
4813 &global_trace, &tracing_ctrl_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004814
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004815 trace_create_file("trace_options", 0644, d_tracer,
4816 NULL, &tracing_iter_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004817
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004818 trace_create_file("tracing_cpumask", 0644, d_tracer,
4819 NULL, &tracing_cpumask_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05004820
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004821 trace_create_file("trace", 0644, d_tracer,
4822 (void *) TRACE_PIPE_ALL_CPU, &tracing_fops);
Ingo Molnarc7078de2008-05-12 21:20:52 +02004823
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004824 trace_create_file("available_tracers", 0444, d_tracer,
4825 &global_trace, &show_traces_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004826
Li Zefan339ae5d2009-04-17 10:34:30 +08004827 trace_create_file("current_tracer", 0644, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004828 &global_trace, &set_tracer_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004829
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04004830#ifdef CONFIG_TRACER_MAX_TRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004831 trace_create_file("tracing_max_latency", 0644, d_tracer,
4832 &tracing_max_latency, &tracing_max_lat_fops);
Tim Bird0e950172010-02-25 15:36:43 -08004833#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004834
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004835 trace_create_file("tracing_thresh", 0644, d_tracer,
4836 &tracing_thresh, &tracing_max_lat_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004837
Li Zefan339ae5d2009-04-17 10:34:30 +08004838 trace_create_file("README", 0444, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004839 NULL, &tracing_readme_fops);
Ingo Molnar7bd2f242008-05-12 21:20:45 +02004840
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004841 trace_create_file("trace_pipe", 0444, d_tracer,
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004842 (void *) TRACE_PIPE_ALL_CPU, &tracing_pipe_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004843
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004844 trace_create_file("buffer_size_kb", 0644, d_tracer,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004845 (void *) RING_BUFFER_ALL_CPUS, &tracing_entries_fops);
Steven Rostedta98a3c32008-05-12 21:20:59 +02004846
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004847 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
4848 &global_trace, &tracing_total_entries_fops);
4849
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004850 trace_create_file("free_buffer", 0644, d_tracer,
4851 &global_trace, &tracing_free_buffer_fops);
4852
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004853 trace_create_file("trace_marker", 0220, d_tracer,
4854 NULL, &tracing_mark_fops);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004855
Avadh Patel69abe6a2009-04-10 16:04:48 -04004856 trace_create_file("saved_cmdlines", 0444, d_tracer,
4857 NULL, &tracing_saved_cmdlines_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004858
Zhaolei5079f322009-08-25 16:12:56 +08004859 trace_create_file("trace_clock", 0644, d_tracer, NULL,
4860 &trace_clock_fops);
4861
Steven Rostedt499e5472012-02-22 15:50:28 -05004862 trace_create_file("tracing_on", 0644, d_tracer,
Steven Rostedt348f0fc2012-04-16 15:41:28 -04004863 &global_trace, &rb_simple_fops);
Steven Rostedt499e5472012-02-22 15:50:28 -05004864
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004865#ifdef CONFIG_DYNAMIC_FTRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004866 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
4867 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004868#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004869
Frederic Weisbecker5452af62009-03-27 00:25:38 +01004870 create_trace_options_dir();
4871
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004872 for_each_tracing_cpu(cpu)
4873 tracing_init_debugfs_percpu(cpu);
4874
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01004875 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004876}
4877
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004878static int trace_panic_handler(struct notifier_block *this,
4879 unsigned long event, void *unused)
4880{
Steven Rostedt944ac422008-10-23 19:26:08 -04004881 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004882 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004883 return NOTIFY_OK;
4884}
4885
4886static struct notifier_block trace_panic_notifier = {
4887 .notifier_call = trace_panic_handler,
4888 .next = NULL,
4889 .priority = 150 /* priority: INT_MAX >= x >= 0 */
4890};
4891
4892static int trace_die_handler(struct notifier_block *self,
4893 unsigned long val,
4894 void *data)
4895{
4896 switch (val) {
4897 case DIE_OOPS:
Steven Rostedt944ac422008-10-23 19:26:08 -04004898 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004899 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004900 break;
4901 default:
4902 break;
4903 }
4904 return NOTIFY_OK;
4905}
4906
4907static struct notifier_block trace_die_notifier = {
4908 .notifier_call = trace_die_handler,
4909 .priority = 200
4910};
4911
4912/*
4913 * printk is set to max of 1024, we really don't need it that big.
4914 * Nothing should be printing 1000 characters anyway.
4915 */
4916#define TRACE_MAX_PRINT 1000
4917
4918/*
4919 * Define here KERN_TRACE so that we have one place to modify
4920 * it if we decide to change what log level the ftrace dump
4921 * should be at.
4922 */
Steven Rostedt428aee12009-01-14 12:24:42 -05004923#define KERN_TRACE KERN_EMERG
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004924
Jason Wessel955b61e2010-08-05 09:22:23 -05004925void
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004926trace_printk_seq(struct trace_seq *s)
4927{
4928 /* Probably should print a warning here. */
4929 if (s->len >= 1000)
4930 s->len = 1000;
4931
4932 /* should be zero ended, but we are paranoid. */
4933 s->buffer[s->len] = 0;
4934
4935 printk(KERN_TRACE "%s", s->buffer);
4936
Steven Rostedtf9520752009-03-02 14:04:40 -05004937 trace_seq_init(s);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004938}
4939
Jason Wessel955b61e2010-08-05 09:22:23 -05004940void trace_init_global_iter(struct trace_iterator *iter)
4941{
4942 iter->tr = &global_trace;
4943 iter->trace = current_trace;
4944 iter->cpu_file = TRACE_PIPE_ALL_CPU;
4945}
4946
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004947static void
4948__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004949{
Thomas Gleixner445c8952009-12-02 19:49:50 +01004950 static arch_spinlock_t ftrace_dump_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01004951 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004952 /* use static because iter can be a bit big for the stack */
4953 static struct trace_iterator iter;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004954 unsigned int old_userobj;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004955 static int dump_ran;
Steven Rostedtd7690412008-10-01 00:29:53 -04004956 unsigned long flags;
4957 int cnt = 0, cpu;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004958
4959 /* only one dump */
Steven Rostedtcd891ae2009-04-28 11:39:34 -04004960 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01004961 arch_spin_lock(&ftrace_dump_lock);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004962 if (dump_ran)
4963 goto out;
4964
4965 dump_ran = 1;
4966
Steven Rostedt0ee6b6c2009-01-14 14:50:19 -05004967 tracing_off();
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004968
Steven Rostedte0a413f2011-09-29 21:26:16 -04004969 /* Did function tracer already get disabled? */
4970 if (ftrace_is_dead()) {
4971 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
4972 printk("# MAY BE MISSING FUNCTION EVENTS\n");
4973 }
4974
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004975 if (disable_tracing)
4976 ftrace_kill();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004977
Jason Wessel955b61e2010-08-05 09:22:23 -05004978 trace_init_global_iter(&iter);
4979
Steven Rostedtd7690412008-10-01 00:29:53 -04004980 for_each_tracing_cpu(cpu) {
Jason Wessel955b61e2010-08-05 09:22:23 -05004981 atomic_inc(&iter.tr->data[cpu]->disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -04004982 }
4983
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01004984 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
4985
Török Edwinb54d3de2008-11-22 13:28:48 +02004986 /* don't look at user memory in panic mode */
4987 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
4988
Steven Rostedte543ad72009-03-04 18:20:36 -05004989 /* Simulate the iterator */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04004990 iter.tr = &global_trace;
4991 iter.trace = current_trace;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02004992
4993 switch (oops_dump_mode) {
4994 case DUMP_ALL:
4995 iter.cpu_file = TRACE_PIPE_ALL_CPU;
4996 break;
4997 case DUMP_ORIG:
4998 iter.cpu_file = raw_smp_processor_id();
4999 break;
5000 case DUMP_NONE:
5001 goto out_enable;
5002 default:
5003 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
5004 iter.cpu_file = TRACE_PIPE_ALL_CPU;
5005 }
5006
5007 printk(KERN_TRACE "Dumping ftrace buffer:\n");
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005008
5009 /*
5010 * We need to stop all tracing on all CPUS to read the
5011 * the next buffer. This is a bit expensive, but is
5012 * not done often. We fill all what we can read,
5013 * and then release the locks again.
5014 */
5015
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005016 while (!trace_empty(&iter)) {
5017
5018 if (!cnt)
5019 printk(KERN_TRACE "---------------------------------\n");
5020
5021 cnt++;
5022
5023 /* reset all but tr, trace, and overruns */
5024 memset(&iter.seq, 0,
5025 sizeof(struct trace_iterator) -
5026 offsetof(struct trace_iterator, seq));
5027 iter.iter_flags |= TRACE_FILE_LAT_FMT;
5028 iter.pos = -1;
5029
Jason Wessel955b61e2010-08-05 09:22:23 -05005030 if (trace_find_next_entry_inc(&iter) != NULL) {
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08005031 int ret;
5032
5033 ret = print_trace_line(&iter);
5034 if (ret != TRACE_TYPE_NO_CONSUME)
5035 trace_consume(&iter);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005036 }
Steven Rostedtb892e5c2012-03-01 22:06:48 -05005037 touch_nmi_watchdog();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005038
5039 trace_printk_seq(&iter.seq);
5040 }
5041
5042 if (!cnt)
5043 printk(KERN_TRACE " (ftrace buffer empty)\n");
5044 else
5045 printk(KERN_TRACE "---------------------------------\n");
5046
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005047 out_enable:
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005048 /* Re-enable tracing if requested */
5049 if (!disable_tracing) {
5050 trace_flags |= old_userobj;
5051
5052 for_each_tracing_cpu(cpu) {
Jason Wessel955b61e2010-08-05 09:22:23 -05005053 atomic_dec(&iter.tr->data[cpu]->disabled);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005054 }
5055 tracing_on();
5056 }
5057
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005058 out:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01005059 arch_spin_unlock(&ftrace_dump_lock);
Steven Rostedtcd891ae2009-04-28 11:39:34 -04005060 local_irq_restore(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005061}
5062
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005063/* By default: disable tracing after the dump */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005064void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005065{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005066 __ftrace_dump(true, oops_dump_mode);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005067}
Paul E. McKenneya8eecf22011-10-02 11:01:15 -07005068EXPORT_SYMBOL_GPL(ftrace_dump);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01005069
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005070__init static int tracer_alloc_buffers(void)
5071{
Steven Rostedt73c51622009-03-11 13:42:01 -04005072 int ring_buf_size;
David Sharp750912f2010-12-08 13:46:47 -08005073 enum ring_buffer_flags rb_flags;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005074 int i;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305075 int ret = -ENOMEM;
5076
David Sharp750912f2010-12-08 13:46:47 -08005077
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305078 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
5079 goto out;
5080
5081 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
5082 goto out_free_buffer_mask;
5083
Steven Rostedt07d777f2011-09-22 14:01:55 -04005084 /* Only allocate trace_printk buffers if a trace_printk exists */
5085 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
5086 trace_printk_init_buffers();
5087
Steven Rostedt73c51622009-03-11 13:42:01 -04005088 /* To save memory, keep the ring buffer size to its minimum */
5089 if (ring_buffer_expanded)
5090 ring_buf_size = trace_buf_size;
5091 else
5092 ring_buf_size = 1;
5093
David Sharp750912f2010-12-08 13:46:47 -08005094 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5095
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305096 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
5097 cpumask_copy(tracing_cpumask, cpu_all_mask);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005098
Steven Rostedtab464282008-05-12 21:21:00 +02005099 /* TODO: make the number of buffers hot pluggable with CPUS */
David Sharp750912f2010-12-08 13:46:47 -08005100 global_trace.buffer = ring_buffer_alloc(ring_buf_size, rb_flags);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005101 if (!global_trace.buffer) {
5102 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
5103 WARN_ON(1);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305104 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005105 }
Steven Rostedt499e5472012-02-22 15:50:28 -05005106 if (global_trace.buffer_disabled)
5107 tracing_off();
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005108
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305109
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005110#ifdef CONFIG_TRACER_MAX_TRACE
David Sharp750912f2010-12-08 13:46:47 -08005111 max_tr.buffer = ring_buffer_alloc(1, rb_flags);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005112 if (!max_tr.buffer) {
5113 printk(KERN_ERR "tracer: failed to allocate max ring buffer!\n");
5114 WARN_ON(1);
5115 ring_buffer_free(global_trace.buffer);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305116 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005117 }
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005118#endif
5119
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02005120 /* Allocate the first page for all buffers */
Steven Rostedtab464282008-05-12 21:21:00 +02005121 for_each_tracing_cpu(i) {
jolsa@redhat.com566b0aa2009-07-16 21:44:26 +02005122 global_trace.data[i] = &per_cpu(global_trace_cpu, i);
Tejun Heo9705f692009-10-29 22:34:13 +09005123 max_tr.data[i] = &per_cpu(max_tr_data, i);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005124 }
5125
Vaibhav Nagarnaika591c732012-05-03 10:40:34 -07005126 set_buffer_entries(&global_trace,
5127 ring_buffer_size(global_trace.buffer, 0));
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08005128#ifdef CONFIG_TRACER_MAX_TRACE
5129 set_buffer_entries(&max_tr, 1);
5130#endif
5131
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005132 trace_init_cmdlines();
5133
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02005134 register_tracer(&nop_trace);
Steven Rostedt79fb0762009-02-02 21:38:33 -05005135 current_trace = &nop_trace;
Steven Rostedt60a11772008-05-12 21:20:44 +02005136 /* All seems OK, enable tracing */
5137 tracing_disabled = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04005138
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005139 atomic_notifier_chain_register(&panic_notifier_list,
5140 &trace_panic_notifier);
5141
5142 register_die_notifier(&trace_die_notifier);
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01005143
5144 return 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005145
Rusty Russell9e01c1b2009-01-01 10:12:22 +10305146out_free_cpumask:
5147 free_cpumask_var(tracing_cpumask);
5148out_free_buffer_mask:
5149 free_cpumask_var(tracing_buffer_mask);
5150out:
5151 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005152}
Steven Rostedtb2821ae2009-02-02 21:38:32 -05005153
5154__init static int clear_boot_tracer(void)
5155{
5156 /*
5157 * The default tracer at boot buffer is an init section.
5158 * This function is called in lateinit. If we did not
5159 * find the boot tracer, then clear it out, to prevent
5160 * later registration from accessing the buffer that is
5161 * about to be freed.
5162 */
5163 if (!default_bootup_tracer)
5164 return 0;
5165
5166 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
5167 default_bootup_tracer);
5168 default_bootup_tracer = NULL;
5169
5170 return 0;
5171}
5172
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01005173early_initcall(tracer_alloc_buffers);
5174fs_initcall(tracer_init_debugfs);
Steven Rostedtb2821ae2009-02-02 21:38:32 -05005175late_initcall(clear_boot_tracer);