blob: cfff63c2148af0a65294f5593f970bd1e0c6eeb8 [file] [log] [blame]
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001/*
2 * ring buffer based function tracer
3 *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +010012 * Copyright (C) 2004 Nadia Yvette Chambers
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020013 */
Steven Rostedt2cadf912008-12-01 22:20:19 -050014#include <linux/ring_buffer.h>
Sam Ravnborg273b2812009-10-18 00:52:28 +020015#include <generated/utsrelease.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050016#include <linux/stacktrace.h>
17#include <linux/writeback.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020018#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040020#include <linux/notifier.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050021#include <linux/irqflags.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020022#include <linux/debugfs.h>
Steven Rostedt4c11d7a2008-05-12 21:20:43 +020023#include <linux/pagemap.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020024#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050027#include <linux/kprobes.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020028#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050031#include <linux/splice.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040032#include <linux/kdebug.h>
Frederic Weisbecker5f0c6c02009-03-27 14:22:10 +010033#include <linux/string.h>
Lai Jiangshan7e53bd42010-01-06 20:08:50 +080034#include <linux/rwsem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090035#include <linux/slab.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020036#include <linux/ctype.h>
37#include <linux/init.h>
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +020038#include <linux/poll.h>
Steven Rostedtb892e5c2012-03-01 22:06:48 -050039#include <linux/nmi.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020040#include <linux/fs.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060041#include <linux/sched/rt.h>
Ingo Molnar86387f72008-05-12 21:20:51 +020042
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020043#include "trace.h"
Steven Rostedtf0868d12008-12-23 23:24:12 -050044#include "trace_output.h"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020045
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010046/*
Steven Rostedt73c51622009-03-11 13:42:01 -040047 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
49 */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -050050bool ring_buffer_expanded;
Steven Rostedt73c51622009-03-11 13:42:01 -040051
52/*
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010053 * We need to change this state when a selftest is running.
Frederic Weisbeckerff325042008-12-04 23:47:35 +010054 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
Ingo Molnar5e1607a2009-03-05 10:24:48 +010056 * insertions into the ring-buffer such as trace_printk could occurred
Frederic Weisbeckerff325042008-12-04 23:47:35 +010057 * at the same time, giving false positive or negative results.
58 */
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010059static bool __read_mostly tracing_selftest_running;
Frederic Weisbeckerff325042008-12-04 23:47:35 +010060
Steven Rostedtb2821ae2009-02-02 21:38:32 -050061/*
62 * If a tracer is running, we do not want to run SELFTEST.
63 */
Li Zefan020e5f82009-07-01 10:47:05 +080064bool __read_mostly tracing_selftest_disabled;
Steven Rostedtb2821ae2009-02-02 21:38:32 -050065
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010066/* For tracers that don't implement custom flags */
67static struct tracer_opt dummy_tracer_opt[] = {
68 { }
69};
70
71static struct tracer_flags dummy_tracer_flags = {
72 .val = 0,
73 .opts = dummy_tracer_opt
74};
75
76static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77{
78 return 0;
79}
Steven Rostedt0f048702008-11-05 16:05:44 -050080
81/*
Steven Rostedt7ffbd482012-10-11 12:14:25 -040082 * To prevent the comm cache from being overwritten when no
83 * tracing is active, only save the comm when a trace event
84 * occurred.
85 */
86static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88/*
Steven Rostedt0f048702008-11-05 16:05:44 -050089 * Kill all tracing for good (never come back).
90 * It is initialized to 1 but will turn to zero if the initialization
91 * of the tracer is successful. But that is the only place that sets
92 * this back to zero.
93 */
Hannes Eder4fd27352009-02-10 19:44:12 +010094static int tracing_disabled = 1;
Steven Rostedt0f048702008-11-05 16:05:44 -050095
Christoph Lameter9288f992009-10-07 19:17:45 -040096DEFINE_PER_CPU(int, ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -040097
Jason Wessel955b61e2010-08-05 09:22:23 -050098cpumask_var_t __read_mostly tracing_buffer_mask;
Steven Rostedtab464282008-05-12 21:21:00 +020099
Steven Rostedt944ac422008-10-23 19:26:08 -0400100/*
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102 *
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
107 * serial console.
108 *
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
Steven Rostedt944ac422008-10-23 19:26:08 -0400114 */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200115
116enum ftrace_dump_mode ftrace_dump_on_oops;
Steven Rostedt944ac422008-10-23 19:26:08 -0400117
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400118/* When set, tracing will stop when a WARN*() is hit */
119int __disable_trace_on_warning;
120
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500121static int tracing_set_tracer(const char *buf);
122
Li Zefanee6c2c12009-09-18 14:06:47 +0800123#define MAX_TRACER_SIZE 100
124static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500125static char *default_bootup_tracer;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100126
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500127static bool allocate_snapshot;
128
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200129static int __init set_cmdline_ftrace(char *str)
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100130{
Chen Gang67012ab2013-04-08 12:06:44 +0800131 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500132 default_bootup_tracer = bootup_tracer_buf;
Steven Rostedt73c51622009-03-11 13:42:01 -0400133 /* We are using ftrace early, expand it */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500134 ring_buffer_expanded = true;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100135 return 1;
136}
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200137__setup("ftrace=", set_cmdline_ftrace);
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100138
Steven Rostedt944ac422008-10-23 19:26:08 -0400139static int __init set_ftrace_dump_on_oops(char *str)
140{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200141 if (*str++ != '=' || !*str) {
142 ftrace_dump_on_oops = DUMP_ALL;
143 return 1;
144 }
145
146 if (!strcmp("orig_cpu", str)) {
147 ftrace_dump_on_oops = DUMP_ORIG;
148 return 1;
149 }
150
151 return 0;
Steven Rostedt944ac422008-10-23 19:26:08 -0400152}
153__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
Steven Rostedt60a11772008-05-12 21:20:44 +0200154
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400155static int __init stop_trace_on_warning(char *str)
156{
157 __disable_trace_on_warning = 1;
158 return 1;
159}
160__setup("traceoff_on_warning=", stop_trace_on_warning);
161
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400162static int __init boot_alloc_snapshot(char *str)
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500163{
164 allocate_snapshot = true;
165 /* We also need the main ring buffer expanded */
166 ring_buffer_expanded = true;
167 return 1;
168}
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400169__setup("alloc_snapshot", boot_alloc_snapshot);
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500170
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400171
172static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
173static char *trace_boot_options __initdata;
174
175static int __init set_trace_boot_options(char *str)
176{
Chen Gang67012ab2013-04-08 12:06:44 +0800177 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400178 trace_boot_options = trace_boot_options_buf;
179 return 0;
180}
181__setup("trace_options=", set_trace_boot_options);
182
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400183
Lai Jiangshancf8e3472009-03-30 13:48:00 +0800184unsigned long long ns2usecs(cycle_t nsec)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200185{
186 nsec += 500;
187 do_div(nsec, 1000);
188 return nsec;
189}
190
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200191/*
192 * The global_trace is the descriptor that holds the tracing
193 * buffers for the live tracing. For each CPU, it contains
194 * a link list of pages that will store trace entries. The
195 * page descriptor of the pages in the memory is used to hold
196 * the link list by linking the lru item in the page descriptor
197 * to each of the pages in the buffer per CPU.
198 *
199 * For each active CPU there is a data field that holds the
200 * pages for the buffer for that CPU. Each CPU has the same number
201 * of pages allocated for its buffer.
202 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200203static struct trace_array global_trace;
204
Steven Rostedtae63b312012-05-03 23:09:03 -0400205LIST_HEAD(ftrace_trace_arrays);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200206
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -0400207int trace_array_get(struct trace_array *this_tr)
208{
209 struct trace_array *tr;
210 int ret = -ENODEV;
211
212 mutex_lock(&trace_types_lock);
213 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
214 if (tr == this_tr) {
215 tr->ref++;
216 ret = 0;
217 break;
218 }
219 }
220 mutex_unlock(&trace_types_lock);
221
222 return ret;
223}
224
225static void __trace_array_put(struct trace_array *this_tr)
226{
227 WARN_ON(!this_tr->ref);
228 this_tr->ref--;
229}
230
231void trace_array_put(struct trace_array *this_tr)
232{
233 mutex_lock(&trace_types_lock);
234 __trace_array_put(this_tr);
235 mutex_unlock(&trace_types_lock);
236}
237
Steven Rostedte77405a2009-09-02 14:17:06 -0400238int filter_current_check_discard(struct ring_buffer *buffer,
239 struct ftrace_event_call *call, void *rec,
Tom Zanussieb02ce02009-04-08 03:15:54 -0500240 struct ring_buffer_event *event)
241{
Steven Rostedte77405a2009-09-02 14:17:06 -0400242 return filter_check_discard(call, rec, buffer, event);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500243}
Steven Rostedt17c873e2009-04-10 18:12:50 -0400244EXPORT_SYMBOL_GPL(filter_current_check_discard);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500245
Steven Rostedt37886f62009-03-17 17:22:06 -0400246cycle_t ftrace_now(int cpu)
247{
248 u64 ts;
249
250 /* Early boot up does not have a buffer yet */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500251 if (!global_trace.trace_buffer.buffer)
Steven Rostedt37886f62009-03-17 17:22:06 -0400252 return trace_clock_local();
253
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500254 ts = ring_buffer_time_stamp(global_trace.trace_buffer.buffer, cpu);
255 ring_buffer_normalize_time_stamp(global_trace.trace_buffer.buffer, cpu, &ts);
Steven Rostedt37886f62009-03-17 17:22:06 -0400256
257 return ts;
258}
259
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400260/**
261 * tracing_is_enabled - Show if global_trace has been disabled
262 *
263 * Shows if the global trace has been enabled or not. It uses the
264 * mirror flag "buffer_disabled" to be used in fast paths such as for
265 * the irqsoff tracer. But it may be inaccurate due to races. If you
266 * need to know the accurate state, use tracing_is_on() which is a little
267 * slower, but accurate.
268 */
Steven Rostedt90369902008-11-05 16:05:44 -0500269int tracing_is_enabled(void)
270{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400271 /*
272 * For quick access (irqsoff uses this in fast path), just
273 * return the mirror variable of the state of the ring buffer.
274 * It's a little racy, but we don't really care.
275 */
276 smp_rmb();
277 return !global_trace.buffer_disabled;
Steven Rostedt90369902008-11-05 16:05:44 -0500278}
279
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200280/*
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400281 * trace_buf_size is the size in bytes that is allocated
282 * for a buffer. Note, the number of bytes is always rounded
283 * to page size.
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400284 *
285 * This number is purposely set to a low number of 16384.
286 * If the dump on oops happens, it will be much appreciated
287 * to not have to wait for all that output. Anyway this can be
288 * boot time and run time configurable.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200289 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400290#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400291
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400292static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200293
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200294/* trace_types holds a link list of available tracers. */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200295static struct tracer *trace_types __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200296
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200297/*
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200298 * trace_types_lock is used to protect the trace_types list.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200299 */
Alexander Z Lama8227412013-07-01 19:37:54 -0700300DEFINE_MUTEX(trace_types_lock);
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200301
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800302/*
303 * serialize the access of the ring buffer
304 *
305 * ring buffer serializes readers, but it is low level protection.
306 * The validity of the events (which returns by ring_buffer_peek() ..etc)
307 * are not protected by ring buffer.
308 *
309 * The content of events may become garbage if we allow other process consumes
310 * these events concurrently:
311 * A) the page of the consumed events may become a normal page
312 * (not reader page) in ring buffer, and this page will be rewrited
313 * by events producer.
314 * B) The page of the consumed events may become a page for splice_read,
315 * and this page will be returned to system.
316 *
317 * These primitives allow multi process access to different cpu ring buffer
318 * concurrently.
319 *
320 * These primitives don't distinguish read-only and read-consume access.
321 * Multi read-only access are also serialized.
322 */
323
324#ifdef CONFIG_SMP
325static DECLARE_RWSEM(all_cpu_access_lock);
326static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
327
328static inline void trace_access_lock(int cpu)
329{
Steven Rostedtae3b5092013-01-23 15:22:59 -0500330 if (cpu == RING_BUFFER_ALL_CPUS) {
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800331 /* gain it for accessing the whole ring buffer. */
332 down_write(&all_cpu_access_lock);
333 } else {
334 /* gain it for accessing a cpu ring buffer. */
335
Steven Rostedtae3b5092013-01-23 15:22:59 -0500336 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800337 down_read(&all_cpu_access_lock);
338
339 /* Secondly block other access to this @cpu ring buffer. */
340 mutex_lock(&per_cpu(cpu_access_lock, cpu));
341 }
342}
343
344static inline void trace_access_unlock(int cpu)
345{
Steven Rostedtae3b5092013-01-23 15:22:59 -0500346 if (cpu == RING_BUFFER_ALL_CPUS) {
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800347 up_write(&all_cpu_access_lock);
348 } else {
349 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
350 up_read(&all_cpu_access_lock);
351 }
352}
353
354static inline void trace_access_lock_init(void)
355{
356 int cpu;
357
358 for_each_possible_cpu(cpu)
359 mutex_init(&per_cpu(cpu_access_lock, cpu));
360}
361
362#else
363
364static DEFINE_MUTEX(access_lock);
365
366static inline void trace_access_lock(int cpu)
367{
368 (void)cpu;
369 mutex_lock(&access_lock);
370}
371
372static inline void trace_access_unlock(int cpu)
373{
374 (void)cpu;
375 mutex_unlock(&access_lock);
376}
377
378static inline void trace_access_lock_init(void)
379{
380}
381
382#endif
383
Steven Rostedtee6bce52008-11-12 17:52:37 -0500384/* trace_flags holds trace_options default values */
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500385unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
Steven Rostedta2a16d62009-03-24 23:17:58 -0400386 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
Steven Rostedt77271ce2011-11-17 09:34:33 -0500387 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
Steven Rostedt (Red Hat)328df472013-03-14 12:10:40 -0400388 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
Vaibhav Nagarnaike7e2ee82011-05-10 13:27:21 -0700389
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400390static void tracer_tracing_on(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400391{
392 if (tr->trace_buffer.buffer)
393 ring_buffer_record_on(tr->trace_buffer.buffer);
394 /*
395 * This flag is looked at when buffers haven't been allocated
396 * yet, or by some tracers (like irqsoff), that just want to
397 * know if the ring buffer has been disabled, but it can handle
398 * races of where it gets disabled but we still do a record.
399 * As the check is in the fast path of the tracers, it is more
400 * important to be fast than accurate.
401 */
402 tr->buffer_disabled = 0;
403 /* Make the flag seen by readers */
404 smp_wmb();
405}
406
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200407/**
Steven Rostedt499e5472012-02-22 15:50:28 -0500408 * tracing_on - enable tracing buffers
409 *
410 * This function enables tracing buffers that may have been
411 * disabled with tracing_off.
412 */
413void tracing_on(void)
414{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400415 tracer_tracing_on(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500416}
417EXPORT_SYMBOL_GPL(tracing_on);
418
419/**
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500420 * __trace_puts - write a constant string into the trace buffer.
421 * @ip: The address of the caller
422 * @str: The constant string to write
423 * @size: The size of the string.
424 */
425int __trace_puts(unsigned long ip, const char *str, int size)
426{
427 struct ring_buffer_event *event;
428 struct ring_buffer *buffer;
429 struct print_entry *entry;
430 unsigned long irq_flags;
431 int alloc;
432
433 alloc = sizeof(*entry) + size + 2; /* possible \n added */
434
435 local_save_flags(irq_flags);
436 buffer = global_trace.trace_buffer.buffer;
437 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
438 irq_flags, preempt_count());
439 if (!event)
440 return 0;
441
442 entry = ring_buffer_event_data(event);
443 entry->ip = ip;
444
445 memcpy(&entry->buf, str, size);
446
447 /* Add a newline if necessary */
448 if (entry->buf[size - 1] != '\n') {
449 entry->buf[size] = '\n';
450 entry->buf[size + 1] = '\0';
451 } else
452 entry->buf[size] = '\0';
453
454 __buffer_unlock_commit(buffer, event);
455
456 return size;
457}
458EXPORT_SYMBOL_GPL(__trace_puts);
459
460/**
461 * __trace_bputs - write the pointer to a constant string into trace buffer
462 * @ip: The address of the caller
463 * @str: The constant string to write to the buffer to
464 */
465int __trace_bputs(unsigned long ip, const char *str)
466{
467 struct ring_buffer_event *event;
468 struct ring_buffer *buffer;
469 struct bputs_entry *entry;
470 unsigned long irq_flags;
471 int size = sizeof(struct bputs_entry);
472
473 local_save_flags(irq_flags);
474 buffer = global_trace.trace_buffer.buffer;
475 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
476 irq_flags, preempt_count());
477 if (!event)
478 return 0;
479
480 entry = ring_buffer_event_data(event);
481 entry->ip = ip;
482 entry->str = str;
483
484 __buffer_unlock_commit(buffer, event);
485
486 return 1;
487}
488EXPORT_SYMBOL_GPL(__trace_bputs);
489
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500490#ifdef CONFIG_TRACER_SNAPSHOT
491/**
492 * trace_snapshot - take a snapshot of the current buffer.
493 *
494 * This causes a swap between the snapshot buffer and the current live
495 * tracing buffer. You can use this to take snapshots of the live
496 * trace when some condition is triggered, but continue to trace.
497 *
498 * Note, make sure to allocate the snapshot with either
499 * a tracing_snapshot_alloc(), or by doing it manually
500 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
501 *
502 * If the snapshot buffer is not allocated, it will stop tracing.
503 * Basically making a permanent snapshot.
504 */
505void tracing_snapshot(void)
506{
507 struct trace_array *tr = &global_trace;
508 struct tracer *tracer = tr->current_trace;
509 unsigned long flags;
510
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500511 if (in_nmi()) {
512 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
513 internal_trace_puts("*** snapshot is being ignored ***\n");
514 return;
515 }
516
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500517 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)ca268da2013-03-09 00:40:58 -0500518 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
519 internal_trace_puts("*** stopping trace here! ***\n");
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500520 tracing_off();
521 return;
522 }
523
524 /* Note, snapshot can not be used when the tracer uses it */
525 if (tracer->use_max_tr) {
Steven Rostedt (Red Hat)ca268da2013-03-09 00:40:58 -0500526 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
527 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500528 return;
529 }
530
531 local_irq_save(flags);
532 update_max_tr(tr, current, smp_processor_id());
533 local_irq_restore(flags);
534}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500535EXPORT_SYMBOL_GPL(tracing_snapshot);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500536
537static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
538 struct trace_buffer *size_buf, int cpu_id);
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400539static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
540
541static int alloc_snapshot(struct trace_array *tr)
542{
543 int ret;
544
545 if (!tr->allocated_snapshot) {
546
547 /* allocate spare buffer */
548 ret = resize_buffer_duplicate_size(&tr->max_buffer,
549 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
550 if (ret < 0)
551 return ret;
552
553 tr->allocated_snapshot = true;
554 }
555
556 return 0;
557}
558
559void free_snapshot(struct trace_array *tr)
560{
561 /*
562 * We don't free the ring buffer. instead, resize it because
563 * The max_tr ring buffer has some state (e.g. ring->clock) and
564 * we want preserve it.
565 */
566 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
567 set_buffer_entries(&tr->max_buffer, 1);
568 tracing_reset_online_cpus(&tr->max_buffer);
569 tr->allocated_snapshot = false;
570}
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500571
572/**
573 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
574 *
575 * This is similar to trace_snapshot(), but it will allocate the
576 * snapshot buffer if it isn't already allocated. Use this only
577 * where it is safe to sleep, as the allocation may sleep.
578 *
579 * This causes a swap between the snapshot buffer and the current live
580 * tracing buffer. You can use this to take snapshots of the live
581 * trace when some condition is triggered, but continue to trace.
582 */
583void tracing_snapshot_alloc(void)
584{
585 struct trace_array *tr = &global_trace;
586 int ret;
587
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400588 ret = alloc_snapshot(tr);
589 if (WARN_ON(ret < 0))
590 return;
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500591
592 tracing_snapshot();
593}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500594EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500595#else
596void tracing_snapshot(void)
597{
598 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
599}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500600EXPORT_SYMBOL_GPL(tracing_snapshot);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500601void tracing_snapshot_alloc(void)
602{
603 /* Give warning */
604 tracing_snapshot();
605}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500606EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500607#endif /* CONFIG_TRACER_SNAPSHOT */
608
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400609static void tracer_tracing_off(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400610{
611 if (tr->trace_buffer.buffer)
612 ring_buffer_record_off(tr->trace_buffer.buffer);
613 /*
614 * This flag is looked at when buffers haven't been allocated
615 * yet, or by some tracers (like irqsoff), that just want to
616 * know if the ring buffer has been disabled, but it can handle
617 * races of where it gets disabled but we still do a record.
618 * As the check is in the fast path of the tracers, it is more
619 * important to be fast than accurate.
620 */
621 tr->buffer_disabled = 1;
622 /* Make the flag seen by readers */
623 smp_wmb();
624}
625
Steven Rostedt499e5472012-02-22 15:50:28 -0500626/**
627 * tracing_off - turn off tracing buffers
628 *
629 * This function stops the tracing buffers from recording data.
630 * It does not disable any overhead the tracers themselves may
631 * be causing. This function simply causes all recording to
632 * the ring buffers to fail.
633 */
634void tracing_off(void)
635{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400636 tracer_tracing_off(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500637}
638EXPORT_SYMBOL_GPL(tracing_off);
639
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400640void disable_trace_on_warning(void)
641{
642 if (__disable_trace_on_warning)
643 tracing_off();
644}
645
Steven Rostedt499e5472012-02-22 15:50:28 -0500646/**
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400647 * tracer_tracing_is_on - show real state of ring buffer enabled
648 * @tr : the trace array to know if ring buffer is enabled
649 *
650 * Shows real state of the ring buffer if it is enabled or not.
651 */
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400652static int tracer_tracing_is_on(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400653{
654 if (tr->trace_buffer.buffer)
655 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
656 return !tr->buffer_disabled;
657}
658
Steven Rostedt499e5472012-02-22 15:50:28 -0500659/**
660 * tracing_is_on - show state of ring buffers enabled
661 */
662int tracing_is_on(void)
663{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400664 return tracer_tracing_is_on(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500665}
666EXPORT_SYMBOL_GPL(tracing_is_on);
667
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400668static int __init set_buf_size(char *str)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200669{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400670 unsigned long buf_size;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200671
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200672 if (!str)
673 return 0;
Li Zefan9d612be2009-06-24 17:33:15 +0800674 buf_size = memparse(str, &str);
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200675 /* nr_entries can not be zero */
Li Zefan9d612be2009-06-24 17:33:15 +0800676 if (buf_size == 0)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200677 return 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400678 trace_buf_size = buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200679 return 1;
680}
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400681__setup("trace_buf_size=", set_buf_size);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200682
Tim Bird0e950172010-02-25 15:36:43 -0800683static int __init set_tracing_thresh(char *str)
684{
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800685 unsigned long threshold;
Tim Bird0e950172010-02-25 15:36:43 -0800686 int ret;
687
688 if (!str)
689 return 0;
Daniel Walterbcd83ea2012-09-26 22:08:38 +0200690 ret = kstrtoul(str, 0, &threshold);
Tim Bird0e950172010-02-25 15:36:43 -0800691 if (ret < 0)
692 return 0;
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800693 tracing_thresh = threshold * 1000;
Tim Bird0e950172010-02-25 15:36:43 -0800694 return 1;
695}
696__setup("tracing_thresh=", set_tracing_thresh);
697
Steven Rostedt57f50be2008-05-12 21:20:44 +0200698unsigned long nsecs_to_usecs(unsigned long nsecs)
699{
700 return nsecs / 1000;
701}
702
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200703/* These must match the bit postions in trace_iterator_flags */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200704static const char *trace_options[] = {
705 "print-parent",
706 "sym-offset",
707 "sym-addr",
708 "verbose",
Ingo Molnarf9896bf2008-05-12 21:20:47 +0200709 "raw",
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +0200710 "hex",
Ingo Molnarcb0f12a2008-05-12 21:20:47 +0200711 "bin",
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +0200712 "block",
Ingo Molnar86387f72008-05-12 21:20:51 +0200713 "stacktrace",
Ingo Molnar5e1607a2009-03-05 10:24:48 +0100714 "trace_printk",
Steven Rostedtb2a866f2008-11-03 23:15:57 -0500715 "ftrace_preempt",
Steven Rostedt9f029e82008-11-12 15:24:24 -0500716 "branch",
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500717 "annotate",
Török Edwin02b67512008-11-22 13:28:47 +0200718 "userstacktrace",
Török Edwinb54d3de2008-11-22 13:28:48 +0200719 "sym-userobj",
Frederic Weisbecker66896a82008-12-13 20:18:13 +0100720 "printk-msg-only",
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -0200721 "context-info",
Steven Rostedtc032ef642009-03-04 20:34:24 -0500722 "latency-format",
Steven Rostedtbe6f1642009-03-24 11:06:24 -0400723 "sleep-time",
Steven Rostedta2a16d62009-03-24 23:17:58 -0400724 "graph-time",
Li Zefane870e9a2010-07-02 11:07:32 +0800725 "record-cmd",
David Sharp750912f2010-12-08 13:46:47 -0800726 "overwrite",
Steven Rostedtcf30cf62011-06-14 22:44:07 -0400727 "disable_on_free",
Steven Rostedt77271ce2011-11-17 09:34:33 -0500728 "irq-info",
Mandeep Singh Baines5224c3a2012-09-07 18:12:19 -0700729 "markers",
Steven Rostedt (Red Hat)328df472013-03-14 12:10:40 -0400730 "function-trace",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200731 NULL
732};
733
Zhaolei5079f322009-08-25 16:12:56 +0800734static struct {
735 u64 (*func)(void);
736 const char *name;
David Sharp8be07092012-11-13 12:18:22 -0800737 int in_ns; /* is this clock in nanoseconds? */
Zhaolei5079f322009-08-25 16:12:56 +0800738} trace_clocks[] = {
David Sharp8be07092012-11-13 12:18:22 -0800739 { trace_clock_local, "local", 1 },
740 { trace_clock_global, "global", 1 },
741 { trace_clock_counter, "counter", 0 },
Steven Rostedt (Red Hat)8aacf012013-03-14 13:13:45 -0400742 { trace_clock_jiffies, "uptime", 1 },
Steven Rostedt (Red Hat)76f11912013-03-14 17:53:25 -0400743 { trace_clock, "perf", 1 },
David Sharp8cbd9cc2012-11-13 12:18:21 -0800744 ARCH_TRACE_CLOCKS
Zhaolei5079f322009-08-25 16:12:56 +0800745};
746
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200747/*
748 * trace_parser_get_init - gets the buffer for trace parser
749 */
750int trace_parser_get_init(struct trace_parser *parser, int size)
751{
752 memset(parser, 0, sizeof(*parser));
753
754 parser->buffer = kmalloc(size, GFP_KERNEL);
755 if (!parser->buffer)
756 return 1;
757
758 parser->size = size;
759 return 0;
760}
761
762/*
763 * trace_parser_put - frees the buffer for trace parser
764 */
765void trace_parser_put(struct trace_parser *parser)
766{
767 kfree(parser->buffer);
768}
769
770/*
771 * trace_get_user - reads the user input string separated by space
772 * (matched by isspace(ch))
773 *
774 * For each string found the 'struct trace_parser' is updated,
775 * and the function returns.
776 *
777 * Returns number of bytes read.
778 *
779 * See kernel/trace/trace.h for 'struct trace_parser' details.
780 */
781int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
782 size_t cnt, loff_t *ppos)
783{
784 char ch;
785 size_t read = 0;
786 ssize_t ret;
787
788 if (!*ppos)
789 trace_parser_clear(parser);
790
791 ret = get_user(ch, ubuf++);
792 if (ret)
793 goto out;
794
795 read++;
796 cnt--;
797
798 /*
799 * The parser is not finished with the last write,
800 * continue reading the user input without skipping spaces.
801 */
802 if (!parser->cont) {
803 /* skip white space */
804 while (cnt && isspace(ch)) {
805 ret = get_user(ch, ubuf++);
806 if (ret)
807 goto out;
808 read++;
809 cnt--;
810 }
811
812 /* only spaces were written */
813 if (isspace(ch)) {
814 *ppos += read;
815 ret = read;
816 goto out;
817 }
818
819 parser->idx = 0;
820 }
821
822 /* read the non-space input */
823 while (cnt && !isspace(ch)) {
Li Zefan3c235a32009-09-22 13:51:54 +0800824 if (parser->idx < parser->size - 1)
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200825 parser->buffer[parser->idx++] = ch;
826 else {
827 ret = -EINVAL;
828 goto out;
829 }
830 ret = get_user(ch, ubuf++);
831 if (ret)
832 goto out;
833 read++;
834 cnt--;
835 }
836
837 /* We either got finished input or we have to wait for another call. */
838 if (isspace(ch)) {
839 parser->buffer[parser->idx] = 0;
840 parser->cont = false;
841 } else {
842 parser->cont = true;
843 parser->buffer[parser->idx++] = ch;
844 }
845
846 *ppos += read;
847 ret = read;
848
849out:
850 return ret;
851}
852
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200853ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
854{
855 int len;
856 int ret;
857
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500858 if (!cnt)
859 return 0;
860
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200861 if (s->len <= s->readpos)
862 return -EBUSY;
863
864 len = s->len - s->readpos;
865 if (cnt > len)
866 cnt = len;
867 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500868 if (ret == cnt)
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200869 return -EFAULT;
870
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500871 cnt -= ret;
872
Steven Rostedte74da522009-03-04 20:31:11 -0500873 s->readpos += cnt;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200874 return cnt;
Steven Rostedt214023c2008-05-12 21:20:46 +0200875}
876
Dmitri Vorobievb8b94262009-03-22 19:11:11 +0200877static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200878{
879 int len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200880
881 if (s->len <= s->readpos)
882 return -EBUSY;
883
884 len = s->len - s->readpos;
885 if (cnt > len)
886 cnt = len;
Dan Carpenter5a26c8f2012-04-20 09:31:45 +0300887 memcpy(buf, s->buffer + s->readpos, cnt);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200888
Steven Rostedte74da522009-03-04 20:31:11 -0500889 s->readpos += cnt;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200890 return cnt;
891}
892
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400893/*
894 * ftrace_max_lock is used to protect the swapping of buffers
895 * when taking a max snapshot. The buffers themselves are
896 * protected by per_cpu spinlocks. But the action of the swap
897 * needs its own lock.
898 *
Thomas Gleixner445c8952009-12-02 19:49:50 +0100899 * This is defined as a arch_spinlock_t in order to help
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400900 * with performance when lockdep debugging is enabled.
901 *
902 * It is also used in other places outside the update_max_tr
903 * so it needs to be defined outside of the
904 * CONFIG_TRACER_MAX_TRACE.
905 */
Thomas Gleixner445c8952009-12-02 19:49:50 +0100906static arch_spinlock_t ftrace_max_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100907 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400908
Tim Bird0e950172010-02-25 15:36:43 -0800909unsigned long __read_mostly tracing_thresh;
910
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400911#ifdef CONFIG_TRACER_MAX_TRACE
912unsigned long __read_mostly tracing_max_latency;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400913
914/*
915 * Copy the new maximum trace into the separate maximum-trace
916 * structure. (this way the maximum trace is permanently saved,
917 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
918 */
919static void
920__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
921{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500922 struct trace_buffer *trace_buf = &tr->trace_buffer;
923 struct trace_buffer *max_buf = &tr->max_buffer;
924 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
925 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400926
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500927 max_buf->cpu = cpu;
928 max_buf->time_start = data->preempt_timestamp;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400929
Steven Rostedt8248ac02009-09-02 12:27:41 -0400930 max_data->saved_latency = tracing_max_latency;
931 max_data->critical_start = data->critical_start;
932 max_data->critical_end = data->critical_end;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400933
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300934 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
Steven Rostedt8248ac02009-09-02 12:27:41 -0400935 max_data->pid = tsk->pid;
Steven Rostedt (Red Hat)f17a5192013-05-30 21:10:37 -0400936 /*
937 * If tsk == current, then use current_uid(), as that does not use
938 * RCU. The irq tracer can be called out of RCU scope.
939 */
940 if (tsk == current)
941 max_data->uid = current_uid();
942 else
943 max_data->uid = task_uid(tsk);
944
Steven Rostedt8248ac02009-09-02 12:27:41 -0400945 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
946 max_data->policy = tsk->policy;
947 max_data->rt_priority = tsk->rt_priority;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400948
949 /* record this tasks comm */
950 tracing_record_cmdline(tsk);
951}
952
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200953/**
954 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
955 * @tr: tracer
956 * @tsk: the task with the latency
957 * @cpu: The cpu that initiated the trace.
958 *
959 * Flip the buffers between the @tr and the max_tr and record information
960 * about which task was the cause of this latency.
961 */
Ingo Molnare309b412008-05-12 21:20:51 +0200962void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200963update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
964{
Steven Rostedt (Red Hat)2721e722013-03-12 11:32:32 -0400965 struct ring_buffer *buf;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200966
Steven Rostedt2b6080f2012-05-11 13:29:49 -0400967 if (tr->stop_count)
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400968 return;
969
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200970 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt34600f02013-01-22 13:35:11 -0500971
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -0500972 if (!tr->allocated_snapshot) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +0900973 /* Only the nop tracer should hit this when disabling */
Steven Rostedt2b6080f2012-05-11 13:29:49 -0400974 WARN_ON_ONCE(tr->current_trace != &nop_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900975 return;
Hiraku Toyookadebdd572012-12-26 11:53:00 +0900976 }
Steven Rostedt34600f02013-01-22 13:35:11 -0500977
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100978 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400979
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500980 buf = tr->trace_buffer.buffer;
981 tr->trace_buffer.buffer = tr->max_buffer.buffer;
982 tr->max_buffer.buffer = buf;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400983
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200984 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100985 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200986}
987
988/**
989 * update_max_tr_single - only copy one trace over, and reset the rest
990 * @tr - tracer
991 * @tsk - task with the latency
992 * @cpu - the cpu of the buffer to copy.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200993 *
994 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200995 */
Ingo Molnare309b412008-05-12 21:20:51 +0200996void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200997update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
998{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400999 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001000
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001001 if (tr->stop_count)
Steven Rostedtb8de7bd12009-08-31 22:32:27 -04001002 return;
1003
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02001004 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt6c244992013-04-29 20:08:14 -04001005 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)2930e042013-03-26 17:33:00 -04001006 /* Only the nop tracer should hit this when disabling */
Linus Torvalds9e8529a2013-04-29 13:55:38 -07001007 WARN_ON_ONCE(tr->current_trace != &nop_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001008 return;
Steven Rostedt (Red Hat)2930e042013-03-26 17:33:00 -04001009 }
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001010
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001011 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001012
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001013 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001014
Steven Rostedte8165db2009-09-03 19:13:05 -04001015 if (ret == -EBUSY) {
1016 /*
1017 * We failed to swap the buffer due to a commit taking
1018 * place on this CPU. We fail to record, but we reset
1019 * the max trace buffer (no one writes directly to it)
1020 * and flag that it failed.
1021 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001022 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
Steven Rostedte8165db2009-09-03 19:13:05 -04001023 "Failed to swap buffers due to commit in progress\n");
1024 }
1025
Steven Rostedte8165db2009-09-03 19:13:05 -04001026 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001027
1028 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001029 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001030}
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001031#endif /* CONFIG_TRACER_MAX_TRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001032
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001033static void default_wait_pipe(struct trace_iterator *iter)
1034{
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05001035 /* Iterators are static, they should be filled or empty */
1036 if (trace_buffer_iter(iter, iter->cpu_file))
1037 return;
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001038
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001039 ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001040}
1041
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001042#ifdef CONFIG_FTRACE_STARTUP_TEST
1043static int run_tracer_selftest(struct tracer *type)
1044{
1045 struct trace_array *tr = &global_trace;
1046 struct tracer *saved_tracer = tr->current_trace;
1047 int ret;
1048
1049 if (!type->selftest || tracing_selftest_disabled)
1050 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001051
1052 /*
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001053 * Run a selftest on this tracer.
1054 * Here we reset the trace buffer, and set the current
1055 * tracer to be this tracer. The tracer can then run some
1056 * internal tracing to verify that everything is in order.
1057 * If we fail, we do not register this tracer.
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001058 */
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001059 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001060
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001061 tr->current_trace = type;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001062
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001063#ifdef CONFIG_TRACER_MAX_TRACE
1064 if (type->use_max_tr) {
1065 /* If we expanded the buffers, make sure the max is expanded too */
1066 if (ring_buffer_expanded)
1067 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1068 RING_BUFFER_ALL_CPUS);
1069 tr->allocated_snapshot = true;
1070 }
1071#endif
1072
1073 /* the test is responsible for initializing and enabling */
1074 pr_info("Testing tracer %s: ", type->name);
1075 ret = type->selftest(type, tr);
1076 /* the test is responsible for resetting too */
1077 tr->current_trace = saved_tracer;
1078 if (ret) {
1079 printk(KERN_CONT "FAILED!\n");
1080 /* Add the warning after printing 'FAILED' */
1081 WARN_ON(1);
1082 return -1;
1083 }
1084 /* Only reset on passing, to avoid touching corrupted buffers */
1085 tracing_reset_online_cpus(&tr->trace_buffer);
1086
1087#ifdef CONFIG_TRACER_MAX_TRACE
1088 if (type->use_max_tr) {
1089 tr->allocated_snapshot = false;
1090
1091 /* Shrink the max buffer again */
1092 if (ring_buffer_expanded)
1093 ring_buffer_resize(tr->max_buffer.buffer, 1,
1094 RING_BUFFER_ALL_CPUS);
1095 }
1096#endif
1097
1098 printk(KERN_CONT "PASSED\n");
1099 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001100}
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001101#else
1102static inline int run_tracer_selftest(struct tracer *type)
1103{
1104 return 0;
1105}
1106#endif /* CONFIG_FTRACE_STARTUP_TEST */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001107
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001108/**
1109 * register_tracer - register a tracer with the ftrace system.
1110 * @type - the plugin for the tracer
1111 *
1112 * Register a new plugin tracer.
1113 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001114int register_tracer(struct tracer *type)
1115{
1116 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001117 int ret = 0;
1118
1119 if (!type->name) {
1120 pr_info("Tracer must have a name\n");
1121 return -1;
1122 }
1123
Dan Carpenter24a461d2010-07-10 12:06:44 +02001124 if (strlen(type->name) >= MAX_TRACER_SIZE) {
Li Zefanee6c2c12009-09-18 14:06:47 +08001125 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1126 return -1;
1127 }
1128
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001129 mutex_lock(&trace_types_lock);
Ingo Molnar86fa2f62008-11-19 10:00:15 +01001130
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +01001131 tracing_selftest_running = true;
1132
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001133 for (t = trace_types; t; t = t->next) {
1134 if (strcmp(type->name, t->name) == 0) {
1135 /* already found */
Li Zefanee6c2c12009-09-18 14:06:47 +08001136 pr_info("Tracer %s already registered\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001137 type->name);
1138 ret = -1;
1139 goto out;
1140 }
1141 }
1142
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01001143 if (!type->set_flag)
1144 type->set_flag = &dummy_set_flag;
1145 if (!type->flags)
1146 type->flags = &dummy_tracer_flags;
1147 else
1148 if (!type->flags->opts)
1149 type->flags->opts = dummy_tracer_opt;
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01001150 if (!type->wait_pipe)
1151 type->wait_pipe = default_wait_pipe;
1152
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001153 ret = run_tracer_selftest(type);
1154 if (ret < 0)
1155 goto out;
Steven Rostedt60a11772008-05-12 21:20:44 +02001156
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001157 type->next = trace_types;
1158 trace_types = type;
Steven Rostedt60a11772008-05-12 21:20:44 +02001159
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001160 out:
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +01001161 tracing_selftest_running = false;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001162 mutex_unlock(&trace_types_lock);
1163
Steven Rostedtdac74942009-02-05 01:13:38 -05001164 if (ret || !default_bootup_tracer)
1165 goto out_unlock;
Steven Rostedtb2821ae2009-02-02 21:38:32 -05001166
Li Zefanee6c2c12009-09-18 14:06:47 +08001167 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
Steven Rostedtdac74942009-02-05 01:13:38 -05001168 goto out_unlock;
1169
1170 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1171 /* Do we want this tracer to start on bootup? */
1172 tracing_set_tracer(type->name);
1173 default_bootup_tracer = NULL;
1174 /* disable other selftests, since this will break it. */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05001175 tracing_selftest_disabled = true;
Steven Rostedtdac74942009-02-05 01:13:38 -05001176#ifdef CONFIG_FTRACE_STARTUP_TEST
1177 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1178 type->name);
1179#endif
1180
1181 out_unlock:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001182 return ret;
1183}
1184
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001185void tracing_reset(struct trace_buffer *buf, int cpu)
Steven Rostedtf6339032009-09-04 12:35:16 -04001186{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001187 struct ring_buffer *buffer = buf->buffer;
Steven Rostedtf6339032009-09-04 12:35:16 -04001188
Hiraku Toyookaa5416412012-12-19 16:02:34 +09001189 if (!buffer)
1190 return;
1191
Steven Rostedtf6339032009-09-04 12:35:16 -04001192 ring_buffer_record_disable(buffer);
1193
1194 /* Make sure all commits have finished */
1195 synchronize_sched();
Steven Rostedt68179682012-05-08 20:57:53 -04001196 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedtf6339032009-09-04 12:35:16 -04001197
1198 ring_buffer_record_enable(buffer);
1199}
1200
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001201void tracing_reset_online_cpus(struct trace_buffer *buf)
Pekka J Enberg213cc062008-12-19 12:08:39 +02001202{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001203 struct ring_buffer *buffer = buf->buffer;
Pekka J Enberg213cc062008-12-19 12:08:39 +02001204 int cpu;
1205
Hiraku Toyookaa5416412012-12-19 16:02:34 +09001206 if (!buffer)
1207 return;
1208
Steven Rostedt621968c2009-09-04 12:02:35 -04001209 ring_buffer_record_disable(buffer);
1210
1211 /* Make sure all commits have finished */
1212 synchronize_sched();
1213
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001214 buf->time_start = ftrace_now(buf->cpu);
Pekka J Enberg213cc062008-12-19 12:08:39 +02001215
1216 for_each_online_cpu(cpu)
Steven Rostedt68179682012-05-08 20:57:53 -04001217 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedt621968c2009-09-04 12:02:35 -04001218
1219 ring_buffer_record_enable(buffer);
Pekka J Enberg213cc062008-12-19 12:08:39 +02001220}
1221
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001222void tracing_reset_current(int cpu)
1223{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001224 tracing_reset(&global_trace.trace_buffer, cpu);
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001225}
1226
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001227void tracing_reset_all_online_cpus(void)
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001228{
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001229 struct trace_array *tr;
1230
1231 mutex_lock(&trace_types_lock);
1232 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001233 tracing_reset_online_cpus(&tr->trace_buffer);
1234#ifdef CONFIG_TRACER_MAX_TRACE
1235 tracing_reset_online_cpus(&tr->max_buffer);
1236#endif
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001237 }
1238 mutex_unlock(&trace_types_lock);
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001239}
1240
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001241#define SAVED_CMDLINES 128
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001242#define NO_CMDLINE_MAP UINT_MAX
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001243static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1244static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1245static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1246static int cmdline_idx;
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001247static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt25b0b442008-05-12 21:21:00 +02001248
Steven Rostedt25b0b442008-05-12 21:21:00 +02001249/* temporary disable recording */
Hannes Eder4fd27352009-02-10 19:44:12 +01001250static atomic_t trace_record_cmdline_disabled __read_mostly;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001251
1252static void trace_init_cmdlines(void)
1253{
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001254 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1255 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001256 cmdline_idx = 0;
1257}
1258
Carsten Emdeb5130b12009-09-13 01:43:07 +02001259int is_tracing_stopped(void)
1260{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001261 return global_trace.stop_count;
Carsten Emdeb5130b12009-09-13 01:43:07 +02001262}
1263
Steven Rostedt0f048702008-11-05 16:05:44 -05001264/**
Steven Rostedt69bb54e2008-11-21 12:59:38 -05001265 * ftrace_off_permanent - disable all ftrace code permanently
1266 *
1267 * This should only be called when a serious anomally has
1268 * been detected. This will turn off the function tracing,
1269 * ring buffers, and other tracing utilites. It takes no
1270 * locks and can be called from any context.
1271 */
1272void ftrace_off_permanent(void)
1273{
1274 tracing_disabled = 1;
1275 ftrace_stop();
1276 tracing_off_permanent();
1277}
1278
1279/**
Steven Rostedt0f048702008-11-05 16:05:44 -05001280 * tracing_start - quick start of the tracer
1281 *
1282 * If tracing is enabled but was stopped by tracing_stop,
1283 * this will start the tracer back up.
1284 */
1285void tracing_start(void)
1286{
1287 struct ring_buffer *buffer;
1288 unsigned long flags;
1289
1290 if (tracing_disabled)
1291 return;
1292
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001293 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1294 if (--global_trace.stop_count) {
1295 if (global_trace.stop_count < 0) {
Steven Rostedtb06a8302009-01-22 14:26:15 -05001296 /* Someone screwed up their debugging */
1297 WARN_ON_ONCE(1);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001298 global_trace.stop_count = 0;
Steven Rostedtb06a8302009-01-22 14:26:15 -05001299 }
Steven Rostedt0f048702008-11-05 16:05:44 -05001300 goto out;
1301 }
1302
Steven Rostedta2f80712010-03-12 19:56:00 -05001303 /* Prevent the buffers from switching */
1304 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt0f048702008-11-05 16:05:44 -05001305
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001306 buffer = global_trace.trace_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001307 if (buffer)
1308 ring_buffer_record_enable(buffer);
1309
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001310#ifdef CONFIG_TRACER_MAX_TRACE
1311 buffer = global_trace.max_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001312 if (buffer)
1313 ring_buffer_record_enable(buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001314#endif
Steven Rostedt0f048702008-11-05 16:05:44 -05001315
Steven Rostedta2f80712010-03-12 19:56:00 -05001316 arch_spin_unlock(&ftrace_max_lock);
1317
Steven Rostedt0f048702008-11-05 16:05:44 -05001318 ftrace_start();
1319 out:
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001320 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1321}
1322
1323static void tracing_start_tr(struct trace_array *tr)
1324{
1325 struct ring_buffer *buffer;
1326 unsigned long flags;
1327
1328 if (tracing_disabled)
1329 return;
1330
1331 /* If global, we need to also start the max tracer */
1332 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1333 return tracing_start();
1334
1335 raw_spin_lock_irqsave(&tr->start_lock, flags);
1336
1337 if (--tr->stop_count) {
1338 if (tr->stop_count < 0) {
1339 /* Someone screwed up their debugging */
1340 WARN_ON_ONCE(1);
1341 tr->stop_count = 0;
1342 }
1343 goto out;
1344 }
1345
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001346 buffer = tr->trace_buffer.buffer;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001347 if (buffer)
1348 ring_buffer_record_enable(buffer);
1349
1350 out:
1351 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001352}
1353
1354/**
1355 * tracing_stop - quick stop of the tracer
1356 *
1357 * Light weight way to stop tracing. Use in conjunction with
1358 * tracing_start.
1359 */
1360void tracing_stop(void)
1361{
1362 struct ring_buffer *buffer;
1363 unsigned long flags;
1364
1365 ftrace_stop();
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001366 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1367 if (global_trace.stop_count++)
Steven Rostedt0f048702008-11-05 16:05:44 -05001368 goto out;
1369
Steven Rostedta2f80712010-03-12 19:56:00 -05001370 /* Prevent the buffers from switching */
1371 arch_spin_lock(&ftrace_max_lock);
1372
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001373 buffer = global_trace.trace_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001374 if (buffer)
1375 ring_buffer_record_disable(buffer);
1376
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001377#ifdef CONFIG_TRACER_MAX_TRACE
1378 buffer = global_trace.max_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001379 if (buffer)
1380 ring_buffer_record_disable(buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001381#endif
Steven Rostedt0f048702008-11-05 16:05:44 -05001382
Steven Rostedta2f80712010-03-12 19:56:00 -05001383 arch_spin_unlock(&ftrace_max_lock);
1384
Steven Rostedt0f048702008-11-05 16:05:44 -05001385 out:
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001386 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1387}
1388
1389static void tracing_stop_tr(struct trace_array *tr)
1390{
1391 struct ring_buffer *buffer;
1392 unsigned long flags;
1393
1394 /* If global, we need to also stop the max tracer */
1395 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1396 return tracing_stop();
1397
1398 raw_spin_lock_irqsave(&tr->start_lock, flags);
1399 if (tr->stop_count++)
1400 goto out;
1401
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001402 buffer = tr->trace_buffer.buffer;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001403 if (buffer)
1404 ring_buffer_record_disable(buffer);
1405
1406 out:
1407 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001408}
1409
Ingo Molnare309b412008-05-12 21:20:51 +02001410void trace_stop_cmdline_recording(void);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001411
Ingo Molnare309b412008-05-12 21:20:51 +02001412static void trace_save_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001413{
Carsten Emdea635cf02009-03-18 09:00:41 +01001414 unsigned pid, idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001415
1416 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1417 return;
1418
1419 /*
1420 * It's not the end of the world if we don't get
1421 * the lock, but we also don't want to spin
1422 * nor do we want to disable interrupts,
1423 * so if we miss here, then better luck next time.
1424 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001425 if (!arch_spin_trylock(&trace_cmdline_lock))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001426 return;
1427
1428 idx = map_pid_to_cmdline[tsk->pid];
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001429 if (idx == NO_CMDLINE_MAP) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001430 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1431
Carsten Emdea635cf02009-03-18 09:00:41 +01001432 /*
1433 * Check whether the cmdline buffer at idx has a pid
1434 * mapped. We are going to overwrite that entry so we
1435 * need to clear the map_pid_to_cmdline. Otherwise we
1436 * would read the new comm for the old pid.
1437 */
1438 pid = map_cmdline_to_pid[idx];
1439 if (pid != NO_CMDLINE_MAP)
1440 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001441
Carsten Emdea635cf02009-03-18 09:00:41 +01001442 map_cmdline_to_pid[idx] = tsk->pid;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001443 map_pid_to_cmdline[tsk->pid] = idx;
1444
1445 cmdline_idx = idx;
1446 }
1447
1448 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1449
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001450 arch_spin_unlock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001451}
1452
Steven Rostedt4ca53082009-03-16 19:20:15 -04001453void trace_find_cmdline(int pid, char comm[])
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001454{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001455 unsigned map;
1456
Steven Rostedt4ca53082009-03-16 19:20:15 -04001457 if (!pid) {
1458 strcpy(comm, "<idle>");
1459 return;
1460 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001461
Steven Rostedt74bf4072010-01-25 15:11:53 -05001462 if (WARN_ON_ONCE(pid < 0)) {
1463 strcpy(comm, "<XXX>");
1464 return;
1465 }
1466
Steven Rostedt4ca53082009-03-16 19:20:15 -04001467 if (pid > PID_MAX_DEFAULT) {
1468 strcpy(comm, "<...>");
1469 return;
1470 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001471
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001472 preempt_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001473 arch_spin_lock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001474 map = map_pid_to_cmdline[pid];
Thomas Gleixner50d88752009-03-18 08:58:44 +01001475 if (map != NO_CMDLINE_MAP)
1476 strcpy(comm, saved_cmdlines[map]);
1477 else
1478 strcpy(comm, "<...>");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001479
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001480 arch_spin_unlock(&trace_cmdline_lock);
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001481 preempt_enable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001482}
1483
Ingo Molnare309b412008-05-12 21:20:51 +02001484void tracing_record_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001485{
Steven Rostedt0fb96562012-05-11 14:25:30 -04001486 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001487 return;
1488
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001489 if (!__this_cpu_read(trace_cmdline_save))
1490 return;
1491
1492 __this_cpu_write(trace_cmdline_save, false);
1493
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001494 trace_save_cmdline(tsk);
1495}
1496
Pekka Paalanen45dcd8b2008-09-16 21:56:41 +03001497void
Steven Rostedt38697052008-10-01 13:14:09 -04001498tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1499 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001500{
1501 struct task_struct *tsk = current;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001502
Steven Rostedt777e2082008-09-29 23:02:42 -04001503 entry->preempt_count = pc & 0xff;
1504 entry->pid = (tsk) ? tsk->pid : 0;
1505 entry->flags =
Steven Rostedt92444892008-10-24 09:42:59 -04001506#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
Steven Rostedt2e2ca152008-08-01 12:26:40 -04001507 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
Steven Rostedt92444892008-10-24 09:42:59 -04001508#else
1509 TRACE_FLAG_IRQS_NOSUPPORT |
1510#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001511 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1512 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1513 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1514}
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +02001515EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001516
Steven Rostedte77405a2009-09-02 14:17:06 -04001517struct ring_buffer_event *
1518trace_buffer_lock_reserve(struct ring_buffer *buffer,
1519 int type,
1520 unsigned long len,
1521 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001522{
1523 struct ring_buffer_event *event;
1524
Steven Rostedte77405a2009-09-02 14:17:06 -04001525 event = ring_buffer_lock_reserve(buffer, len);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001526 if (event != NULL) {
1527 struct trace_entry *ent = ring_buffer_event_data(event);
1528
1529 tracing_generic_entry_update(ent, flags, pc);
1530 ent->type = type;
1531 }
1532
1533 return event;
1534}
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001535
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001536void
1537__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1538{
1539 __this_cpu_write(trace_cmdline_save, true);
1540 ring_buffer_unlock_commit(buffer, event);
1541}
1542
Steven Rostedte77405a2009-09-02 14:17:06 -04001543static inline void
1544__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1545 struct ring_buffer_event *event,
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001546 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001547{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001548 __buffer_unlock_commit(buffer, event);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001549
Steven Rostedte77405a2009-09-02 14:17:06 -04001550 ftrace_trace_stack(buffer, flags, 6, pc);
1551 ftrace_trace_userstack(buffer, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001552}
1553
Steven Rostedte77405a2009-09-02 14:17:06 -04001554void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1555 struct ring_buffer_event *event,
1556 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001557{
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001558 __trace_buffer_unlock_commit(buffer, event, flags, pc);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001559}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001560EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001561
Steven Rostedtef5580d2009-02-27 19:38:04 -05001562struct ring_buffer_event *
Steven Rostedtccb469a2012-08-02 10:32:10 -04001563trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1564 struct ftrace_event_file *ftrace_file,
1565 int type, unsigned long len,
1566 unsigned long flags, int pc)
1567{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001568 *current_rb = ftrace_file->tr->trace_buffer.buffer;
Steven Rostedtccb469a2012-08-02 10:32:10 -04001569 return trace_buffer_lock_reserve(*current_rb,
1570 type, len, flags, pc);
1571}
1572EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1573
1574struct ring_buffer_event *
Steven Rostedte77405a2009-09-02 14:17:06 -04001575trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1576 int type, unsigned long len,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001577 unsigned long flags, int pc)
1578{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001579 *current_rb = global_trace.trace_buffer.buffer;
Steven Rostedte77405a2009-09-02 14:17:06 -04001580 return trace_buffer_lock_reserve(*current_rb,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001581 type, len, flags, pc);
1582}
Steven Rostedt94487d62009-05-05 19:22:53 -04001583EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001584
Steven Rostedte77405a2009-09-02 14:17:06 -04001585void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1586 struct ring_buffer_event *event,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001587 unsigned long flags, int pc)
1588{
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001589 __trace_buffer_unlock_commit(buffer, event, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001590}
Steven Rostedt94487d62009-05-05 19:22:53 -04001591EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001592
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001593void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1594 struct ring_buffer_event *event,
1595 unsigned long flags, int pc,
1596 struct pt_regs *regs)
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001597{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001598 __buffer_unlock_commit(buffer, event);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001599
1600 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1601 ftrace_trace_userstack(buffer, flags, pc);
1602}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001603EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001604
Steven Rostedte77405a2009-09-02 14:17:06 -04001605void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1606 struct ring_buffer_event *event)
Steven Rostedt77d9f462009-04-02 01:16:59 -04001607{
Steven Rostedte77405a2009-09-02 14:17:06 -04001608 ring_buffer_discard_commit(buffer, event);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001609}
Steven Rostedt12acd472009-04-17 16:01:56 -04001610EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001611
Ingo Molnare309b412008-05-12 21:20:51 +02001612void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001613trace_function(struct trace_array *tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001614 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1615 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001616{
Tom Zanussie1112b42009-03-31 00:48:49 -05001617 struct ftrace_event_call *call = &event_function;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001618 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001619 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001620 struct ftrace_entry *entry;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001621
Steven Rostedtd7690412008-10-01 00:29:53 -04001622 /* If we are reading the ring buffer, don't trace */
Rusty Russelldd17c8f2009-10-29 22:34:15 +09001623 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
Steven Rostedtd7690412008-10-01 00:29:53 -04001624 return;
1625
Steven Rostedte77405a2009-09-02 14:17:06 -04001626 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001627 flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001628 if (!event)
1629 return;
1630 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001631 entry->ip = ip;
1632 entry->parent_ip = parent_ip;
Tom Zanussie1112b42009-03-31 00:48:49 -05001633
Steven Rostedte77405a2009-09-02 14:17:06 -04001634 if (!filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001635 __buffer_unlock_commit(buffer, event);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001636}
1637
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001638#ifdef CONFIG_STACKTRACE
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001639
1640#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1641struct ftrace_stack {
1642 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1643};
1644
1645static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1646static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1647
Steven Rostedte77405a2009-09-02 14:17:06 -04001648static void __ftrace_trace_stack(struct ring_buffer *buffer,
Steven Rostedt53614992009-01-15 19:12:40 -05001649 unsigned long flags,
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001650 int skip, int pc, struct pt_regs *regs)
Ingo Molnar86387f72008-05-12 21:20:51 +02001651{
Tom Zanussie1112b42009-03-31 00:48:49 -05001652 struct ftrace_event_call *call = &event_kernel_stack;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001653 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001654 struct stack_entry *entry;
Ingo Molnar86387f72008-05-12 21:20:51 +02001655 struct stack_trace trace;
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001656 int use_stack;
1657 int size = FTRACE_STACK_ENTRIES;
Ingo Molnar86387f72008-05-12 21:20:51 +02001658
1659 trace.nr_entries = 0;
Ingo Molnar86387f72008-05-12 21:20:51 +02001660 trace.skip = skip;
Ingo Molnar86387f72008-05-12 21:20:51 +02001661
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001662 /*
1663 * Since events can happen in NMIs there's no safe way to
1664 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1665 * or NMI comes in, it will just have to use the default
1666 * FTRACE_STACK_SIZE.
1667 */
1668 preempt_disable_notrace();
1669
Shan Wei82146522012-11-19 13:21:01 +08001670 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001671 /*
1672 * We don't need any atomic variables, just a barrier.
1673 * If an interrupt comes in, we don't care, because it would
1674 * have exited and put the counter back to what we want.
1675 * We just need a barrier to keep gcc from moving things
1676 * around.
1677 */
1678 barrier();
1679 if (use_stack == 1) {
1680 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1681 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1682
1683 if (regs)
1684 save_stack_trace_regs(regs, &trace);
1685 else
1686 save_stack_trace(&trace);
1687
1688 if (trace.nr_entries > size)
1689 size = trace.nr_entries;
1690 } else
1691 /* From now on, use_stack is a boolean */
1692 use_stack = 0;
1693
1694 size *= sizeof(unsigned long);
1695
1696 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1697 sizeof(*entry) + size, flags, pc);
1698 if (!event)
1699 goto out;
1700 entry = ring_buffer_event_data(event);
1701
1702 memset(&entry->caller, 0, size);
1703
1704 if (use_stack)
1705 memcpy(&entry->caller, trace.entries,
1706 trace.nr_entries * sizeof(unsigned long));
1707 else {
1708 trace.max_entries = FTRACE_STACK_ENTRIES;
1709 trace.entries = entry->caller;
1710 if (regs)
1711 save_stack_trace_regs(regs, &trace);
1712 else
1713 save_stack_trace(&trace);
1714 }
1715
1716 entry->size = trace.nr_entries;
1717
Steven Rostedte77405a2009-09-02 14:17:06 -04001718 if (!filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001719 __buffer_unlock_commit(buffer, event);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001720
1721 out:
1722 /* Again, don't let gcc optimize things here */
1723 barrier();
Shan Wei82146522012-11-19 13:21:01 +08001724 __this_cpu_dec(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001725 preempt_enable_notrace();
1726
Ingo Molnarf0a920d2008-05-12 21:20:47 +02001727}
1728
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001729void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1730 int skip, int pc, struct pt_regs *regs)
1731{
1732 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1733 return;
1734
1735 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1736}
1737
Steven Rostedte77405a2009-09-02 14:17:06 -04001738void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1739 int skip, int pc)
Steven Rostedt53614992009-01-15 19:12:40 -05001740{
1741 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1742 return;
1743
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001744 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
Steven Rostedt53614992009-01-15 19:12:40 -05001745}
1746
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001747void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1748 int pc)
Steven Rostedt38697052008-10-01 13:14:09 -04001749{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001750 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
Steven Rostedt38697052008-10-01 13:14:09 -04001751}
1752
Steven Rostedt03889382009-12-11 09:48:22 -05001753/**
1754 * trace_dump_stack - record a stack back trace in the trace buffer
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001755 * @skip: Number of functions to skip (helper handlers)
Steven Rostedt03889382009-12-11 09:48:22 -05001756 */
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001757void trace_dump_stack(int skip)
Steven Rostedt03889382009-12-11 09:48:22 -05001758{
1759 unsigned long flags;
1760
1761 if (tracing_disabled || tracing_selftest_running)
Steven Rostedte36c5452009-12-14 15:58:33 -05001762 return;
Steven Rostedt03889382009-12-11 09:48:22 -05001763
1764 local_save_flags(flags);
1765
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001766 /*
1767 * Skip 3 more, seems to get us at the caller of
1768 * this function.
1769 */
1770 skip += 3;
1771 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1772 flags, skip, preempt_count(), NULL);
Steven Rostedt03889382009-12-11 09:48:22 -05001773}
1774
Steven Rostedt91e86e52010-11-10 12:56:12 +01001775static DEFINE_PER_CPU(int, user_stack_count);
1776
Steven Rostedte77405a2009-09-02 14:17:06 -04001777void
1778ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
Török Edwin02b67512008-11-22 13:28:47 +02001779{
Tom Zanussie1112b42009-03-31 00:48:49 -05001780 struct ftrace_event_call *call = &event_user_stack;
Török Edwin8d7c6a92008-11-23 12:39:06 +02001781 struct ring_buffer_event *event;
Török Edwin02b67512008-11-22 13:28:47 +02001782 struct userstack_entry *entry;
1783 struct stack_trace trace;
Török Edwin02b67512008-11-22 13:28:47 +02001784
1785 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1786 return;
1787
Steven Rostedtb6345872010-03-12 20:03:30 -05001788 /*
1789 * NMIs can not handle page faults, even with fix ups.
1790 * The save user stack can (and often does) fault.
1791 */
1792 if (unlikely(in_nmi()))
1793 return;
1794
Steven Rostedt91e86e52010-11-10 12:56:12 +01001795 /*
1796 * prevent recursion, since the user stack tracing may
1797 * trigger other kernel events.
1798 */
1799 preempt_disable();
1800 if (__this_cpu_read(user_stack_count))
1801 goto out;
1802
1803 __this_cpu_inc(user_stack_count);
1804
Steven Rostedte77405a2009-09-02 14:17:06 -04001805 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001806 sizeof(*entry), flags, pc);
Török Edwin02b67512008-11-22 13:28:47 +02001807 if (!event)
Li Zefan1dbd1952010-12-09 15:47:56 +08001808 goto out_drop_count;
Török Edwin02b67512008-11-22 13:28:47 +02001809 entry = ring_buffer_event_data(event);
Török Edwin02b67512008-11-22 13:28:47 +02001810
Steven Rostedt48659d32009-09-11 11:36:23 -04001811 entry->tgid = current->tgid;
Török Edwin02b67512008-11-22 13:28:47 +02001812 memset(&entry->caller, 0, sizeof(entry->caller));
1813
1814 trace.nr_entries = 0;
1815 trace.max_entries = FTRACE_STACK_ENTRIES;
1816 trace.skip = 0;
1817 trace.entries = entry->caller;
1818
1819 save_stack_trace_user(&trace);
Steven Rostedte77405a2009-09-02 14:17:06 -04001820 if (!filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001821 __buffer_unlock_commit(buffer, event);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001822
Li Zefan1dbd1952010-12-09 15:47:56 +08001823 out_drop_count:
Steven Rostedt91e86e52010-11-10 12:56:12 +01001824 __this_cpu_dec(user_stack_count);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001825 out:
1826 preempt_enable();
Török Edwin02b67512008-11-22 13:28:47 +02001827}
1828
Hannes Eder4fd27352009-02-10 19:44:12 +01001829#ifdef UNUSED
1830static void __trace_userstack(struct trace_array *tr, unsigned long flags)
Török Edwin02b67512008-11-22 13:28:47 +02001831{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001832 ftrace_trace_userstack(tr, flags, preempt_count());
Török Edwin02b67512008-11-22 13:28:47 +02001833}
Hannes Eder4fd27352009-02-10 19:44:12 +01001834#endif /* UNUSED */
Török Edwin02b67512008-11-22 13:28:47 +02001835
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001836#endif /* CONFIG_STACKTRACE */
1837
Steven Rostedt07d777f2011-09-22 14:01:55 -04001838/* created for use with alloc_percpu */
1839struct trace_buffer_struct {
1840 char buffer[TRACE_BUF_SIZE];
1841};
1842
1843static struct trace_buffer_struct *trace_percpu_buffer;
1844static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1845static struct trace_buffer_struct *trace_percpu_irq_buffer;
1846static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1847
1848/*
1849 * The buffer used is dependent on the context. There is a per cpu
1850 * buffer for normal context, softirq contex, hard irq context and
1851 * for NMI context. Thise allows for lockless recording.
1852 *
1853 * Note, if the buffers failed to be allocated, then this returns NULL
1854 */
1855static char *get_trace_buf(void)
1856{
1857 struct trace_buffer_struct *percpu_buffer;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001858
1859 /*
1860 * If we have allocated per cpu buffers, then we do not
1861 * need to do any locking.
1862 */
1863 if (in_nmi())
1864 percpu_buffer = trace_percpu_nmi_buffer;
1865 else if (in_irq())
1866 percpu_buffer = trace_percpu_irq_buffer;
1867 else if (in_softirq())
1868 percpu_buffer = trace_percpu_sirq_buffer;
1869 else
1870 percpu_buffer = trace_percpu_buffer;
1871
1872 if (!percpu_buffer)
1873 return NULL;
1874
Shan Weid8a03492012-11-13 09:53:04 +08001875 return this_cpu_ptr(&percpu_buffer->buffer[0]);
Steven Rostedt07d777f2011-09-22 14:01:55 -04001876}
1877
1878static int alloc_percpu_trace_buffer(void)
1879{
1880 struct trace_buffer_struct *buffers;
1881 struct trace_buffer_struct *sirq_buffers;
1882 struct trace_buffer_struct *irq_buffers;
1883 struct trace_buffer_struct *nmi_buffers;
1884
1885 buffers = alloc_percpu(struct trace_buffer_struct);
1886 if (!buffers)
1887 goto err_warn;
1888
1889 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1890 if (!sirq_buffers)
1891 goto err_sirq;
1892
1893 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1894 if (!irq_buffers)
1895 goto err_irq;
1896
1897 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1898 if (!nmi_buffers)
1899 goto err_nmi;
1900
1901 trace_percpu_buffer = buffers;
1902 trace_percpu_sirq_buffer = sirq_buffers;
1903 trace_percpu_irq_buffer = irq_buffers;
1904 trace_percpu_nmi_buffer = nmi_buffers;
1905
1906 return 0;
1907
1908 err_nmi:
1909 free_percpu(irq_buffers);
1910 err_irq:
1911 free_percpu(sirq_buffers);
1912 err_sirq:
1913 free_percpu(buffers);
1914 err_warn:
1915 WARN(1, "Could not allocate percpu trace_printk buffer");
1916 return -ENOMEM;
1917}
1918
Steven Rostedt81698832012-10-11 10:15:05 -04001919static int buffers_allocated;
1920
Steven Rostedt07d777f2011-09-22 14:01:55 -04001921void trace_printk_init_buffers(void)
1922{
Steven Rostedt07d777f2011-09-22 14:01:55 -04001923 if (buffers_allocated)
1924 return;
1925
1926 if (alloc_percpu_trace_buffer())
1927 return;
1928
1929 pr_info("ftrace: Allocated trace_printk buffers\n");
1930
Steven Rostedtb382ede62012-10-10 21:44:34 -04001931 /* Expand the buffers to set size */
1932 tracing_update_buffers();
1933
Steven Rostedt07d777f2011-09-22 14:01:55 -04001934 buffers_allocated = 1;
Steven Rostedt81698832012-10-11 10:15:05 -04001935
1936 /*
1937 * trace_printk_init_buffers() can be called by modules.
1938 * If that happens, then we need to start cmdline recording
1939 * directly here. If the global_trace.buffer is already
1940 * allocated here, then this was called by module code.
1941 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001942 if (global_trace.trace_buffer.buffer)
Steven Rostedt81698832012-10-11 10:15:05 -04001943 tracing_start_cmdline_record();
1944}
1945
1946void trace_printk_start_comm(void)
1947{
1948 /* Start tracing comms if trace printk is set */
1949 if (!buffers_allocated)
1950 return;
1951 tracing_start_cmdline_record();
1952}
1953
1954static void trace_printk_start_stop_comm(int enabled)
1955{
1956 if (!buffers_allocated)
1957 return;
1958
1959 if (enabled)
1960 tracing_start_cmdline_record();
1961 else
1962 tracing_stop_cmdline_record();
Steven Rostedt07d777f2011-09-22 14:01:55 -04001963}
1964
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001965/**
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001966 * trace_vbprintk - write binary msg to tracing buffer
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001967 *
1968 */
Steven Rostedt40ce74f2009-03-19 14:03:53 -04001969int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001970{
Tom Zanussie1112b42009-03-31 00:48:49 -05001971 struct ftrace_event_call *call = &event_bprint;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001972 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001973 struct ring_buffer *buffer;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001974 struct trace_array *tr = &global_trace;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001975 struct bprint_entry *entry;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001976 unsigned long flags;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001977 char *tbuffer;
1978 int len = 0, size, pc;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001979
1980 if (unlikely(tracing_selftest_running || tracing_disabled))
1981 return 0;
1982
1983 /* Don't pollute graph traces with trace_vprintk internals */
1984 pause_graph_tracing();
1985
1986 pc = preempt_count();
Steven Rostedt5168ae52010-06-03 09:36:50 -04001987 preempt_disable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001988
Steven Rostedt07d777f2011-09-22 14:01:55 -04001989 tbuffer = get_trace_buf();
1990 if (!tbuffer) {
1991 len = 0;
1992 goto out;
1993 }
1994
1995 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1996
1997 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001998 goto out;
1999
Steven Rostedt07d777f2011-09-22 14:01:55 -04002000 local_save_flags(flags);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002001 size = sizeof(*entry) + sizeof(u32) * len;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002002 buffer = tr->trace_buffer.buffer;
Steven Rostedte77405a2009-09-02 14:17:06 -04002003 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2004 flags, pc);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002005 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04002006 goto out;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002007 entry = ring_buffer_event_data(event);
2008 entry->ip = ip;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002009 entry->fmt = fmt;
2010
Steven Rostedt07d777f2011-09-22 14:01:55 -04002011 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
Steven Rostedtd9313692010-01-06 17:27:11 -05002012 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04002013 __buffer_unlock_commit(buffer, event);
Steven Rostedtd9313692010-01-06 17:27:11 -05002014 ftrace_trace_stack(buffer, flags, 6, pc);
2015 }
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002016
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002017out:
Steven Rostedt5168ae52010-06-03 09:36:50 -04002018 preempt_enable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002019 unpause_graph_tracing();
2020
2021 return len;
2022}
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002023EXPORT_SYMBOL_GPL(trace_vbprintk);
2024
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002025static int
2026__trace_array_vprintk(struct ring_buffer *buffer,
2027 unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002028{
Tom Zanussie1112b42009-03-31 00:48:49 -05002029 struct ftrace_event_call *call = &event_print;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002030 struct ring_buffer_event *event;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002031 int len = 0, size, pc;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002032 struct print_entry *entry;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002033 unsigned long flags;
2034 char *tbuffer;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002035
2036 if (tracing_disabled || tracing_selftest_running)
2037 return 0;
2038
Steven Rostedt07d777f2011-09-22 14:01:55 -04002039 /* Don't pollute graph traces with trace_vprintk internals */
2040 pause_graph_tracing();
2041
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002042 pc = preempt_count();
2043 preempt_disable_notrace();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002044
Steven Rostedt07d777f2011-09-22 14:01:55 -04002045
2046 tbuffer = get_trace_buf();
2047 if (!tbuffer) {
2048 len = 0;
2049 goto out;
2050 }
2051
2052 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2053 if (len > TRACE_BUF_SIZE)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002054 goto out;
2055
Steven Rostedt07d777f2011-09-22 14:01:55 -04002056 local_save_flags(flags);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002057 size = sizeof(*entry) + len + 1;
Steven Rostedte77405a2009-09-02 14:17:06 -04002058 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
Steven Rostedt07d777f2011-09-22 14:01:55 -04002059 flags, pc);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002060 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04002061 goto out;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002062 entry = ring_buffer_event_data(event);
Carsten Emdec13d2f72009-11-16 20:56:13 +01002063 entry->ip = ip;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002064
Steven Rostedt07d777f2011-09-22 14:01:55 -04002065 memcpy(&entry->buf, tbuffer, len);
Carsten Emdec13d2f72009-11-16 20:56:13 +01002066 entry->buf[len] = '\0';
Steven Rostedtd9313692010-01-06 17:27:11 -05002067 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04002068 __buffer_unlock_commit(buffer, event);
Steven Rostedt07d777f2011-09-22 14:01:55 -04002069 ftrace_trace_stack(buffer, flags, 6, pc);
Steven Rostedtd9313692010-01-06 17:27:11 -05002070 }
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002071 out:
2072 preempt_enable_notrace();
Steven Rostedt07d777f2011-09-22 14:01:55 -04002073 unpause_graph_tracing();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002074
2075 return len;
2076}
Steven Rostedt659372d2009-09-03 19:11:07 -04002077
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002078int trace_array_vprintk(struct trace_array *tr,
2079 unsigned long ip, const char *fmt, va_list args)
2080{
2081 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2082}
2083
2084int trace_array_printk(struct trace_array *tr,
2085 unsigned long ip, const char *fmt, ...)
2086{
2087 int ret;
2088 va_list ap;
2089
2090 if (!(trace_flags & TRACE_ITER_PRINTK))
2091 return 0;
2092
2093 va_start(ap, fmt);
2094 ret = trace_array_vprintk(tr, ip, fmt, ap);
2095 va_end(ap);
2096 return ret;
2097}
2098
2099int trace_array_printk_buf(struct ring_buffer *buffer,
2100 unsigned long ip, const char *fmt, ...)
2101{
2102 int ret;
2103 va_list ap;
2104
2105 if (!(trace_flags & TRACE_ITER_PRINTK))
2106 return 0;
2107
2108 va_start(ap, fmt);
2109 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2110 va_end(ap);
2111 return ret;
2112}
2113
Steven Rostedt659372d2009-09-03 19:11:07 -04002114int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2115{
Steven Rostedta813a152009-10-09 01:41:35 -04002116 return trace_array_vprintk(&global_trace, ip, fmt, args);
Steven Rostedt659372d2009-09-03 19:11:07 -04002117}
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002118EXPORT_SYMBOL_GPL(trace_vprintk);
2119
Robert Richtere2ac8ef2008-11-12 12:59:32 +01002120static void trace_iterator_increment(struct trace_iterator *iter)
Steven Rostedt5a90f572008-09-03 17:42:51 -04002121{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002122 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2123
Steven Rostedt5a90f572008-09-03 17:42:51 -04002124 iter->idx++;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002125 if (buf_iter)
2126 ring_buffer_read(buf_iter, NULL);
Steven Rostedt5a90f572008-09-03 17:42:51 -04002127}
2128
Ingo Molnare309b412008-05-12 21:20:51 +02002129static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04002130peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2131 unsigned long *lost_events)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002132{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002133 struct ring_buffer_event *event;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002134 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002135
Steven Rostedtd7690412008-10-01 00:29:53 -04002136 if (buf_iter)
2137 event = ring_buffer_iter_peek(buf_iter, ts);
2138 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002139 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
Steven Rostedtbc21b472010-03-31 19:49:26 -04002140 lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04002141
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04002142 if (event) {
2143 iter->ent_size = ring_buffer_event_length(event);
2144 return ring_buffer_event_data(event);
2145 }
2146 iter->ent_size = 0;
2147 return NULL;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002148}
Steven Rostedtd7690412008-10-01 00:29:53 -04002149
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002150static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04002151__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2152 unsigned long *missing_events, u64 *ent_ts)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002153{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002154 struct ring_buffer *buffer = iter->trace_buffer->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002155 struct trace_entry *ent, *next = NULL;
Lai Jiangshanaa274972010-04-05 17:11:05 +08002156 unsigned long lost_events = 0, next_lost = 0;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002157 int cpu_file = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002158 u64 next_ts = 0, ts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002159 int next_cpu = -1;
Steven Rostedt12b5da32012-03-27 10:43:28 -04002160 int next_size = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002161 int cpu;
2162
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002163 /*
2164 * If we are in a per_cpu trace file, don't bother by iterating over
2165 * all cpu and peek directly.
2166 */
Steven Rostedtae3b5092013-01-23 15:22:59 -05002167 if (cpu_file > RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002168 if (ring_buffer_empty_cpu(buffer, cpu_file))
2169 return NULL;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002170 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002171 if (ent_cpu)
2172 *ent_cpu = cpu_file;
2173
2174 return ent;
2175 }
2176
Steven Rostedtab464282008-05-12 21:21:00 +02002177 for_each_tracing_cpu(cpu) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002178
2179 if (ring_buffer_empty_cpu(buffer, cpu))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002180 continue;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002181
Steven Rostedtbc21b472010-03-31 19:49:26 -04002182 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002183
Ingo Molnarcdd31cd2008-05-12 21:20:46 +02002184 /*
2185 * Pick the entry with the smallest timestamp:
2186 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002187 if (ent && (!next || ts < next_ts)) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002188 next = ent;
2189 next_cpu = cpu;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002190 next_ts = ts;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002191 next_lost = lost_events;
Steven Rostedt12b5da32012-03-27 10:43:28 -04002192 next_size = iter->ent_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002193 }
2194 }
2195
Steven Rostedt12b5da32012-03-27 10:43:28 -04002196 iter->ent_size = next_size;
2197
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002198 if (ent_cpu)
2199 *ent_cpu = next_cpu;
2200
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002201 if (ent_ts)
2202 *ent_ts = next_ts;
2203
Steven Rostedtbc21b472010-03-31 19:49:26 -04002204 if (missing_events)
2205 *missing_events = next_lost;
2206
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002207 return next;
2208}
2209
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002210/* Find the next real entry, without updating the iterator itself */
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002211struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2212 int *ent_cpu, u64 *ent_ts)
Steven Rostedtb3806b42008-05-12 21:20:46 +02002213{
Steven Rostedtbc21b472010-03-31 19:49:26 -04002214 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002215}
Ingo Molnar8c523a92008-05-12 21:20:46 +02002216
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002217/* Find the next real entry, and increment the iterator to the next entry */
Jason Wessel955b61e2010-08-05 09:22:23 -05002218void *trace_find_next_entry_inc(struct trace_iterator *iter)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002219{
Steven Rostedtbc21b472010-03-31 19:49:26 -04002220 iter->ent = __find_next_entry(iter, &iter->cpu,
2221 &iter->lost_events, &iter->ts);
Steven Rostedtb3806b42008-05-12 21:20:46 +02002222
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002223 if (iter->ent)
Robert Richtere2ac8ef2008-11-12 12:59:32 +01002224 trace_iterator_increment(iter);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002225
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002226 return iter->ent ? iter : NULL;
Steven Rostedtb3806b42008-05-12 21:20:46 +02002227}
2228
Ingo Molnare309b412008-05-12 21:20:51 +02002229static void trace_consume(struct trace_iterator *iter)
Steven Rostedtb3806b42008-05-12 21:20:46 +02002230{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002231 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
Steven Rostedtbc21b472010-03-31 19:49:26 -04002232 &iter->lost_events);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002233}
2234
Ingo Molnare309b412008-05-12 21:20:51 +02002235static void *s_next(struct seq_file *m, void *v, loff_t *pos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002236{
2237 struct trace_iterator *iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002238 int i = (int)*pos;
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002239 void *ent;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002240
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002241 WARN_ON_ONCE(iter->leftover);
2242
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002243 (*pos)++;
2244
2245 /* can't go backwards */
2246 if (iter->idx > i)
2247 return NULL;
2248
2249 if (iter->idx < 0)
Jason Wessel955b61e2010-08-05 09:22:23 -05002250 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002251 else
2252 ent = iter;
2253
2254 while (ent && iter->idx < i)
Jason Wessel955b61e2010-08-05 09:22:23 -05002255 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002256
2257 iter->pos = *pos;
2258
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002259 return ent;
2260}
2261
Jason Wessel955b61e2010-08-05 09:22:23 -05002262void tracing_iter_reset(struct trace_iterator *iter, int cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002263{
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002264 struct ring_buffer_event *event;
2265 struct ring_buffer_iter *buf_iter;
2266 unsigned long entries = 0;
2267 u64 ts;
2268
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002269 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002270
Steven Rostedt6d158a82012-06-27 20:46:14 -04002271 buf_iter = trace_buffer_iter(iter, cpu);
2272 if (!buf_iter)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002273 return;
2274
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002275 ring_buffer_iter_reset(buf_iter);
2276
2277 /*
2278 * We could have the case with the max latency tracers
2279 * that a reset never took place on a cpu. This is evident
2280 * by the timestamp being before the start of the buffer.
2281 */
2282 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002283 if (ts >= iter->trace_buffer->time_start)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002284 break;
2285 entries++;
2286 ring_buffer_read(buf_iter, NULL);
2287 }
2288
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002289 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002290}
2291
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002292/*
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002293 * The current tracer is copied to avoid a global locking
2294 * all around.
2295 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002296static void *s_start(struct seq_file *m, loff_t *pos)
2297{
2298 struct trace_iterator *iter = m->private;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002299 struct trace_array *tr = iter->tr;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002300 int cpu_file = iter->cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002301 void *p = NULL;
2302 loff_t l = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002303 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002304
Hiraku Toyooka2fd196e2012-12-26 11:52:52 +09002305 /*
2306 * copy the tracer to avoid using a global lock all around.
2307 * iter->trace is a copy of current_trace, the pointer to the
2308 * name may be used instead of a strcmp(), as iter->trace->name
2309 * will point to the same string as current_trace->name.
2310 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002311 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002312 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2313 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002314 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002315
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002316#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002317 if (iter->snapshot && iter->trace->use_max_tr)
2318 return ERR_PTR(-EBUSY);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002319#endif
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002320
2321 if (!iter->snapshot)
2322 atomic_inc(&trace_record_cmdline_disabled);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002323
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002324 if (*pos != iter->pos) {
2325 iter->ent = NULL;
2326 iter->cpu = 0;
2327 iter->idx = -1;
2328
Steven Rostedtae3b5092013-01-23 15:22:59 -05002329 if (cpu_file == RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002330 for_each_tracing_cpu(cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002331 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002332 } else
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002333 tracing_iter_reset(iter, cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002334
Lai Jiangshanac91d852010-03-02 17:54:50 +08002335 iter->leftover = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002336 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2337 ;
2338
2339 } else {
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002340 /*
2341 * If we overflowed the seq_file before, then we want
2342 * to just reuse the trace_seq buffer again.
2343 */
2344 if (iter->leftover)
2345 p = iter;
2346 else {
2347 l = *pos - 1;
2348 p = s_next(m, p, &l);
2349 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002350 }
2351
Lai Jiangshan4f535962009-05-18 19:35:34 +08002352 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002353 trace_access_lock(cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002354 return p;
2355}
2356
2357static void s_stop(struct seq_file *m, void *p)
2358{
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002359 struct trace_iterator *iter = m->private;
2360
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002361#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002362 if (iter->snapshot && iter->trace->use_max_tr)
2363 return;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002364#endif
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002365
2366 if (!iter->snapshot)
2367 atomic_dec(&trace_record_cmdline_disabled);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002368
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002369 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08002370 trace_event_read_unlock();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002371}
2372
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002373static void
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002374get_total_entries(struct trace_buffer *buf,
2375 unsigned long *total, unsigned long *entries)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002376{
2377 unsigned long count;
2378 int cpu;
2379
2380 *total = 0;
2381 *entries = 0;
2382
2383 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002384 count = ring_buffer_entries_cpu(buf->buffer, cpu);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002385 /*
2386 * If this buffer has skipped entries, then we hold all
2387 * entries for the trace and we need to ignore the
2388 * ones before the time stamp.
2389 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002390 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2391 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002392 /* total is the same as the entries */
2393 *total += count;
2394 } else
2395 *total += count +
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002396 ring_buffer_overrun_cpu(buf->buffer, cpu);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002397 *entries += count;
2398 }
2399}
2400
Ingo Molnare309b412008-05-12 21:20:51 +02002401static void print_lat_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002402{
Michael Ellermana6168352008-08-20 16:36:11 -07002403 seq_puts(m, "# _------=> CPU# \n");
2404 seq_puts(m, "# / _-----=> irqs-off \n");
2405 seq_puts(m, "# | / _----=> need-resched \n");
2406 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2407 seq_puts(m, "# ||| / _--=> preempt-depth \n");
Steven Rostedte6e1e252011-03-09 10:41:56 -05002408 seq_puts(m, "# |||| / delay \n");
2409 seq_puts(m, "# cmd pid ||||| time | caller \n");
2410 seq_puts(m, "# \\ / ||||| \\ | / \n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002411}
2412
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002413static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002414{
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002415 unsigned long total;
2416 unsigned long entries;
2417
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002418 get_total_entries(buf, &total, &entries);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002419 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2420 entries, total, num_online_cpus());
2421 seq_puts(m, "#\n");
2422}
2423
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002424static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002425{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002426 print_event_info(buf, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002427 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
Michael Ellermana6168352008-08-20 16:36:11 -07002428 seq_puts(m, "# | | | | |\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002429}
2430
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002431static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedt77271ce2011-11-17 09:34:33 -05002432{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002433 print_event_info(buf, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002434 seq_puts(m, "# _-----=> irqs-off\n");
2435 seq_puts(m, "# / _----=> need-resched\n");
2436 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2437 seq_puts(m, "# || / _--=> preempt-depth\n");
2438 seq_puts(m, "# ||| / delay\n");
2439 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2440 seq_puts(m, "# | | | |||| | |\n");
2441}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002442
Jiri Olsa62b915f2010-04-02 19:01:22 +02002443void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002444print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2445{
2446 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002447 struct trace_buffer *buf = iter->trace_buffer;
2448 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002449 struct tracer *type = iter->trace;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002450 unsigned long entries;
2451 unsigned long total;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002452 const char *name = "preemption";
2453
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05002454 name = type->name;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002455
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002456 get_total_entries(buf, &total, &entries);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002457
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002458 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002459 name, UTS_RELEASE);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002460 seq_puts(m, "# -----------------------------------"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002461 "---------------------------------\n");
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002462 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002463 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
Steven Rostedt57f50be2008-05-12 21:20:44 +02002464 nsecs_to_usecs(data->saved_latency),
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002465 entries,
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02002466 total,
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002467 buf->cpu,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002468#if defined(CONFIG_PREEMPT_NONE)
2469 "server",
2470#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2471 "desktop",
Steven Rostedtb5c21b42008-07-10 20:58:12 -04002472#elif defined(CONFIG_PREEMPT)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002473 "preempt",
2474#else
2475 "unknown",
2476#endif
2477 /* These are reserved for later use */
2478 0, 0, 0, 0);
2479#ifdef CONFIG_SMP
2480 seq_printf(m, " #P:%d)\n", num_online_cpus());
2481#else
2482 seq_puts(m, ")\n");
2483#endif
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002484 seq_puts(m, "# -----------------\n");
2485 seq_printf(m, "# | task: %.16s-%d "
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002486 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
Eric W. Biedermand20b92a2012-03-13 16:02:19 -07002487 data->comm, data->pid,
2488 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002489 data->policy, data->rt_priority);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002490 seq_puts(m, "# -----------------\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002491
2492 if (data->critical_start) {
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002493 seq_puts(m, "# => started at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002494 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2495 trace_print_seq(m, &iter->seq);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002496 seq_puts(m, "\n# => ended at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002497 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2498 trace_print_seq(m, &iter->seq);
Steven Rostedt8248ac02009-09-02 12:27:41 -04002499 seq_puts(m, "\n#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002500 }
2501
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002502 seq_puts(m, "#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002503}
2504
Steven Rostedta3097202008-11-07 22:36:02 -05002505static void test_cpu_buff_start(struct trace_iterator *iter)
2506{
2507 struct trace_seq *s = &iter->seq;
2508
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002509 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2510 return;
2511
2512 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2513 return;
2514
Rusty Russell44623442009-01-01 10:12:23 +10302515 if (cpumask_test_cpu(iter->cpu, iter->started))
Steven Rostedta3097202008-11-07 22:36:02 -05002516 return;
2517
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002518 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002519 return;
2520
Rusty Russell44623442009-01-01 10:12:23 +10302521 cpumask_set_cpu(iter->cpu, iter->started);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002522
2523 /* Don't print started cpu buffer for the first entry of the trace */
2524 if (iter->idx > 1)
2525 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2526 iter->cpu);
Steven Rostedta3097202008-11-07 22:36:02 -05002527}
2528
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002529static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002530{
Steven Rostedt214023c2008-05-12 21:20:46 +02002531 struct trace_seq *s = &iter->seq;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002532 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002533 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002534 struct trace_event *event;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002535
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002536 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002537
Steven Rostedta3097202008-11-07 22:36:02 -05002538 test_cpu_buff_start(iter);
2539
Steven Rostedtf633cef2008-12-23 23:24:13 -05002540 event = ftrace_find_event(entry->type);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002541
2542 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt27d48be2009-03-04 21:57:29 -05002543 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2544 if (!trace_print_lat_context(iter))
2545 goto partial;
2546 } else {
2547 if (!trace_print_context(iter))
2548 goto partial;
2549 }
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002550 }
2551
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002552 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002553 return event->funcs->trace(iter, sym_flags, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002554
2555 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2556 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04002557
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002558 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002559partial:
2560 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002561}
2562
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002563static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002564{
2565 struct trace_seq *s = &iter->seq;
2566 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002567 struct trace_event *event;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002568
2569 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002570
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002571 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002572 if (!trace_seq_printf(s, "%d %d %llu ",
2573 entry->pid, iter->cpu, iter->ts))
2574 goto partial;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002575 }
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002576
Steven Rostedtf633cef2008-12-23 23:24:13 -05002577 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002578 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002579 return event->funcs->raw(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002580
2581 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2582 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04002583
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002584 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002585partial:
2586 return TRACE_TYPE_PARTIAL_LINE;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002587}
2588
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002589static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002590{
2591 struct trace_seq *s = &iter->seq;
2592 unsigned char newline = '\n';
2593 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002594 struct trace_event *event;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002595
2596 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002597
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002598 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2599 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2600 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2601 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2602 }
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002603
Steven Rostedtf633cef2008-12-23 23:24:13 -05002604 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002605 if (event) {
Steven Rostedta9a57762010-04-22 18:46:14 -04002606 enum print_line_t ret = event->funcs->hex(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002607 if (ret != TRACE_TYPE_HANDLED)
2608 return ret;
2609 }
Steven Rostedt7104f302008-10-01 10:52:51 -04002610
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002611 SEQ_PUT_FIELD_RET(s, newline);
2612
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002613 return TRACE_TYPE_HANDLED;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002614}
2615
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002616static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002617{
2618 struct trace_seq *s = &iter->seq;
2619 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002620 struct trace_event *event;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002621
2622 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002623
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002624 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2625 SEQ_PUT_FIELD_RET(s, entry->pid);
Steven Rostedt1830b52d2009-02-07 19:38:43 -05002626 SEQ_PUT_FIELD_RET(s, iter->cpu);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002627 SEQ_PUT_FIELD_RET(s, iter->ts);
2628 }
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002629
Steven Rostedtf633cef2008-12-23 23:24:13 -05002630 event = ftrace_find_event(entry->type);
Steven Rostedta9a57762010-04-22 18:46:14 -04002631 return event ? event->funcs->binary(iter, 0, event) :
2632 TRACE_TYPE_HANDLED;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002633}
2634
Jiri Olsa62b915f2010-04-02 19:01:22 +02002635int trace_empty(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002636{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002637 struct ring_buffer_iter *buf_iter;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002638 int cpu;
2639
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002640 /* If we are looking at one CPU buffer, only check that one */
Steven Rostedtae3b5092013-01-23 15:22:59 -05002641 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002642 cpu = iter->cpu_file;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002643 buf_iter = trace_buffer_iter(iter, cpu);
2644 if (buf_iter) {
2645 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002646 return 0;
2647 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002648 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002649 return 0;
2650 }
2651 return 1;
2652 }
2653
Steven Rostedtab464282008-05-12 21:21:00 +02002654 for_each_tracing_cpu(cpu) {
Steven Rostedt6d158a82012-06-27 20:46:14 -04002655 buf_iter = trace_buffer_iter(iter, cpu);
2656 if (buf_iter) {
2657 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedtd7690412008-10-01 00:29:53 -04002658 return 0;
2659 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002660 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
Steven Rostedtd7690412008-10-01 00:29:53 -04002661 return 0;
2662 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002663 }
Steven Rostedtd7690412008-10-01 00:29:53 -04002664
Frederic Weisbecker797d3712008-09-30 18:13:45 +02002665 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002666}
2667
Lai Jiangshan4f535962009-05-18 19:35:34 +08002668/* Called with trace_event_read_lock() held. */
Jason Wessel955b61e2010-08-05 09:22:23 -05002669enum print_line_t print_trace_line(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002670{
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002671 enum print_line_t ret;
2672
Jiri Olsaee5e51f2011-03-25 12:05:18 +01002673 if (iter->lost_events &&
2674 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2675 iter->cpu, iter->lost_events))
2676 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002677
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002678 if (iter->trace && iter->trace->print_line) {
2679 ret = iter->trace->print_line(iter);
2680 if (ret != TRACE_TYPE_UNHANDLED)
2681 return ret;
2682 }
Thomas Gleixner72829bc2008-05-23 21:37:28 +02002683
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -05002684 if (iter->ent->type == TRACE_BPUTS &&
2685 trace_flags & TRACE_ITER_PRINTK &&
2686 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2687 return trace_print_bputs_msg_only(iter);
2688
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002689 if (iter->ent->type == TRACE_BPRINT &&
2690 trace_flags & TRACE_ITER_PRINTK &&
2691 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002692 return trace_print_bprintk_msg_only(iter);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002693
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002694 if (iter->ent->type == TRACE_PRINT &&
2695 trace_flags & TRACE_ITER_PRINTK &&
2696 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002697 return trace_print_printk_msg_only(iter);
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002698
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002699 if (trace_flags & TRACE_ITER_BIN)
2700 return print_bin_fmt(iter);
2701
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002702 if (trace_flags & TRACE_ITER_HEX)
2703 return print_hex_fmt(iter);
2704
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002705 if (trace_flags & TRACE_ITER_RAW)
2706 return print_raw_fmt(iter);
2707
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002708 return print_trace_fmt(iter);
2709}
2710
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002711void trace_latency_header(struct seq_file *m)
2712{
2713 struct trace_iterator *iter = m->private;
2714
2715 /* print nothing if the buffers are empty */
2716 if (trace_empty(iter))
2717 return;
2718
2719 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2720 print_trace_header(m, iter);
2721
2722 if (!(trace_flags & TRACE_ITER_VERBOSE))
2723 print_lat_help_header(m);
2724}
2725
Jiri Olsa62b915f2010-04-02 19:01:22 +02002726void trace_default_header(struct seq_file *m)
2727{
2728 struct trace_iterator *iter = m->private;
2729
Jiri Olsaf56e7f82011-06-03 16:58:49 +02002730 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2731 return;
2732
Jiri Olsa62b915f2010-04-02 19:01:22 +02002733 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2734 /* print nothing if the buffers are empty */
2735 if (trace_empty(iter))
2736 return;
2737 print_trace_header(m, iter);
2738 if (!(trace_flags & TRACE_ITER_VERBOSE))
2739 print_lat_help_header(m);
2740 } else {
Steven Rostedt77271ce2011-11-17 09:34:33 -05002741 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2742 if (trace_flags & TRACE_ITER_IRQ_INFO)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002743 print_func_help_header_irq(iter->trace_buffer, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002744 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002745 print_func_help_header(iter->trace_buffer, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002746 }
Jiri Olsa62b915f2010-04-02 19:01:22 +02002747 }
2748}
2749
Steven Rostedte0a413f2011-09-29 21:26:16 -04002750static void test_ftrace_alive(struct seq_file *m)
2751{
2752 if (!ftrace_is_dead())
2753 return;
2754 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2755 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2756}
2757
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002758#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002759static void show_snapshot_main_help(struct seq_file *m)
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002760{
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002761 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2762 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2763 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2764 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2765 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2766 seq_printf(m, "# is not a '0' or '1')\n");
2767}
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002768
2769static void show_snapshot_percpu_help(struct seq_file *m)
2770{
2771 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2772#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2773 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2774 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2775#else
2776 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2777 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2778#endif
2779 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2780 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2781 seq_printf(m, "# is not a '0' or '1')\n");
2782}
2783
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002784static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2785{
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05002786 if (iter->tr->allocated_snapshot)
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002787 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2788 else
2789 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2790
2791 seq_printf(m, "# Snapshot commands:\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002792 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2793 show_snapshot_main_help(m);
2794 else
2795 show_snapshot_percpu_help(m);
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002796}
2797#else
2798/* Should never be called */
2799static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2800#endif
2801
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002802static int s_show(struct seq_file *m, void *v)
2803{
2804 struct trace_iterator *iter = v;
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002805 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002806
2807 if (iter->ent == NULL) {
2808 if (iter->tr) {
2809 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2810 seq_puts(m, "#\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002811 test_ftrace_alive(m);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002812 }
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002813 if (iter->snapshot && trace_empty(iter))
2814 print_snapshot_help(m, iter);
2815 else if (iter->trace && iter->trace->print_header)
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002816 iter->trace->print_header(m);
Jiri Olsa62b915f2010-04-02 19:01:22 +02002817 else
2818 trace_default_header(m);
2819
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002820 } else if (iter->leftover) {
2821 /*
2822 * If we filled the seq_file buffer earlier, we
2823 * want to just show it now.
2824 */
2825 ret = trace_print_seq(m, &iter->seq);
2826
2827 /* ret should this time be zero, but you never know */
2828 iter->leftover = ret;
2829
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002830 } else {
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002831 print_trace_line(iter);
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002832 ret = trace_print_seq(m, &iter->seq);
2833 /*
2834 * If we overflow the seq_file buffer, then it will
2835 * ask us for this data again at start up.
2836 * Use that instead.
2837 * ret is 0 if seq_file write succeeded.
2838 * -1 otherwise.
2839 */
2840 iter->leftover = ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002841 }
2842
2843 return 0;
2844}
2845
Oleg Nesterov649e9c72013-07-23 17:25:54 +02002846/*
2847 * Should be used after trace_array_get(), trace_types_lock
2848 * ensures that i_cdev was already initialized.
2849 */
2850static inline int tracing_get_cpu(struct inode *inode)
2851{
2852 if (inode->i_cdev) /* See trace_create_cpu_file() */
2853 return (long)inode->i_cdev - 1;
2854 return RING_BUFFER_ALL_CPUS;
2855}
2856
James Morris88e9d342009-09-22 16:43:43 -07002857static const struct seq_operations tracer_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002858 .start = s_start,
2859 .next = s_next,
2860 .stop = s_stop,
2861 .show = s_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002862};
2863
Ingo Molnare309b412008-05-12 21:20:51 +02002864static struct trace_iterator *
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04002865__tracing_open(struct trace_array *tr, struct trace_cpu *tc,
2866 struct inode *inode, struct file *file, bool snapshot)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002867{
2868 struct trace_iterator *iter;
Jiri Olsa50e18b92012-04-25 10:23:39 +02002869 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002870
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002871 if (tracing_disabled)
2872 return ERR_PTR(-ENODEV);
Steven Rostedt60a11772008-05-12 21:20:44 +02002873
Jiri Olsa50e18b92012-04-25 10:23:39 +02002874 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002875 if (!iter)
2876 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002877
Steven Rostedt6d158a82012-06-27 20:46:14 -04002878 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2879 GFP_KERNEL);
Dan Carpenter93574fc2012-07-11 09:35:08 +03002880 if (!iter->buffer_iter)
2881 goto release;
2882
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002883 /*
2884 * We make a copy of the current tracer to avoid concurrent
2885 * changes on it while we are reading.
2886 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002887 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002888 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002889 if (!iter->trace)
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002890 goto fail;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002891
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002892 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002893
Li Zefan79f55992009-06-15 14:58:26 +08002894 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002895 goto fail;
2896
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002897 iter->tr = tr;
2898
2899#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002900 /* Currently only the top directory has a snapshot */
2901 if (tr->current_trace->print_max || snapshot)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002902 iter->trace_buffer = &tr->max_buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002903 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002904#endif
2905 iter->trace_buffer = &tr->trace_buffer;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002906 iter->snapshot = snapshot;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002907 iter->pos = -1;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002908 mutex_init(&iter->mutex);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002909 iter->cpu_file = tc->cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002910
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002911 /* Notify the tracer early; before we stop tracing. */
2912 if (iter->trace && iter->trace->open)
Markus Metzgera93751c2008-12-11 13:53:26 +01002913 iter->trace->open(iter);
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002914
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002915 /* Annotate start of buffers if we had overruns */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002916 if (ring_buffer_overruns(iter->trace_buffer->buffer))
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002917 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2918
David Sharp8be07092012-11-13 12:18:22 -08002919 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09002920 if (trace_clocks[tr->clock_id].in_ns)
David Sharp8be07092012-11-13 12:18:22 -08002921 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2922
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002923 /* stop the trace while dumping if we are not opening "snapshot" */
2924 if (!iter->snapshot)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002925 tracing_stop_tr(tr);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002926
Steven Rostedtae3b5092013-01-23 15:22:59 -05002927 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002928 for_each_tracing_cpu(cpu) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002929 iter->buffer_iter[cpu] =
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002930 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
David Miller72c9ddf2010-04-20 15:47:11 -07002931 }
2932 ring_buffer_read_prepare_sync();
2933 for_each_tracing_cpu(cpu) {
2934 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002935 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002936 }
2937 } else {
2938 cpu = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002939 iter->buffer_iter[cpu] =
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002940 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
David Miller72c9ddf2010-04-20 15:47:11 -07002941 ring_buffer_read_prepare_sync();
2942 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002943 tracing_iter_reset(iter, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002944 }
2945
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002946 mutex_unlock(&trace_types_lock);
2947
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002948 return iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002949
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002950 fail:
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002951 mutex_unlock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002952 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04002953 kfree(iter->buffer_iter);
Dan Carpenter93574fc2012-07-11 09:35:08 +03002954release:
Jiri Olsa50e18b92012-04-25 10:23:39 +02002955 seq_release_private(inode, file);
2956 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002957}
2958
2959int tracing_open_generic(struct inode *inode, struct file *filp)
2960{
Steven Rostedt60a11772008-05-12 21:20:44 +02002961 if (tracing_disabled)
2962 return -ENODEV;
2963
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002964 filp->private_data = inode->i_private;
2965 return 0;
2966}
2967
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04002968/*
2969 * Open and update trace_array ref count.
2970 * Must have the current trace_array passed to it.
2971 */
Steven Rostedt (Red Hat)dcc30222013-07-02 20:30:52 -04002972static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04002973{
2974 struct trace_array *tr = inode->i_private;
2975
2976 if (tracing_disabled)
2977 return -ENODEV;
2978
2979 if (trace_array_get(tr) < 0)
2980 return -ENODEV;
2981
2982 filp->private_data = inode->i_private;
2983
2984 return 0;
2985
2986}
2987
Steven Rostedt (Red Hat)dcc30222013-07-02 20:30:52 -04002988static int tracing_open_generic_tc(struct inode *inode, struct file *filp)
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04002989{
2990 struct trace_cpu *tc = inode->i_private;
2991 struct trace_array *tr = tc->tr;
2992
2993 if (tracing_disabled)
2994 return -ENODEV;
2995
2996 if (trace_array_get(tr) < 0)
2997 return -ENODEV;
2998
2999 filp->private_data = inode->i_private;
3000
3001 return 0;
3002
3003}
3004
Hannes Eder4fd27352009-02-10 19:44:12 +01003005static int tracing_release(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003006{
matt mooney907f2782010-09-27 19:04:53 -07003007 struct seq_file *m = file->private_data;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003008 struct trace_iterator *iter;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003009 struct trace_array *tr;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003010 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003011
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003012 /* Writes do not use seq_file, need to grab tr from inode */
3013 if (!(file->f_mode & FMODE_READ)) {
3014 struct trace_cpu *tc = inode->i_private;
3015
3016 trace_array_put(tc->tr);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003017 return 0;
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003018 }
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003019
3020 iter = m->private;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003021 tr = iter->tr;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003022
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003023 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05003024
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003025 for_each_tracing_cpu(cpu) {
3026 if (iter->buffer_iter[cpu])
3027 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3028 }
3029
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003030 if (iter->trace && iter->trace->close)
3031 iter->trace->close(iter);
3032
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003033 if (!iter->snapshot)
3034 /* reenable tracing if it was previously enabled */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003035 tracing_start_tr(tr);
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003036
3037 __trace_array_put(tr);
3038
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003039 mutex_unlock(&trace_types_lock);
3040
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003041 mutex_destroy(&iter->mutex);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02003042 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003043 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04003044 kfree(iter->buffer_iter);
Jiri Olsa50e18b92012-04-25 10:23:39 +02003045 seq_release_private(inode, file);
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003046
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003047 return 0;
3048}
3049
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003050static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3051{
3052 struct trace_array *tr = inode->i_private;
3053
3054 trace_array_put(tr);
3055 return 0;
3056}
3057
3058static int tracing_release_generic_tc(struct inode *inode, struct file *file)
3059{
3060 struct trace_cpu *tc = inode->i_private;
3061 struct trace_array *tr = tc->tr;
3062
3063 trace_array_put(tr);
3064 return 0;
3065}
3066
3067static int tracing_single_release_tr(struct inode *inode, struct file *file)
3068{
3069 struct trace_array *tr = inode->i_private;
3070
3071 trace_array_put(tr);
3072
3073 return single_release(inode, file);
3074}
3075
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003076static int tracing_open(struct inode *inode, struct file *file)
3077{
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003078 struct trace_cpu *tc = inode->i_private;
3079 struct trace_array *tr = tc->tr;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003080 struct trace_iterator *iter;
3081 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003082
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003083 if (trace_array_get(tr) < 0)
3084 return -ENODEV;
3085
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003086 /* If this file was open for write, then erase contents */
3087 if ((file->f_mode & FMODE_WRITE) &&
Steven Rostedt8650ae32009-07-22 23:29:30 -04003088 (file->f_flags & O_TRUNC)) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003089 if (tc->cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003090 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003091 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003092 tracing_reset(&tr->trace_buffer, tc->cpu);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003093 }
3094
3095 if (file->f_mode & FMODE_READ) {
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003096 iter = __tracing_open(tr, tc, inode, file, false);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003097 if (IS_ERR(iter))
3098 ret = PTR_ERR(iter);
3099 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3100 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3101 }
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003102
3103 if (ret < 0)
3104 trace_array_put(tr);
3105
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003106 return ret;
3107}
3108
Ingo Molnare309b412008-05-12 21:20:51 +02003109static void *
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003110t_next(struct seq_file *m, void *v, loff_t *pos)
3111{
Li Zefanf129e962009-06-24 09:53:44 +08003112 struct tracer *t = v;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003113
3114 (*pos)++;
3115
3116 if (t)
3117 t = t->next;
3118
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003119 return t;
3120}
3121
3122static void *t_start(struct seq_file *m, loff_t *pos)
3123{
Li Zefanf129e962009-06-24 09:53:44 +08003124 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003125 loff_t l = 0;
3126
3127 mutex_lock(&trace_types_lock);
Li Zefanf129e962009-06-24 09:53:44 +08003128 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003129 ;
3130
3131 return t;
3132}
3133
3134static void t_stop(struct seq_file *m, void *p)
3135{
3136 mutex_unlock(&trace_types_lock);
3137}
3138
3139static int t_show(struct seq_file *m, void *v)
3140{
3141 struct tracer *t = v;
3142
3143 if (!t)
3144 return 0;
3145
3146 seq_printf(m, "%s", t->name);
3147 if (t->next)
3148 seq_putc(m, ' ');
3149 else
3150 seq_putc(m, '\n');
3151
3152 return 0;
3153}
3154
James Morris88e9d342009-09-22 16:43:43 -07003155static const struct seq_operations show_traces_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003156 .start = t_start,
3157 .next = t_next,
3158 .stop = t_stop,
3159 .show = t_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003160};
3161
3162static int show_traces_open(struct inode *inode, struct file *file)
3163{
Steven Rostedt60a11772008-05-12 21:20:44 +02003164 if (tracing_disabled)
3165 return -ENODEV;
3166
Li Zefanf129e962009-06-24 09:53:44 +08003167 return seq_open(file, &show_traces_seq_ops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003168}
3169
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003170static ssize_t
3171tracing_write_stub(struct file *filp, const char __user *ubuf,
3172 size_t count, loff_t *ppos)
3173{
3174 return count;
3175}
3176
Slava Pestov364829b2010-11-24 15:13:16 -08003177static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
3178{
3179 if (file->f_mode & FMODE_READ)
3180 return seq_lseek(file, offset, origin);
3181 else
3182 return 0;
3183}
3184
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003185static const struct file_operations tracing_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003186 .open = tracing_open,
3187 .read = seq_read,
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003188 .write = tracing_write_stub,
Slava Pestov364829b2010-11-24 15:13:16 -08003189 .llseek = tracing_seek,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003190 .release = tracing_release,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003191};
3192
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003193static const struct file_operations show_traces_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003194 .open = show_traces_open,
3195 .read = seq_read,
3196 .release = seq_release,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003197 .llseek = seq_lseek,
Ingo Molnarc7078de2008-05-12 21:20:52 +02003198};
3199
Ingo Molnar36dfe922008-05-12 21:20:52 +02003200/*
3201 * Only trace on a CPU if the bitmask is set:
3202 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303203static cpumask_var_t tracing_cpumask;
Ingo Molnar36dfe922008-05-12 21:20:52 +02003204
3205/*
3206 * The tracer itself will not take this lock, but still we want
3207 * to provide a consistent cpumask to user-space:
3208 */
3209static DEFINE_MUTEX(tracing_cpumask_update_lock);
3210
3211/*
3212 * Temporary storage for the character representation of the
3213 * CPU bitmask (and one more byte for the newline):
3214 */
3215static char mask_str[NR_CPUS + 1];
3216
Ingo Molnarc7078de2008-05-12 21:20:52 +02003217static ssize_t
3218tracing_cpumask_read(struct file *filp, char __user *ubuf,
3219 size_t count, loff_t *ppos)
3220{
Ingo Molnar36dfe922008-05-12 21:20:52 +02003221 int len;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003222
3223 mutex_lock(&tracing_cpumask_update_lock);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003224
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303225 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003226 if (count - len < 2) {
3227 count = -EINVAL;
3228 goto out_err;
3229 }
3230 len += sprintf(mask_str + len, "\n");
3231 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3232
3233out_err:
Ingo Molnarc7078de2008-05-12 21:20:52 +02003234 mutex_unlock(&tracing_cpumask_update_lock);
3235
3236 return count;
3237}
3238
3239static ssize_t
3240tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3241 size_t count, loff_t *ppos)
3242{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003243 struct trace_array *tr = filp->private_data;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303244 cpumask_var_t tracing_cpumask_new;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003245 int err, cpu;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303246
3247 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3248 return -ENOMEM;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003249
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303250 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003251 if (err)
3252 goto err_unlock;
3253
Li Zefan215368e2009-06-15 10:56:42 +08003254 mutex_lock(&tracing_cpumask_update_lock);
3255
Steven Rostedta5e25882008-12-02 15:34:05 -05003256 local_irq_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01003257 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtab464282008-05-12 21:21:00 +02003258 for_each_tracing_cpu(cpu) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02003259 /*
3260 * Increase/decrease the disabled counter if we are
3261 * about to flip a bit in the cpumask:
3262 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303263 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
3264 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003265 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3266 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003267 }
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303268 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
3269 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003270 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3271 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003272 }
3273 }
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01003274 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedta5e25882008-12-02 15:34:05 -05003275 local_irq_enable();
Ingo Molnar36dfe922008-05-12 21:20:52 +02003276
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303277 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003278
Ingo Molnarc7078de2008-05-12 21:20:52 +02003279 mutex_unlock(&tracing_cpumask_update_lock);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303280 free_cpumask_var(tracing_cpumask_new);
Ingo Molnarc7078de2008-05-12 21:20:52 +02003281
Ingo Molnarc7078de2008-05-12 21:20:52 +02003282 return count;
Ingo Molnar36dfe922008-05-12 21:20:52 +02003283
3284err_unlock:
Li Zefan215368e2009-06-15 10:56:42 +08003285 free_cpumask_var(tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003286
3287 return err;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003288}
3289
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003290static const struct file_operations tracing_cpumask_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003291 .open = tracing_open_generic,
3292 .read = tracing_cpumask_read,
3293 .write = tracing_cpumask_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003294 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003295};
3296
Li Zefanfdb372e2009-12-08 11:15:59 +08003297static int tracing_trace_options_show(struct seq_file *m, void *v)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003298{
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003299 struct tracer_opt *trace_opts;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003300 struct trace_array *tr = m->private;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003301 u32 tracer_flags;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003302 int i;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003303
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003304 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003305 tracer_flags = tr->current_trace->flags->val;
3306 trace_opts = tr->current_trace->flags->opts;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003307
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003308 for (i = 0; trace_options[i]; i++) {
3309 if (trace_flags & (1 << i))
Li Zefanfdb372e2009-12-08 11:15:59 +08003310 seq_printf(m, "%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003311 else
Li Zefanfdb372e2009-12-08 11:15:59 +08003312 seq_printf(m, "no%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003313 }
3314
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003315 for (i = 0; trace_opts[i].name; i++) {
3316 if (tracer_flags & trace_opts[i].bit)
Li Zefanfdb372e2009-12-08 11:15:59 +08003317 seq_printf(m, "%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003318 else
Li Zefanfdb372e2009-12-08 11:15:59 +08003319 seq_printf(m, "no%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003320 }
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003321 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003322
Li Zefanfdb372e2009-12-08 11:15:59 +08003323 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003324}
3325
Li Zefan8d18eaa2009-12-08 11:17:06 +08003326static int __set_tracer_option(struct tracer *trace,
3327 struct tracer_flags *tracer_flags,
3328 struct tracer_opt *opts, int neg)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003329{
Li Zefan8d18eaa2009-12-08 11:17:06 +08003330 int ret;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003331
Li Zefan8d18eaa2009-12-08 11:17:06 +08003332 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003333 if (ret)
3334 return ret;
3335
3336 if (neg)
Zhaolei77708412009-08-07 18:53:21 +08003337 tracer_flags->val &= ~opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003338 else
Zhaolei77708412009-08-07 18:53:21 +08003339 tracer_flags->val |= opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003340 return 0;
3341}
3342
Li Zefan8d18eaa2009-12-08 11:17:06 +08003343/* Try to assign a tracer specific option */
3344static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3345{
3346 struct tracer_flags *tracer_flags = trace->flags;
3347 struct tracer_opt *opts = NULL;
3348 int i;
3349
3350 for (i = 0; tracer_flags->opts[i].name; i++) {
3351 opts = &tracer_flags->opts[i];
3352
3353 if (strcmp(cmp, opts->name) == 0)
3354 return __set_tracer_option(trace, trace->flags,
3355 opts, neg);
3356 }
3357
3358 return -EINVAL;
3359}
3360
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003361/* Some tracers require overwrite to stay enabled */
3362int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3363{
3364 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3365 return -1;
3366
3367 return 0;
3368}
3369
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003370int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003371{
3372 /* do nothing if flag is already set */
3373 if (!!(trace_flags & mask) == !!enabled)
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003374 return 0;
3375
3376 /* Give the tracer a chance to approve the change */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003377 if (tr->current_trace->flag_changed)
3378 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003379 return -EINVAL;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003380
3381 if (enabled)
3382 trace_flags |= mask;
3383 else
3384 trace_flags &= ~mask;
Li Zefane870e9a2010-07-02 11:07:32 +08003385
3386 if (mask == TRACE_ITER_RECORD_CMD)
3387 trace_event_enable_cmd_record(enabled);
David Sharp750912f2010-12-08 13:46:47 -08003388
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003389 if (mask == TRACE_ITER_OVERWRITE) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003390 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003391#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003392 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003393#endif
3394 }
Steven Rostedt81698832012-10-11 10:15:05 -04003395
3396 if (mask == TRACE_ITER_PRINTK)
3397 trace_printk_start_stop_comm(enabled);
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003398
3399 return 0;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003400}
3401
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003402static int trace_set_options(struct trace_array *tr, char *option)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003403{
Li Zefan8d18eaa2009-12-08 11:17:06 +08003404 char *cmp;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003405 int neg = 0;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003406 int ret = -ENODEV;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003407 int i;
3408
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003409 cmp = strstrip(option);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003410
Li Zefan8d18eaa2009-12-08 11:17:06 +08003411 if (strncmp(cmp, "no", 2) == 0) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003412 neg = 1;
3413 cmp += 2;
3414 }
3415
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003416 mutex_lock(&trace_types_lock);
3417
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003418 for (i = 0; trace_options[i]; i++) {
Li Zefan8d18eaa2009-12-08 11:17:06 +08003419 if (strcmp(cmp, trace_options[i]) == 0) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003420 ret = set_tracer_flag(tr, 1 << i, !neg);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003421 break;
3422 }
3423 }
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003424
3425 /* If no option could be set, test the specific tracer options */
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003426 if (!trace_options[i])
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003427 ret = set_tracer_option(tr->current_trace, cmp, neg);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003428
3429 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003430
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003431 return ret;
3432}
3433
3434static ssize_t
3435tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3436 size_t cnt, loff_t *ppos)
3437{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003438 struct seq_file *m = filp->private_data;
3439 struct trace_array *tr = m->private;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003440 char buf[64];
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003441 int ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003442
3443 if (cnt >= sizeof(buf))
3444 return -EINVAL;
3445
3446 if (copy_from_user(&buf, ubuf, cnt))
3447 return -EFAULT;
3448
Steven Rostedta8dd2172013-01-09 20:54:17 -05003449 buf[cnt] = 0;
3450
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003451 ret = trace_set_options(tr, buf);
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003452 if (ret < 0)
3453 return ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003454
Jiri Olsacf8517c2009-10-23 19:36:16 -04003455 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003456
3457 return cnt;
3458}
3459
Li Zefanfdb372e2009-12-08 11:15:59 +08003460static int tracing_trace_options_open(struct inode *inode, struct file *file)
3461{
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003462 struct trace_array *tr = inode->i_private;
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003463 int ret;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003464
Li Zefanfdb372e2009-12-08 11:15:59 +08003465 if (tracing_disabled)
3466 return -ENODEV;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003467
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003468 if (trace_array_get(tr) < 0)
3469 return -ENODEV;
3470
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003471 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3472 if (ret < 0)
3473 trace_array_put(tr);
3474
3475 return ret;
Li Zefanfdb372e2009-12-08 11:15:59 +08003476}
3477
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003478static const struct file_operations tracing_iter_fops = {
Li Zefanfdb372e2009-12-08 11:15:59 +08003479 .open = tracing_trace_options_open,
3480 .read = seq_read,
3481 .llseek = seq_lseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003482 .release = tracing_single_release_tr,
Steven Rostedtee6bce52008-11-12 17:52:37 -05003483 .write = tracing_trace_options_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003484};
3485
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003486static const char readme_msg[] =
3487 "tracing mini-HOWTO:\n\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003488 "# echo 0 > tracing_on : quick way to disable tracing\n"
3489 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3490 " Important files:\n"
3491 " trace\t\t\t- The static contents of the buffer\n"
3492 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3493 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3494 " current_tracer\t- function and latency tracers\n"
3495 " available_tracers\t- list of configured tracers for current_tracer\n"
3496 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3497 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3498 " trace_clock\t\t-change the clock used to order events\n"
3499 " local: Per cpu clock but may not be synced across CPUs\n"
3500 " global: Synced across CPUs but slows tracing down.\n"
3501 " counter: Not a clock, but just an increment\n"
3502 " uptime: Jiffy counter from time of boot\n"
3503 " perf: Same clock that perf events use\n"
3504#ifdef CONFIG_X86_64
3505 " x86-tsc: TSC cycle counter\n"
3506#endif
3507 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3508 " tracing_cpumask\t- Limit which CPUs to trace\n"
3509 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3510 "\t\t\t Remove sub-buffer with rmdir\n"
3511 " trace_options\t\t- Set format or modify how tracing happens\n"
3512 "\t\t\t Disable an option by adding a suffix 'no' to the option name\n"
3513#ifdef CONFIG_DYNAMIC_FTRACE
3514 "\n available_filter_functions - list of functions that can be filtered on\n"
3515 " set_ftrace_filter\t- echo function name in here to only trace these functions\n"
3516 " accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3517 " modules: Can select a group via module\n"
3518 " Format: :mod:<module-name>\n"
3519 " example: echo :mod:ext3 > set_ftrace_filter\n"
3520 " triggers: a command to perform when function is hit\n"
3521 " Format: <function>:<trigger>[:count]\n"
3522 " trigger: traceon, traceoff\n"
3523 " enable_event:<system>:<event>\n"
3524 " disable_event:<system>:<event>\n"
3525#ifdef CONFIG_STACKTRACE
3526 " stacktrace\n"
3527#endif
3528#ifdef CONFIG_TRACER_SNAPSHOT
3529 " snapshot\n"
3530#endif
3531 " example: echo do_fault:traceoff > set_ftrace_filter\n"
3532 " echo do_trap:traceoff:3 > set_ftrace_filter\n"
3533 " The first one will disable tracing every time do_fault is hit\n"
3534 " The second will disable tracing at most 3 times when do_trap is hit\n"
3535 " The first time do trap is hit and it disables tracing, the counter\n"
3536 " will decrement to 2. If tracing is already disabled, the counter\n"
3537 " will not decrement. It only decrements when the trigger did work\n"
3538 " To remove trigger without count:\n"
3539 " echo '!<function>:<trigger> > set_ftrace_filter\n"
3540 " To remove trigger with a count:\n"
3541 " echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3542 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3543 " accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3544 " modules: Can select a group via module command :mod:\n"
3545 " Does not accept triggers\n"
3546#endif /* CONFIG_DYNAMIC_FTRACE */
3547#ifdef CONFIG_FUNCTION_TRACER
3548 " set_ftrace_pid\t- Write pid(s) to only function trace those pids (function)\n"
3549#endif
3550#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3551 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3552 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3553#endif
3554#ifdef CONFIG_TRACER_SNAPSHOT
3555 "\n snapshot\t\t- Like 'trace' but shows the content of the static snapshot buffer\n"
3556 "\t\t\t Read the contents for more information\n"
3557#endif
zhangwei(Jovi)991821c2013-07-15 16:32:34 +08003558#ifdef CONFIG_STACK_TRACER
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003559 " stack_trace\t\t- Shows the max stack trace when active\n"
3560 " stack_max_size\t- Shows current max stack size that was traced\n"
3561 "\t\t\t Write into this file to reset the max size (trigger a new trace)\n"
3562#ifdef CONFIG_DYNAMIC_FTRACE
3563 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace traces\n"
3564#endif
zhangwei(Jovi)991821c2013-07-15 16:32:34 +08003565#endif /* CONFIG_STACK_TRACER */
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003566;
3567
3568static ssize_t
3569tracing_readme_read(struct file *filp, char __user *ubuf,
3570 size_t cnt, loff_t *ppos)
3571{
3572 return simple_read_from_buffer(ubuf, cnt, ppos,
3573 readme_msg, strlen(readme_msg));
3574}
3575
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003576static const struct file_operations tracing_readme_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003577 .open = tracing_open_generic,
3578 .read = tracing_readme_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003579 .llseek = generic_file_llseek,
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003580};
3581
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003582static ssize_t
Avadh Patel69abe6a2009-04-10 16:04:48 -04003583tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3584 size_t cnt, loff_t *ppos)
3585{
3586 char *buf_comm;
3587 char *file_buf;
3588 char *buf;
3589 int len = 0;
3590 int pid;
3591 int i;
3592
3593 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3594 if (!file_buf)
3595 return -ENOMEM;
3596
3597 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3598 if (!buf_comm) {
3599 kfree(file_buf);
3600 return -ENOMEM;
3601 }
3602
3603 buf = file_buf;
3604
3605 for (i = 0; i < SAVED_CMDLINES; i++) {
3606 int r;
3607
3608 pid = map_cmdline_to_pid[i];
3609 if (pid == -1 || pid == NO_CMDLINE_MAP)
3610 continue;
3611
3612 trace_find_cmdline(pid, buf_comm);
3613 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3614 buf += r;
3615 len += r;
3616 }
3617
3618 len = simple_read_from_buffer(ubuf, cnt, ppos,
3619 file_buf, len);
3620
3621 kfree(file_buf);
3622 kfree(buf_comm);
3623
3624 return len;
3625}
3626
3627static const struct file_operations tracing_saved_cmdlines_fops = {
3628 .open = tracing_open_generic,
3629 .read = tracing_saved_cmdlines_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003630 .llseek = generic_file_llseek,
Avadh Patel69abe6a2009-04-10 16:04:48 -04003631};
3632
3633static ssize_t
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003634tracing_set_trace_read(struct file *filp, char __user *ubuf,
3635 size_t cnt, loff_t *ppos)
3636{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003637 struct trace_array *tr = filp->private_data;
Li Zefanee6c2c12009-09-18 14:06:47 +08003638 char buf[MAX_TRACER_SIZE+2];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003639 int r;
3640
3641 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003642 r = sprintf(buf, "%s\n", tr->current_trace->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003643 mutex_unlock(&trace_types_lock);
3644
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003645 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003646}
3647
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003648int tracer_init(struct tracer *t, struct trace_array *tr)
3649{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003650 tracing_reset_online_cpus(&tr->trace_buffer);
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003651 return t->init(tr);
3652}
3653
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003654static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003655{
3656 int cpu;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05003657
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003658 for_each_tracing_cpu(cpu)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003659 per_cpu_ptr(buf->data, cpu)->entries = val;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003660}
3661
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003662#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookad60da502012-10-17 11:56:16 +09003663/* resize @tr's buffer to the size of @size_tr's entries */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003664static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3665 struct trace_buffer *size_buf, int cpu_id)
Hiraku Toyookad60da502012-10-17 11:56:16 +09003666{
3667 int cpu, ret = 0;
3668
3669 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3670 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003671 ret = ring_buffer_resize(trace_buf->buffer,
3672 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
Hiraku Toyookad60da502012-10-17 11:56:16 +09003673 if (ret < 0)
3674 break;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003675 per_cpu_ptr(trace_buf->data, cpu)->entries =
3676 per_cpu_ptr(size_buf->data, cpu)->entries;
Hiraku Toyookad60da502012-10-17 11:56:16 +09003677 }
3678 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003679 ret = ring_buffer_resize(trace_buf->buffer,
3680 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
Hiraku Toyookad60da502012-10-17 11:56:16 +09003681 if (ret == 0)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003682 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3683 per_cpu_ptr(size_buf->data, cpu_id)->entries;
Hiraku Toyookad60da502012-10-17 11:56:16 +09003684 }
3685
3686 return ret;
3687}
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003688#endif /* CONFIG_TRACER_MAX_TRACE */
Hiraku Toyookad60da502012-10-17 11:56:16 +09003689
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003690static int __tracing_resize_ring_buffer(struct trace_array *tr,
3691 unsigned long size, int cpu)
Steven Rostedt73c51622009-03-11 13:42:01 -04003692{
3693 int ret;
3694
3695 /*
3696 * If kernel or user changes the size of the ring buffer
Steven Rostedta123c522009-03-12 11:21:08 -04003697 * we use the size that was given, and we can forget about
3698 * expanding it later.
Steven Rostedt73c51622009-03-11 13:42:01 -04003699 */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05003700 ring_buffer_expanded = true;
Steven Rostedt73c51622009-03-11 13:42:01 -04003701
Steven Rostedtb382ede62012-10-10 21:44:34 -04003702 /* May be called before buffers are initialized */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003703 if (!tr->trace_buffer.buffer)
Steven Rostedtb382ede62012-10-10 21:44:34 -04003704 return 0;
3705
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003706 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003707 if (ret < 0)
3708 return ret;
3709
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003710#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003711 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3712 !tr->current_trace->use_max_tr)
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003713 goto out;
3714
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003715 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003716 if (ret < 0) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003717 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3718 &tr->trace_buffer, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003719 if (r < 0) {
Steven Rostedta123c522009-03-12 11:21:08 -04003720 /*
3721 * AARGH! We are left with different
3722 * size max buffer!!!!
3723 * The max buffer is our "snapshot" buffer.
3724 * When a tracer needs a snapshot (one of the
3725 * latency tracers), it swaps the max buffer
3726 * with the saved snap shot. We succeeded to
3727 * update the size of the main buffer, but failed to
3728 * update the size of the max buffer. But when we tried
3729 * to reset the main buffer to the original size, we
3730 * failed there too. This is very unlikely to
3731 * happen, but if it does, warn and kill all
3732 * tracing.
3733 */
Steven Rostedt73c51622009-03-11 13:42:01 -04003734 WARN_ON(1);
3735 tracing_disabled = 1;
3736 }
3737 return ret;
3738 }
3739
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003740 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003741 set_buffer_entries(&tr->max_buffer, size);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003742 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003743 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003744
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003745 out:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003746#endif /* CONFIG_TRACER_MAX_TRACE */
3747
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003748 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003749 set_buffer_entries(&tr->trace_buffer, size);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003750 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003751 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
Steven Rostedt73c51622009-03-11 13:42:01 -04003752
3753 return ret;
3754}
3755
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003756static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3757 unsigned long size, int cpu_id)
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003758{
Vaibhav Nagarnaik83f40312012-05-03 18:59:50 -07003759 int ret = size;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003760
3761 mutex_lock(&trace_types_lock);
3762
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003763 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3764 /* make sure, this cpu is enabled in the mask */
3765 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3766 ret = -EINVAL;
3767 goto out;
3768 }
3769 }
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003770
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003771 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003772 if (ret < 0)
3773 ret = -ENOMEM;
3774
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003775out:
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003776 mutex_unlock(&trace_types_lock);
3777
3778 return ret;
3779}
3780
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003781
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003782/**
3783 * tracing_update_buffers - used by tracing facility to expand ring buffers
3784 *
3785 * To save on memory when the tracing is never used on a system with it
3786 * configured in. The ring buffers are set to a minimum size. But once
3787 * a user starts to use the tracing facility, then they need to grow
3788 * to their default size.
3789 *
3790 * This function is to be called when a tracer is about to be used.
3791 */
3792int tracing_update_buffers(void)
3793{
3794 int ret = 0;
3795
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003796 mutex_lock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003797 if (!ring_buffer_expanded)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003798 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003799 RING_BUFFER_ALL_CPUS);
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003800 mutex_unlock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003801
3802 return ret;
3803}
3804
Steven Rostedt577b7852009-02-26 23:43:05 -05003805struct trace_option_dentry;
3806
3807static struct trace_option_dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003808create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
Steven Rostedt577b7852009-02-26 23:43:05 -05003809
3810static void
3811destroy_trace_option_files(struct trace_option_dentry *topts);
3812
Steven Rostedtb2821ae2009-02-02 21:38:32 -05003813static int tracing_set_tracer(const char *buf)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003814{
Steven Rostedt577b7852009-02-26 23:43:05 -05003815 static struct trace_option_dentry *topts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003816 struct trace_array *tr = &global_trace;
3817 struct tracer *t;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003818#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt34600f02013-01-22 13:35:11 -05003819 bool had_max_tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003820#endif
Peter Zijlstrad9e54072008-11-01 19:57:37 +01003821 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003822
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003823 mutex_lock(&trace_types_lock);
3824
Steven Rostedt73c51622009-03-11 13:42:01 -04003825 if (!ring_buffer_expanded) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003826 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003827 RING_BUFFER_ALL_CPUS);
Steven Rostedt73c51622009-03-11 13:42:01 -04003828 if (ret < 0)
Frederic Weisbecker59f586d2009-03-15 22:10:39 +01003829 goto out;
Steven Rostedt73c51622009-03-11 13:42:01 -04003830 ret = 0;
3831 }
3832
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003833 for (t = trace_types; t; t = t->next) {
3834 if (strcmp(t->name, buf) == 0)
3835 break;
3836 }
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02003837 if (!t) {
3838 ret = -EINVAL;
3839 goto out;
3840 }
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003841 if (t == tr->current_trace)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003842 goto out;
3843
Steven Rostedt9f029e82008-11-12 15:24:24 -05003844 trace_branch_disable();
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003845
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003846 tr->current_trace->enabled = false;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003847
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003848 if (tr->current_trace->reset)
3849 tr->current_trace->reset(tr);
Steven Rostedt34600f02013-01-22 13:35:11 -05003850
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003851 /* Current trace needs to be nop_trace before synchronize_sched */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003852 tr->current_trace = &nop_trace;
Steven Rostedt34600f02013-01-22 13:35:11 -05003853
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05003854#ifdef CONFIG_TRACER_MAX_TRACE
3855 had_max_tr = tr->allocated_snapshot;
Steven Rostedt34600f02013-01-22 13:35:11 -05003856
3857 if (had_max_tr && !t->use_max_tr) {
3858 /*
3859 * We need to make sure that the update_max_tr sees that
3860 * current_trace changed to nop_trace to keep it from
3861 * swapping the buffers after we resize it.
3862 * The update_max_tr is called from interrupts disabled
3863 * so a synchronized_sched() is sufficient.
3864 */
3865 synchronize_sched();
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04003866 free_snapshot(tr);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003867 }
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003868#endif
Steven Rostedt577b7852009-02-26 23:43:05 -05003869 destroy_trace_option_files(topts);
3870
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003871 topts = create_trace_option_files(tr, t);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003872
3873#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt34600f02013-01-22 13:35:11 -05003874 if (t->use_max_tr && !had_max_tr) {
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04003875 ret = alloc_snapshot(tr);
Hiraku Toyookad60da502012-10-17 11:56:16 +09003876 if (ret < 0)
3877 goto out;
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003878 }
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003879#endif
Steven Rostedt577b7852009-02-26 23:43:05 -05003880
Frederic Weisbecker1c800252008-11-16 05:57:26 +01003881 if (t->init) {
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003882 ret = tracer_init(t, tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +01003883 if (ret)
3884 goto out;
3885 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003886
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003887 tr->current_trace = t;
3888 tr->current_trace->enabled = true;
Steven Rostedt9f029e82008-11-12 15:24:24 -05003889 trace_branch_enable(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003890 out:
3891 mutex_unlock(&trace_types_lock);
3892
Peter Zijlstrad9e54072008-11-01 19:57:37 +01003893 return ret;
3894}
3895
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003896static ssize_t
3897tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3898 size_t cnt, loff_t *ppos)
3899{
Li Zefanee6c2c12009-09-18 14:06:47 +08003900 char buf[MAX_TRACER_SIZE+1];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003901 int i;
3902 size_t ret;
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01003903 int err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003904
Steven Rostedt60063a62008-10-28 10:44:24 -04003905 ret = cnt;
3906
Li Zefanee6c2c12009-09-18 14:06:47 +08003907 if (cnt > MAX_TRACER_SIZE)
3908 cnt = MAX_TRACER_SIZE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003909
3910 if (copy_from_user(&buf, ubuf, cnt))
3911 return -EFAULT;
3912
3913 buf[cnt] = 0;
3914
3915 /* strip ending whitespace. */
3916 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3917 buf[i] = 0;
3918
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01003919 err = tracing_set_tracer(buf);
3920 if (err)
3921 return err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003922
Jiri Olsacf8517c2009-10-23 19:36:16 -04003923 *ppos += ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003924
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02003925 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003926}
3927
3928static ssize_t
3929tracing_max_lat_read(struct file *filp, char __user *ubuf,
3930 size_t cnt, loff_t *ppos)
3931{
3932 unsigned long *ptr = filp->private_data;
3933 char buf[64];
3934 int r;
3935
Steven Rostedtcffae432008-05-12 21:21:00 +02003936 r = snprintf(buf, sizeof(buf), "%ld\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003937 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
Steven Rostedtcffae432008-05-12 21:21:00 +02003938 if (r > sizeof(buf))
3939 r = sizeof(buf);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003940 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003941}
3942
3943static ssize_t
3944tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3945 size_t cnt, loff_t *ppos)
3946{
Hannes Eder5e398412009-02-10 19:44:34 +01003947 unsigned long *ptr = filp->private_data;
Hannes Eder5e398412009-02-10 19:44:34 +01003948 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003949 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003950
Peter Huewe22fe9b52011-06-07 21:58:27 +02003951 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3952 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003953 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003954
3955 *ptr = val * 1000;
3956
3957 return cnt;
3958}
3959
Steven Rostedtb3806b42008-05-12 21:20:46 +02003960static int tracing_open_pipe(struct inode *inode, struct file *filp)
3961{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003962 struct trace_cpu *tc = inode->i_private;
3963 struct trace_array *tr = tc->tr;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003964 struct trace_iterator *iter;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003965 int ret = 0;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003966
3967 if (tracing_disabled)
3968 return -ENODEV;
3969
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003970 if (trace_array_get(tr) < 0)
3971 return -ENODEV;
3972
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003973 mutex_lock(&trace_types_lock);
3974
Steven Rostedtb3806b42008-05-12 21:20:46 +02003975 /* create a buffer to store the information to pass to userspace */
3976 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003977 if (!iter) {
3978 ret = -ENOMEM;
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003979 __trace_array_put(tr);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003980 goto out;
3981 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003982
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003983 /*
3984 * We make a copy of the current tracer to avoid concurrent
3985 * changes on it while we are reading.
3986 */
3987 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3988 if (!iter->trace) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003989 ret = -ENOMEM;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003990 goto fail;
3991 }
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003992 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003993
3994 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3995 ret = -ENOMEM;
3996 goto fail;
Rusty Russell44623442009-01-01 10:12:23 +10303997 }
3998
Steven Rostedta3097202008-11-07 22:36:02 -05003999 /* trace pipe does not show start of buffer */
Rusty Russell44623442009-01-01 10:12:23 +10304000 cpumask_setall(iter->started);
Steven Rostedta3097202008-11-07 22:36:02 -05004001
Steven Rostedt112f38a72009-06-01 15:16:05 -04004002 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4003 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4004
David Sharp8be07092012-11-13 12:18:22 -08004005 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09004006 if (trace_clocks[tr->clock_id].in_ns)
David Sharp8be07092012-11-13 12:18:22 -08004007 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4008
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004009 iter->cpu_file = tc->cpu;
4010 iter->tr = tc->tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004011 iter->trace_buffer = &tc->tr->trace_buffer;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004012 mutex_init(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004013 filp->private_data = iter;
4014
Steven Rostedt107bad82008-05-12 21:21:01 +02004015 if (iter->trace->pipe_open)
4016 iter->trace->pipe_open(iter);
Steven Rostedt107bad82008-05-12 21:21:01 +02004017
Arnd Bergmannb4447862010-07-07 23:40:11 +02004018 nonseekable_open(inode, filp);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004019out:
4020 mutex_unlock(&trace_types_lock);
4021 return ret;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004022
4023fail:
4024 kfree(iter->trace);
4025 kfree(iter);
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004026 __trace_array_put(tr);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004027 mutex_unlock(&trace_types_lock);
4028 return ret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004029}
4030
4031static int tracing_release_pipe(struct inode *inode, struct file *file)
4032{
4033 struct trace_iterator *iter = file->private_data;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004034 struct trace_cpu *tc = inode->i_private;
4035 struct trace_array *tr = tc->tr;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004036
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004037 mutex_lock(&trace_types_lock);
4038
Steven Rostedt29bf4a52009-12-09 12:37:43 -05004039 if (iter->trace->pipe_close)
Steven Rostedtc521efd2009-12-07 09:06:24 -05004040 iter->trace->pipe_close(iter);
4041
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004042 mutex_unlock(&trace_types_lock);
4043
Rusty Russell44623442009-01-01 10:12:23 +10304044 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004045 mutex_destroy(&iter->mutex);
4046 kfree(iter->trace);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004047 kfree(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004048
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004049 trace_array_put(tr);
4050
Steven Rostedtb3806b42008-05-12 21:20:46 +02004051 return 0;
4052}
4053
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004054static unsigned int
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004055trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004056{
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004057 /* Iterators are static, they should be filled or empty */
4058 if (trace_buffer_iter(iter, iter->cpu_file))
4059 return POLLIN | POLLRDNORM;
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004060
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004061 if (trace_flags & TRACE_ITER_BLOCK)
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004062 /*
4063 * Always select as readable when in blocking mode
4064 */
4065 return POLLIN | POLLRDNORM;
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004066 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004067 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004068 filp, poll_table);
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004069}
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004070
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004071static unsigned int
4072tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4073{
4074 struct trace_iterator *iter = filp->private_data;
4075
4076 return trace_poll(iter, filp, poll_table);
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004077}
4078
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01004079/*
4080 * This is a make-shift waitqueue.
4081 * A tracer might use this callback on some rare cases:
4082 *
4083 * 1) the current tracer might hold the runqueue lock when it wakes up
4084 * a reader, hence a deadlock (sched, function, and function graph tracers)
4085 * 2) the function tracers, trace all functions, we don't want
4086 * the overhead of calling wake_up and friends
4087 * (and tracing them too)
4088 *
4089 * Anyway, this is really very primitive wakeup.
4090 */
4091void poll_wait_pipe(struct trace_iterator *iter)
4092{
4093 set_current_state(TASK_INTERRUPTIBLE);
4094 /* sleep for 100 msecs, and try again. */
4095 schedule_timeout(HZ / 10);
4096}
4097
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004098/* Must be called with trace_types_lock mutex held. */
4099static int tracing_wait_pipe(struct file *filp)
4100{
4101 struct trace_iterator *iter = filp->private_data;
4102
4103 while (trace_empty(iter)) {
4104
4105 if ((filp->f_flags & O_NONBLOCK)) {
4106 return -EAGAIN;
4107 }
4108
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004109 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004110
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01004111 iter->trace->wait_pipe(iter);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004112
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004113 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004114
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01004115 if (signal_pending(current))
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004116 return -EINTR;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004117
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004118 /*
Liu Bo250bfd32013-01-14 10:54:11 +08004119 * We block until we read something and tracing is disabled.
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004120 * We still block if tracing is disabled, but we have never
4121 * read anything. This allows a user to cat this file, and
4122 * then enable tracing. But after we have read something,
4123 * we give an EOF when tracing is again disabled.
4124 *
4125 * iter->pos will be 0 if we haven't read anything.
4126 */
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04004127 if (!tracing_is_on() && iter->pos)
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004128 break;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004129 }
4130
4131 return 1;
4132}
4133
Steven Rostedtb3806b42008-05-12 21:20:46 +02004134/*
4135 * Consumer reader.
4136 */
4137static ssize_t
4138tracing_read_pipe(struct file *filp, char __user *ubuf,
4139 size_t cnt, loff_t *ppos)
4140{
4141 struct trace_iterator *iter = filp->private_data;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004142 struct trace_array *tr = iter->tr;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004143 ssize_t sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004144
4145 /* return any leftover data */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004146 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4147 if (sret != -EBUSY)
4148 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004149
Steven Rostedtf9520752009-03-02 14:04:40 -05004150 trace_seq_init(&iter->seq);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004151
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004152 /* copy the tracer to avoid using a global lock all around */
Steven Rostedt107bad82008-05-12 21:21:01 +02004153 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004154 if (unlikely(iter->trace->name != tr->current_trace->name))
4155 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004156 mutex_unlock(&trace_types_lock);
4157
4158 /*
4159 * Avoid more than one consumer on a single file descriptor
4160 * This is just a matter of traces coherency, the ring buffer itself
4161 * is protected.
4162 */
4163 mutex_lock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02004164 if (iter->trace->read) {
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004165 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4166 if (sret)
Steven Rostedt107bad82008-05-12 21:21:01 +02004167 goto out;
Steven Rostedt107bad82008-05-12 21:21:01 +02004168 }
4169
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004170waitagain:
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004171 sret = tracing_wait_pipe(filp);
4172 if (sret <= 0)
4173 goto out;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004174
4175 /* stop when tracing is finished */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004176 if (trace_empty(iter)) {
4177 sret = 0;
Steven Rostedt107bad82008-05-12 21:21:01 +02004178 goto out;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004179 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02004180
4181 if (cnt >= PAGE_SIZE)
4182 cnt = PAGE_SIZE - 1;
4183
Steven Rostedt53d0aa72008-05-12 21:21:01 +02004184 /* reset all but tr, trace, and overruns */
Steven Rostedt53d0aa72008-05-12 21:21:01 +02004185 memset(&iter->seq, 0,
4186 sizeof(struct trace_iterator) -
4187 offsetof(struct trace_iterator, seq));
Steven Rostedt4823ed72008-05-12 21:21:01 +02004188 iter->pos = -1;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004189
Lai Jiangshan4f535962009-05-18 19:35:34 +08004190 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004191 trace_access_lock(iter->cpu_file);
Jason Wessel955b61e2010-08-05 09:22:23 -05004192 while (trace_find_next_entry_inc(iter) != NULL) {
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02004193 enum print_line_t ret;
Steven Rostedt088b1e422008-05-12 21:20:48 +02004194 int len = iter->seq.len;
4195
Ingo Molnarf9896bf2008-05-12 21:20:47 +02004196 ret = print_trace_line(iter);
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02004197 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt088b1e422008-05-12 21:20:48 +02004198 /* don't print partial lines */
4199 iter->seq.len = len;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004200 break;
Steven Rostedt088b1e422008-05-12 21:20:48 +02004201 }
Frederic Weisbeckerb91facc2009-02-06 18:30:44 +01004202 if (ret != TRACE_TYPE_NO_CONSUME)
4203 trace_consume(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004204
4205 if (iter->seq.len >= cnt)
4206 break;
Jiri Olsaee5e51f2011-03-25 12:05:18 +01004207
4208 /*
4209 * Setting the full flag means we reached the trace_seq buffer
4210 * size and we should leave by partial output condition above.
4211 * One of the trace_seq_* functions is not used properly.
4212 */
4213 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4214 iter->ent->type);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004215 }
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004216 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004217 trace_event_read_unlock();
Steven Rostedtb3806b42008-05-12 21:20:46 +02004218
Steven Rostedtb3806b42008-05-12 21:20:46 +02004219 /* Now copy what we have to the user */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004220 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4221 if (iter->seq.readpos >= iter->seq.len)
Steven Rostedtf9520752009-03-02 14:04:40 -05004222 trace_seq_init(&iter->seq);
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004223
4224 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004225 * If there was nothing to send to user, in spite of consuming trace
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004226 * entries, go back to wait for more entries.
4227 */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004228 if (sret == -EBUSY)
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004229 goto waitagain;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004230
Steven Rostedt107bad82008-05-12 21:21:01 +02004231out:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004232 mutex_unlock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02004233
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004234 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004235}
4236
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004237static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
4238 struct pipe_buffer *buf)
4239{
4240 __free_page(buf->page);
4241}
4242
4243static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4244 unsigned int idx)
4245{
4246 __free_page(spd->pages[idx]);
4247}
4248
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08004249static const struct pipe_buf_operations tracing_pipe_buf_ops = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05004250 .can_merge = 0,
4251 .map = generic_pipe_buf_map,
4252 .unmap = generic_pipe_buf_unmap,
4253 .confirm = generic_pipe_buf_confirm,
4254 .release = tracing_pipe_buf_release,
4255 .steal = generic_pipe_buf_steal,
4256 .get = generic_pipe_buf_get,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004257};
4258
Steven Rostedt34cd4992009-02-09 12:06:29 -05004259static size_t
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01004260tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004261{
4262 size_t count;
4263 int ret;
4264
4265 /* Seq buffer is page-sized, exactly what we need. */
4266 for (;;) {
4267 count = iter->seq.len;
4268 ret = print_trace_line(iter);
4269 count = iter->seq.len - count;
4270 if (rem < count) {
4271 rem = 0;
4272 iter->seq.len -= count;
4273 break;
4274 }
4275 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4276 iter->seq.len -= count;
4277 break;
4278 }
4279
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08004280 if (ret != TRACE_TYPE_NO_CONSUME)
4281 trace_consume(iter);
Steven Rostedt34cd4992009-02-09 12:06:29 -05004282 rem -= count;
Jason Wessel955b61e2010-08-05 09:22:23 -05004283 if (!trace_find_next_entry_inc(iter)) {
Steven Rostedt34cd4992009-02-09 12:06:29 -05004284 rem = 0;
4285 iter->ent = NULL;
4286 break;
4287 }
4288 }
4289
4290 return rem;
4291}
4292
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004293static ssize_t tracing_splice_read_pipe(struct file *filp,
4294 loff_t *ppos,
4295 struct pipe_inode_info *pipe,
4296 size_t len,
4297 unsigned int flags)
4298{
Jens Axboe35f3d142010-05-20 10:43:18 +02004299 struct page *pages_def[PIPE_DEF_BUFFERS];
4300 struct partial_page partial_def[PIPE_DEF_BUFFERS];
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004301 struct trace_iterator *iter = filp->private_data;
4302 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02004303 .pages = pages_def,
4304 .partial = partial_def,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004305 .nr_pages = 0, /* This gets updated below. */
Eric Dumazet047fe362012-06-12 15:24:40 +02004306 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004307 .flags = flags,
4308 .ops = &tracing_pipe_buf_ops,
4309 .spd_release = tracing_spd_release_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004310 };
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004311 struct trace_array *tr = iter->tr;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004312 ssize_t ret;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004313 size_t rem;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004314 unsigned int i;
4315
Jens Axboe35f3d142010-05-20 10:43:18 +02004316 if (splice_grow_spd(pipe, &spd))
4317 return -ENOMEM;
4318
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004319 /* copy the tracer to avoid using a global lock all around */
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004320 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004321 if (unlikely(iter->trace->name != tr->current_trace->name))
4322 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004323 mutex_unlock(&trace_types_lock);
4324
4325 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004326
4327 if (iter->trace->splice_read) {
4328 ret = iter->trace->splice_read(iter, filp,
4329 ppos, pipe, len, flags);
4330 if (ret)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004331 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004332 }
4333
4334 ret = tracing_wait_pipe(filp);
4335 if (ret <= 0)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004336 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004337
Jason Wessel955b61e2010-08-05 09:22:23 -05004338 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004339 ret = -EFAULT;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004340 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004341 }
4342
Lai Jiangshan4f535962009-05-18 19:35:34 +08004343 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004344 trace_access_lock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004345
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004346 /* Fill as many pages as possible. */
Jens Axboe35f3d142010-05-20 10:43:18 +02004347 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4348 spd.pages[i] = alloc_page(GFP_KERNEL);
4349 if (!spd.pages[i])
Steven Rostedt34cd4992009-02-09 12:06:29 -05004350 break;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004351
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01004352 rem = tracing_fill_pipe_page(rem, iter);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004353
4354 /* Copy the data into the page, so we can start over. */
4355 ret = trace_seq_to_buffer(&iter->seq,
Jens Axboe35f3d142010-05-20 10:43:18 +02004356 page_address(spd.pages[i]),
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004357 iter->seq.len);
4358 if (ret < 0) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004359 __free_page(spd.pages[i]);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004360 break;
4361 }
Jens Axboe35f3d142010-05-20 10:43:18 +02004362 spd.partial[i].offset = 0;
4363 spd.partial[i].len = iter->seq.len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004364
Steven Rostedtf9520752009-03-02 14:04:40 -05004365 trace_seq_init(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004366 }
4367
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004368 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004369 trace_event_read_unlock();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004370 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004371
4372 spd.nr_pages = i;
4373
Jens Axboe35f3d142010-05-20 10:43:18 +02004374 ret = splice_to_pipe(pipe, &spd);
4375out:
Eric Dumazet047fe362012-06-12 15:24:40 +02004376 splice_shrink_spd(&spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02004377 return ret;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004378
Steven Rostedt34cd4992009-02-09 12:06:29 -05004379out_err:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004380 mutex_unlock(&iter->mutex);
Jens Axboe35f3d142010-05-20 10:43:18 +02004381 goto out;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004382}
4383
Steven Rostedta98a3c32008-05-12 21:20:59 +02004384static ssize_t
4385tracing_entries_read(struct file *filp, char __user *ubuf,
4386 size_t cnt, loff_t *ppos)
4387{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004388 struct trace_cpu *tc = filp->private_data;
4389 struct trace_array *tr = tc->tr;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004390 char buf[64];
4391 int r = 0;
4392 ssize_t ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004393
Steven Rostedtdb526ca2009-03-12 13:53:25 -04004394 mutex_lock(&trace_types_lock);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004395
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004396 if (tc->cpu == RING_BUFFER_ALL_CPUS) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004397 int cpu, buf_size_same;
4398 unsigned long size;
4399
4400 size = 0;
4401 buf_size_same = 1;
4402 /* check if all cpu sizes are same */
4403 for_each_tracing_cpu(cpu) {
4404 /* fill in the size from first enabled cpu */
4405 if (size == 0)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004406 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4407 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004408 buf_size_same = 0;
4409 break;
4410 }
4411 }
4412
4413 if (buf_size_same) {
4414 if (!ring_buffer_expanded)
4415 r = sprintf(buf, "%lu (expanded: %lu)\n",
4416 size >> 10,
4417 trace_buf_size >> 10);
4418 else
4419 r = sprintf(buf, "%lu\n", size >> 10);
4420 } else
4421 r = sprintf(buf, "X\n");
4422 } else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004423 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, tc->cpu)->entries >> 10);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004424
Steven Rostedtdb526ca2009-03-12 13:53:25 -04004425 mutex_unlock(&trace_types_lock);
4426
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004427 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4428 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004429}
4430
4431static ssize_t
4432tracing_entries_write(struct file *filp, const char __user *ubuf,
4433 size_t cnt, loff_t *ppos)
4434{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004435 struct trace_cpu *tc = filp->private_data;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004436 unsigned long val;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004437 int ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004438
Peter Huewe22fe9b52011-06-07 21:58:27 +02004439 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4440 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02004441 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004442
4443 /* must have at least 1 entry */
4444 if (!val)
4445 return -EINVAL;
4446
Steven Rostedt1696b2b2008-11-13 00:09:35 -05004447 /* value is in KB */
4448 val <<= 10;
4449
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004450 ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004451 if (ret < 0)
4452 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004453
Jiri Olsacf8517c2009-10-23 19:36:16 -04004454 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004455
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004456 return cnt;
4457}
Steven Rostedtbf5e6512008-11-10 21:46:00 -05004458
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004459static ssize_t
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004460tracing_total_entries_read(struct file *filp, char __user *ubuf,
4461 size_t cnt, loff_t *ppos)
4462{
4463 struct trace_array *tr = filp->private_data;
4464 char buf[64];
4465 int r, cpu;
4466 unsigned long size = 0, expanded_size = 0;
4467
4468 mutex_lock(&trace_types_lock);
4469 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004470 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004471 if (!ring_buffer_expanded)
4472 expanded_size += trace_buf_size >> 10;
4473 }
4474 if (ring_buffer_expanded)
4475 r = sprintf(buf, "%lu\n", size);
4476 else
4477 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4478 mutex_unlock(&trace_types_lock);
4479
4480 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4481}
4482
4483static ssize_t
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004484tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4485 size_t cnt, loff_t *ppos)
4486{
4487 /*
4488 * There is no need to read what the user has written, this function
4489 * is just to make sure that there is no error when "echo" is used
4490 */
4491
4492 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004493
4494 return cnt;
4495}
4496
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004497static int
4498tracing_free_buffer_release(struct inode *inode, struct file *filp)
4499{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004500 struct trace_array *tr = inode->i_private;
4501
Steven Rostedtcf30cf62011-06-14 22:44:07 -04004502 /* disable tracing ? */
4503 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4504 tracing_off();
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004505 /* resize the ring buffer to 0 */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004506 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004507
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004508 trace_array_put(tr);
4509
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004510 return 0;
4511}
4512
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004513static ssize_t
4514tracing_mark_write(struct file *filp, const char __user *ubuf,
4515 size_t cnt, loff_t *fpos)
4516{
Steven Rostedtd696b582011-09-22 11:50:27 -04004517 unsigned long addr = (unsigned long)ubuf;
Alexander Z Lam2d716192013-07-01 15:31:24 -07004518 struct trace_array *tr = filp->private_data;
Steven Rostedtd696b582011-09-22 11:50:27 -04004519 struct ring_buffer_event *event;
4520 struct ring_buffer *buffer;
4521 struct print_entry *entry;
4522 unsigned long irq_flags;
4523 struct page *pages[2];
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004524 void *map_page[2];
Steven Rostedtd696b582011-09-22 11:50:27 -04004525 int nr_pages = 1;
4526 ssize_t written;
Steven Rostedtd696b582011-09-22 11:50:27 -04004527 int offset;
4528 int size;
4529 int len;
4530 int ret;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004531 int i;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004532
Steven Rostedtc76f0692008-11-07 22:36:02 -05004533 if (tracing_disabled)
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004534 return -EINVAL;
4535
Mandeep Singh Baines5224c3a2012-09-07 18:12:19 -07004536 if (!(trace_flags & TRACE_ITER_MARKERS))
4537 return -EINVAL;
4538
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004539 if (cnt > TRACE_BUF_SIZE)
4540 cnt = TRACE_BUF_SIZE;
4541
Steven Rostedtd696b582011-09-22 11:50:27 -04004542 /*
4543 * Userspace is injecting traces into the kernel trace buffer.
4544 * We want to be as non intrusive as possible.
4545 * To do so, we do not want to allocate any special buffers
4546 * or take any locks, but instead write the userspace data
4547 * straight into the ring buffer.
4548 *
4549 * First we need to pin the userspace buffer into memory,
4550 * which, most likely it is, because it just referenced it.
4551 * But there's no guarantee that it is. By using get_user_pages_fast()
4552 * and kmap_atomic/kunmap_atomic() we can get access to the
4553 * pages directly. We then write the data directly into the
4554 * ring buffer.
4555 */
4556 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004557
Steven Rostedtd696b582011-09-22 11:50:27 -04004558 /* check if we cross pages */
4559 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4560 nr_pages = 2;
4561
4562 offset = addr & (PAGE_SIZE - 1);
4563 addr &= PAGE_MASK;
4564
4565 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4566 if (ret < nr_pages) {
4567 while (--ret >= 0)
4568 put_page(pages[ret]);
4569 written = -EFAULT;
4570 goto out;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004571 }
4572
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004573 for (i = 0; i < nr_pages; i++)
4574 map_page[i] = kmap_atomic(pages[i]);
Steven Rostedtd696b582011-09-22 11:50:27 -04004575
4576 local_save_flags(irq_flags);
4577 size = sizeof(*entry) + cnt + 2; /* possible \n added */
Alexander Z Lam2d716192013-07-01 15:31:24 -07004578 buffer = tr->trace_buffer.buffer;
Steven Rostedtd696b582011-09-22 11:50:27 -04004579 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4580 irq_flags, preempt_count());
4581 if (!event) {
4582 /* Ring buffer disabled, return as if not open for write */
4583 written = -EBADF;
4584 goto out_unlock;
4585 }
4586
4587 entry = ring_buffer_event_data(event);
4588 entry->ip = _THIS_IP_;
4589
4590 if (nr_pages == 2) {
4591 len = PAGE_SIZE - offset;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004592 memcpy(&entry->buf, map_page[0] + offset, len);
4593 memcpy(&entry->buf[len], map_page[1], cnt - len);
Steven Rostedtd696b582011-09-22 11:50:27 -04004594 } else
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004595 memcpy(&entry->buf, map_page[0] + offset, cnt);
Steven Rostedtd696b582011-09-22 11:50:27 -04004596
4597 if (entry->buf[cnt - 1] != '\n') {
4598 entry->buf[cnt] = '\n';
4599 entry->buf[cnt + 1] = '\0';
4600 } else
4601 entry->buf[cnt] = '\0';
4602
Steven Rostedt7ffbd482012-10-11 12:14:25 -04004603 __buffer_unlock_commit(buffer, event);
Steven Rostedtd696b582011-09-22 11:50:27 -04004604
4605 written = cnt;
4606
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02004607 *fpos += written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004608
Steven Rostedtd696b582011-09-22 11:50:27 -04004609 out_unlock:
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004610 for (i = 0; i < nr_pages; i++){
4611 kunmap_atomic(map_page[i]);
4612 put_page(pages[i]);
4613 }
Steven Rostedtd696b582011-09-22 11:50:27 -04004614 out:
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02004615 return written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004616}
4617
Li Zefan13f16d22009-12-08 11:16:11 +08004618static int tracing_clock_show(struct seq_file *m, void *v)
Zhaolei5079f322009-08-25 16:12:56 +08004619{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004620 struct trace_array *tr = m->private;
Zhaolei5079f322009-08-25 16:12:56 +08004621 int i;
4622
4623 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
Li Zefan13f16d22009-12-08 11:16:11 +08004624 seq_printf(m,
Zhaolei5079f322009-08-25 16:12:56 +08004625 "%s%s%s%s", i ? " " : "",
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004626 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4627 i == tr->clock_id ? "]" : "");
Li Zefan13f16d22009-12-08 11:16:11 +08004628 seq_putc(m, '\n');
Zhaolei5079f322009-08-25 16:12:56 +08004629
Li Zefan13f16d22009-12-08 11:16:11 +08004630 return 0;
Zhaolei5079f322009-08-25 16:12:56 +08004631}
4632
4633static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4634 size_t cnt, loff_t *fpos)
4635{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004636 struct seq_file *m = filp->private_data;
4637 struct trace_array *tr = m->private;
Zhaolei5079f322009-08-25 16:12:56 +08004638 char buf[64];
4639 const char *clockstr;
4640 int i;
4641
4642 if (cnt >= sizeof(buf))
4643 return -EINVAL;
4644
4645 if (copy_from_user(&buf, ubuf, cnt))
4646 return -EFAULT;
4647
4648 buf[cnt] = 0;
4649
4650 clockstr = strstrip(buf);
4651
4652 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4653 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4654 break;
4655 }
4656 if (i == ARRAY_SIZE(trace_clocks))
4657 return -EINVAL;
4658
Zhaolei5079f322009-08-25 16:12:56 +08004659 mutex_lock(&trace_types_lock);
4660
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004661 tr->clock_id = i;
4662
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004663 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
Zhaolei5079f322009-08-25 16:12:56 +08004664
David Sharp60303ed2012-10-11 16:27:52 -07004665 /*
4666 * New clock may not be consistent with the previous clock.
4667 * Reset the buffer so that it doesn't have incomparable timestamps.
4668 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004669 tracing_reset_online_cpus(&global_trace.trace_buffer);
4670
4671#ifdef CONFIG_TRACER_MAX_TRACE
4672 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4673 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4674 tracing_reset_online_cpus(&global_trace.max_buffer);
4675#endif
David Sharp60303ed2012-10-11 16:27:52 -07004676
Zhaolei5079f322009-08-25 16:12:56 +08004677 mutex_unlock(&trace_types_lock);
4678
4679 *fpos += cnt;
4680
4681 return cnt;
4682}
4683
Li Zefan13f16d22009-12-08 11:16:11 +08004684static int tracing_clock_open(struct inode *inode, struct file *file)
4685{
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004686 struct trace_array *tr = inode->i_private;
4687 int ret;
4688
Li Zefan13f16d22009-12-08 11:16:11 +08004689 if (tracing_disabled)
4690 return -ENODEV;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004691
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004692 if (trace_array_get(tr))
4693 return -ENODEV;
4694
4695 ret = single_open(file, tracing_clock_show, inode->i_private);
4696 if (ret < 0)
4697 trace_array_put(tr);
4698
4699 return ret;
Li Zefan13f16d22009-12-08 11:16:11 +08004700}
4701
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004702struct ftrace_buffer_info {
4703 struct trace_iterator iter;
4704 void *spare;
4705 unsigned int read;
4706};
4707
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004708#ifdef CONFIG_TRACER_SNAPSHOT
4709static int tracing_snapshot_open(struct inode *inode, struct file *file)
4710{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004711 struct trace_cpu *tc = inode->i_private;
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04004712 struct trace_array *tr = tc->tr;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004713 struct trace_iterator *iter;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004714 struct seq_file *m;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004715 int ret = 0;
4716
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04004717 if (trace_array_get(tr) < 0)
4718 return -ENODEV;
4719
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004720 if (file->f_mode & FMODE_READ) {
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04004721 iter = __tracing_open(tr, tc, inode, file, true);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004722 if (IS_ERR(iter))
4723 ret = PTR_ERR(iter);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004724 } else {
4725 /* Writes still need the seq_file to hold the private data */
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07004726 ret = -ENOMEM;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004727 m = kzalloc(sizeof(*m), GFP_KERNEL);
4728 if (!m)
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07004729 goto out;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004730 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4731 if (!iter) {
4732 kfree(m);
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07004733 goto out;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004734 }
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07004735 ret = 0;
4736
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04004737 iter->tr = tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004738 iter->trace_buffer = &tc->tr->max_buffer;
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004739 iter->cpu_file = tc->cpu;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004740 m->private = iter;
4741 file->private_data = m;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004742 }
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07004743out:
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04004744 if (ret < 0)
4745 trace_array_put(tr);
4746
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004747 return ret;
4748}
4749
4750static ssize_t
4751tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4752 loff_t *ppos)
4753{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004754 struct seq_file *m = filp->private_data;
4755 struct trace_iterator *iter = m->private;
4756 struct trace_array *tr = iter->tr;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004757 unsigned long val;
4758 int ret;
4759
4760 ret = tracing_update_buffers();
4761 if (ret < 0)
4762 return ret;
4763
4764 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4765 if (ret)
4766 return ret;
4767
4768 mutex_lock(&trace_types_lock);
4769
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004770 if (tr->current_trace->use_max_tr) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004771 ret = -EBUSY;
4772 goto out;
4773 }
4774
4775 switch (val) {
4776 case 0:
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004777 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4778 ret = -EINVAL;
4779 break;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004780 }
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04004781 if (tr->allocated_snapshot)
4782 free_snapshot(tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004783 break;
4784 case 1:
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004785/* Only allow per-cpu swap if the ring buffer supports it */
4786#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4787 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4788 ret = -EINVAL;
4789 break;
4790 }
4791#endif
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05004792 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04004793 ret = alloc_snapshot(tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004794 if (ret < 0)
4795 break;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004796 }
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004797 local_irq_disable();
4798 /* Now, we're going to swap */
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004799 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05004800 update_max_tr(tr, current, smp_processor_id());
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004801 else
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05004802 update_max_tr_single(tr, current, iter->cpu_file);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004803 local_irq_enable();
4804 break;
4805 default:
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05004806 if (tr->allocated_snapshot) {
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004807 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4808 tracing_reset_online_cpus(&tr->max_buffer);
4809 else
4810 tracing_reset(&tr->max_buffer, iter->cpu_file);
4811 }
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004812 break;
4813 }
4814
4815 if (ret >= 0) {
4816 *ppos += cnt;
4817 ret = cnt;
4818 }
4819out:
4820 mutex_unlock(&trace_types_lock);
4821 return ret;
4822}
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004823
4824static int tracing_snapshot_release(struct inode *inode, struct file *file)
4825{
4826 struct seq_file *m = file->private_data;
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04004827 int ret;
4828
4829 ret = tracing_release(inode, file);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004830
4831 if (file->f_mode & FMODE_READ)
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04004832 return ret;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004833
4834 /* If write only, the seq_file is just a stub */
4835 if (m)
4836 kfree(m->private);
4837 kfree(m);
4838
4839 return 0;
4840}
4841
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004842static int tracing_buffers_open(struct inode *inode, struct file *filp);
4843static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4844 size_t count, loff_t *ppos);
4845static int tracing_buffers_release(struct inode *inode, struct file *file);
4846static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4847 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4848
4849static int snapshot_raw_open(struct inode *inode, struct file *filp)
4850{
4851 struct ftrace_buffer_info *info;
4852 int ret;
4853
4854 ret = tracing_buffers_open(inode, filp);
4855 if (ret < 0)
4856 return ret;
4857
4858 info = filp->private_data;
4859
4860 if (info->iter.trace->use_max_tr) {
4861 tracing_buffers_release(inode, filp);
4862 return -EBUSY;
4863 }
4864
4865 info->iter.snapshot = true;
4866 info->iter.trace_buffer = &info->iter.tr->max_buffer;
4867
4868 return ret;
4869}
4870
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004871#endif /* CONFIG_TRACER_SNAPSHOT */
4872
4873
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004874static const struct file_operations tracing_max_lat_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004875 .open = tracing_open_generic,
4876 .read = tracing_max_lat_read,
4877 .write = tracing_max_lat_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004878 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004879};
4880
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004881static const struct file_operations set_tracer_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004882 .open = tracing_open_generic,
4883 .read = tracing_set_trace_read,
4884 .write = tracing_set_trace_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004885 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004886};
4887
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004888static const struct file_operations tracing_pipe_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004889 .open = tracing_open_pipe,
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004890 .poll = tracing_poll_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004891 .read = tracing_read_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004892 .splice_read = tracing_splice_read_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004893 .release = tracing_release_pipe,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004894 .llseek = no_llseek,
Steven Rostedtb3806b42008-05-12 21:20:46 +02004895};
4896
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004897static const struct file_operations tracing_entries_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004898 .open = tracing_open_generic_tc,
Steven Rostedta98a3c32008-05-12 21:20:59 +02004899 .read = tracing_entries_read,
4900 .write = tracing_entries_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004901 .llseek = generic_file_llseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004902 .release = tracing_release_generic_tc,
Steven Rostedta98a3c32008-05-12 21:20:59 +02004903};
4904
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004905static const struct file_operations tracing_total_entries_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004906 .open = tracing_open_generic_tr,
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004907 .read = tracing_total_entries_read,
4908 .llseek = generic_file_llseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004909 .release = tracing_release_generic_tr,
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004910};
4911
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004912static const struct file_operations tracing_free_buffer_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004913 .open = tracing_open_generic_tr,
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004914 .write = tracing_free_buffer_write,
4915 .release = tracing_free_buffer_release,
4916};
4917
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004918static const struct file_operations tracing_mark_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004919 .open = tracing_open_generic_tr,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004920 .write = tracing_mark_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004921 .llseek = generic_file_llseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004922 .release = tracing_release_generic_tr,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004923};
4924
Zhaolei5079f322009-08-25 16:12:56 +08004925static const struct file_operations trace_clock_fops = {
Li Zefan13f16d22009-12-08 11:16:11 +08004926 .open = tracing_clock_open,
4927 .read = seq_read,
4928 .llseek = seq_lseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004929 .release = tracing_single_release_tr,
Zhaolei5079f322009-08-25 16:12:56 +08004930 .write = tracing_clock_write,
4931};
4932
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004933#ifdef CONFIG_TRACER_SNAPSHOT
4934static const struct file_operations snapshot_fops = {
4935 .open = tracing_snapshot_open,
4936 .read = seq_read,
4937 .write = tracing_snapshot_write,
4938 .llseek = tracing_seek,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004939 .release = tracing_snapshot_release,
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004940};
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004941
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004942static const struct file_operations snapshot_raw_fops = {
4943 .open = snapshot_raw_open,
4944 .read = tracing_buffers_read,
4945 .release = tracing_buffers_release,
4946 .splice_read = tracing_buffers_splice_read,
4947 .llseek = no_llseek,
Steven Rostedt2cadf912008-12-01 22:20:19 -05004948};
4949
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004950#endif /* CONFIG_TRACER_SNAPSHOT */
4951
Steven Rostedt2cadf912008-12-01 22:20:19 -05004952static int tracing_buffers_open(struct inode *inode, struct file *filp)
4953{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004954 struct trace_cpu *tc = inode->i_private;
4955 struct trace_array *tr = tc->tr;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004956 struct ftrace_buffer_info *info;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004957 int ret;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004958
4959 if (tracing_disabled)
4960 return -ENODEV;
4961
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004962 if (trace_array_get(tr) < 0)
4963 return -ENODEV;
4964
Steven Rostedt2cadf912008-12-01 22:20:19 -05004965 info = kzalloc(sizeof(*info), GFP_KERNEL);
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004966 if (!info) {
4967 trace_array_put(tr);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004968 return -ENOMEM;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004969 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05004970
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05004971 mutex_lock(&trace_types_lock);
4972
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004973 info->iter.tr = tr;
4974 info->iter.cpu_file = tc->cpu;
Steven Rostedtb6273442013-02-28 13:44:11 -05004975 info->iter.trace = tr->current_trace;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004976 info->iter.trace_buffer = &tr->trace_buffer;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004977 info->spare = NULL;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004978 /* Force reading ring buffer for first read */
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004979 info->read = (unsigned int)-1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004980
4981 filp->private_data = info;
4982
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05004983 mutex_unlock(&trace_types_lock);
4984
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004985 ret = nonseekable_open(inode, filp);
4986 if (ret < 0)
4987 trace_array_put(tr);
4988
4989 return ret;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004990}
4991
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004992static unsigned int
4993tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4994{
4995 struct ftrace_buffer_info *info = filp->private_data;
4996 struct trace_iterator *iter = &info->iter;
4997
4998 return trace_poll(iter, filp, poll_table);
4999}
5000
Steven Rostedt2cadf912008-12-01 22:20:19 -05005001static ssize_t
5002tracing_buffers_read(struct file *filp, char __user *ubuf,
5003 size_t count, loff_t *ppos)
5004{
5005 struct ftrace_buffer_info *info = filp->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005006 struct trace_iterator *iter = &info->iter;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005007 ssize_t ret;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005008 ssize_t size;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005009
Steven Rostedt2dc5d122009-03-04 19:10:05 -05005010 if (!count)
5011 return 0;
5012
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005013 mutex_lock(&trace_types_lock);
5014
5015#ifdef CONFIG_TRACER_MAX_TRACE
5016 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5017 size = -EBUSY;
5018 goto out_unlock;
5019 }
5020#endif
5021
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005022 if (!info->spare)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005023 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5024 iter->cpu_file);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005025 size = -ENOMEM;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005026 if (!info->spare)
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005027 goto out_unlock;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005028
Steven Rostedt2cadf912008-12-01 22:20:19 -05005029 /* Do we have previous read data to read? */
5030 if (info->read < PAGE_SIZE)
5031 goto read;
5032
Steven Rostedtb6273442013-02-28 13:44:11 -05005033 again:
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005034 trace_access_lock(iter->cpu_file);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005035 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005036 &info->spare,
5037 count,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005038 iter->cpu_file, 0);
5039 trace_access_unlock(iter->cpu_file);
Steven Rostedtb6273442013-02-28 13:44:11 -05005040
5041 if (ret < 0) {
5042 if (trace_empty(iter)) {
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005043 if ((filp->f_flags & O_NONBLOCK)) {
5044 size = -EAGAIN;
5045 goto out_unlock;
5046 }
5047 mutex_unlock(&trace_types_lock);
Steven Rostedtb6273442013-02-28 13:44:11 -05005048 iter->trace->wait_pipe(iter);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005049 mutex_lock(&trace_types_lock);
5050 if (signal_pending(current)) {
5051 size = -EINTR;
5052 goto out_unlock;
5053 }
Steven Rostedtb6273442013-02-28 13:44:11 -05005054 goto again;
5055 }
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005056 size = 0;
5057 goto out_unlock;
Steven Rostedtb6273442013-02-28 13:44:11 -05005058 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05005059
Steven Rostedt436fc282011-10-14 10:44:25 -04005060 info->read = 0;
Steven Rostedtb6273442013-02-28 13:44:11 -05005061 read:
Steven Rostedt2cadf912008-12-01 22:20:19 -05005062 size = PAGE_SIZE - info->read;
5063 if (size > count)
5064 size = count;
5065
5066 ret = copy_to_user(ubuf, info->spare + info->read, size);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005067 if (ret == size) {
5068 size = -EFAULT;
5069 goto out_unlock;
5070 }
Steven Rostedt2dc5d122009-03-04 19:10:05 -05005071 size -= ret;
5072
Steven Rostedt2cadf912008-12-01 22:20:19 -05005073 *ppos += size;
5074 info->read += size;
5075
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005076 out_unlock:
5077 mutex_unlock(&trace_types_lock);
5078
Steven Rostedt2cadf912008-12-01 22:20:19 -05005079 return size;
5080}
5081
5082static int tracing_buffers_release(struct inode *inode, struct file *file)
5083{
5084 struct ftrace_buffer_info *info = file->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005085 struct trace_iterator *iter = &info->iter;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005086
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005087 mutex_lock(&trace_types_lock);
5088
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005089 __trace_array_put(iter->tr);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005090
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005091 if (info->spare)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005092 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005093 kfree(info);
5094
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005095 mutex_unlock(&trace_types_lock);
5096
Steven Rostedt2cadf912008-12-01 22:20:19 -05005097 return 0;
5098}
5099
5100struct buffer_ref {
5101 struct ring_buffer *buffer;
5102 void *page;
5103 int ref;
5104};
5105
5106static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5107 struct pipe_buffer *buf)
5108{
5109 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5110
5111 if (--ref->ref)
5112 return;
5113
5114 ring_buffer_free_read_page(ref->buffer, ref->page);
5115 kfree(ref);
5116 buf->private = 0;
5117}
5118
Steven Rostedt2cadf912008-12-01 22:20:19 -05005119static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5120 struct pipe_buffer *buf)
5121{
5122 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5123
5124 ref->ref++;
5125}
5126
5127/* Pipe buffer operations for a buffer. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08005128static const struct pipe_buf_operations buffer_pipe_buf_ops = {
Steven Rostedt2cadf912008-12-01 22:20:19 -05005129 .can_merge = 0,
5130 .map = generic_pipe_buf_map,
5131 .unmap = generic_pipe_buf_unmap,
5132 .confirm = generic_pipe_buf_confirm,
5133 .release = buffer_pipe_buf_release,
Masami Hiramatsud55cb6c2012-08-09 21:31:10 +09005134 .steal = generic_pipe_buf_steal,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005135 .get = buffer_pipe_buf_get,
5136};
5137
5138/*
5139 * Callback from splice_to_pipe(), if we need to release some pages
5140 * at the end of the spd in case we error'ed out in filling the pipe.
5141 */
5142static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5143{
5144 struct buffer_ref *ref =
5145 (struct buffer_ref *)spd->partial[i].private;
5146
5147 if (--ref->ref)
5148 return;
5149
5150 ring_buffer_free_read_page(ref->buffer, ref->page);
5151 kfree(ref);
5152 spd->partial[i].private = 0;
5153}
5154
5155static ssize_t
5156tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5157 struct pipe_inode_info *pipe, size_t len,
5158 unsigned int flags)
5159{
5160 struct ftrace_buffer_info *info = file->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005161 struct trace_iterator *iter = &info->iter;
Jens Axboe35f3d142010-05-20 10:43:18 +02005162 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5163 struct page *pages_def[PIPE_DEF_BUFFERS];
Steven Rostedt2cadf912008-12-01 22:20:19 -05005164 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02005165 .pages = pages_def,
5166 .partial = partial_def,
Eric Dumazet047fe362012-06-12 15:24:40 +02005167 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005168 .flags = flags,
5169 .ops = &buffer_pipe_buf_ops,
5170 .spd_release = buffer_spd_release,
5171 };
5172 struct buffer_ref *ref;
Steven Rostedt93459c62009-04-29 00:23:13 -04005173 int entries, size, i;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005174 ssize_t ret;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005175
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005176 mutex_lock(&trace_types_lock);
5177
5178#ifdef CONFIG_TRACER_MAX_TRACE
5179 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5180 ret = -EBUSY;
5181 goto out;
5182 }
5183#endif
5184
5185 if (splice_grow_spd(pipe, &spd)) {
5186 ret = -ENOMEM;
5187 goto out;
5188 }
Jens Axboe35f3d142010-05-20 10:43:18 +02005189
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005190 if (*ppos & (PAGE_SIZE - 1)) {
Jens Axboe35f3d142010-05-20 10:43:18 +02005191 ret = -EINVAL;
5192 goto out;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005193 }
5194
5195 if (len & (PAGE_SIZE - 1)) {
Jens Axboe35f3d142010-05-20 10:43:18 +02005196 if (len < PAGE_SIZE) {
5197 ret = -EINVAL;
5198 goto out;
5199 }
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005200 len &= PAGE_MASK;
5201 }
5202
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005203 again:
5204 trace_access_lock(iter->cpu_file);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005205 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt93459c62009-04-29 00:23:13 -04005206
Jens Axboe35f3d142010-05-20 10:43:18 +02005207 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05005208 struct page *page;
5209 int r;
5210
5211 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5212 if (!ref)
5213 break;
5214
Steven Rostedt7267fa62009-04-29 00:16:21 -04005215 ref->ref = 1;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005216 ref->buffer = iter->trace_buffer->buffer;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005217 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005218 if (!ref->page) {
5219 kfree(ref);
5220 break;
5221 }
5222
5223 r = ring_buffer_read_page(ref->buffer, &ref->page,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005224 len, iter->cpu_file, 1);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005225 if (r < 0) {
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07005226 ring_buffer_free_read_page(ref->buffer, ref->page);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005227 kfree(ref);
5228 break;
5229 }
5230
5231 /*
5232 * zero out any left over data, this is going to
5233 * user land.
5234 */
5235 size = ring_buffer_page_len(ref->page);
5236 if (size < PAGE_SIZE)
5237 memset(ref->page + size, 0, PAGE_SIZE - size);
5238
5239 page = virt_to_page(ref->page);
5240
5241 spd.pages[i] = page;
5242 spd.partial[i].len = PAGE_SIZE;
5243 spd.partial[i].offset = 0;
5244 spd.partial[i].private = (unsigned long)ref;
5245 spd.nr_pages++;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005246 *ppos += PAGE_SIZE;
Steven Rostedt93459c62009-04-29 00:23:13 -04005247
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005248 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005249 }
5250
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005251 trace_access_unlock(iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005252 spd.nr_pages = i;
5253
5254 /* did we read anything? */
5255 if (!spd.nr_pages) {
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005256 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05005257 ret = -EAGAIN;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005258 goto out;
5259 }
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005260 mutex_unlock(&trace_types_lock);
Steven Rostedtb6273442013-02-28 13:44:11 -05005261 iter->trace->wait_pipe(iter);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005262 mutex_lock(&trace_types_lock);
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005263 if (signal_pending(current)) {
5264 ret = -EINTR;
5265 goto out;
5266 }
5267 goto again;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005268 }
5269
5270 ret = splice_to_pipe(pipe, &spd);
Eric Dumazet047fe362012-06-12 15:24:40 +02005271 splice_shrink_spd(&spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02005272out:
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005273 mutex_unlock(&trace_types_lock);
5274
Steven Rostedt2cadf912008-12-01 22:20:19 -05005275 return ret;
5276}
5277
5278static const struct file_operations tracing_buffers_fops = {
5279 .open = tracing_buffers_open,
5280 .read = tracing_buffers_read,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005281 .poll = tracing_buffers_poll,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005282 .release = tracing_buffers_release,
5283 .splice_read = tracing_buffers_splice_read,
5284 .llseek = no_llseek,
5285};
5286
Steven Rostedtc8d77182009-04-29 18:03:45 -04005287static ssize_t
5288tracing_stats_read(struct file *filp, char __user *ubuf,
5289 size_t count, loff_t *ppos)
5290{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005291 struct trace_cpu *tc = filp->private_data;
5292 struct trace_array *tr = tc->tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005293 struct trace_buffer *trace_buf = &tr->trace_buffer;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005294 struct trace_seq *s;
5295 unsigned long cnt;
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005296 unsigned long long t;
5297 unsigned long usec_rem;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005298 int cpu = tc->cpu;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005299
Li Zefane4f2d102009-06-15 10:57:28 +08005300 s = kmalloc(sizeof(*s), GFP_KERNEL);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005301 if (!s)
Roel Kluina6463652009-11-11 22:26:35 +01005302 return -ENOMEM;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005303
5304 trace_seq_init(s);
5305
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005306 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005307 trace_seq_printf(s, "entries: %ld\n", cnt);
5308
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005309 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005310 trace_seq_printf(s, "overrun: %ld\n", cnt);
5311
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005312 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005313 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5314
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005315 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005316 trace_seq_printf(s, "bytes: %ld\n", cnt);
5317
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09005318 if (trace_clocks[tr->clock_id].in_ns) {
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005319 /* local or global for trace_clock */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005320 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005321 usec_rem = do_div(t, USEC_PER_SEC);
5322 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5323 t, usec_rem);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005324
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005325 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005326 usec_rem = do_div(t, USEC_PER_SEC);
5327 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5328 } else {
5329 /* counter or tsc mode for trace_clock */
5330 trace_seq_printf(s, "oldest event ts: %llu\n",
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005331 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005332
5333 trace_seq_printf(s, "now ts: %llu\n",
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005334 ring_buffer_time_stamp(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005335 }
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005336
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005337 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
Slava Pestov884bfe82011-07-15 14:23:58 -07005338 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5339
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005340 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
Steven Rostedt (Red Hat)ad964702013-01-29 17:45:49 -05005341 trace_seq_printf(s, "read events: %ld\n", cnt);
5342
Steven Rostedtc8d77182009-04-29 18:03:45 -04005343 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5344
5345 kfree(s);
5346
5347 return count;
5348}
5349
5350static const struct file_operations tracing_stats_fops = {
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005351 .open = tracing_open_generic_tc,
Steven Rostedtc8d77182009-04-29 18:03:45 -04005352 .read = tracing_stats_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005353 .llseek = generic_file_llseek,
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005354 .release = tracing_release_generic_tc,
Steven Rostedtc8d77182009-04-29 18:03:45 -04005355};
5356
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005357#ifdef CONFIG_DYNAMIC_FTRACE
5358
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005359int __weak ftrace_arch_read_dyn_info(char *buf, int size)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005360{
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005361 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005362}
5363
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005364static ssize_t
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005365tracing_read_dyn_info(struct file *filp, char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005366 size_t cnt, loff_t *ppos)
5367{
Steven Rostedta26a2a22008-10-31 00:03:22 -04005368 static char ftrace_dyn_info_buffer[1024];
5369 static DEFINE_MUTEX(dyn_info_mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005370 unsigned long *p = filp->private_data;
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005371 char *buf = ftrace_dyn_info_buffer;
Steven Rostedta26a2a22008-10-31 00:03:22 -04005372 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005373 int r;
5374
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005375 mutex_lock(&dyn_info_mutex);
5376 r = sprintf(buf, "%ld ", *p);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005377
Steven Rostedta26a2a22008-10-31 00:03:22 -04005378 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005379 buf[r++] = '\n';
5380
5381 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5382
5383 mutex_unlock(&dyn_info_mutex);
5384
5385 return r;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005386}
5387
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005388static const struct file_operations tracing_dyn_info_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005389 .open = tracing_open_generic,
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005390 .read = tracing_read_dyn_info,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005391 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005392};
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005393#endif /* CONFIG_DYNAMIC_FTRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005394
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005395#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5396static void
5397ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005398{
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005399 tracing_snapshot();
5400}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005401
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005402static void
5403ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5404{
5405 unsigned long *count = (long *)data;
5406
5407 if (!*count)
5408 return;
5409
5410 if (*count != -1)
5411 (*count)--;
5412
5413 tracing_snapshot();
5414}
5415
5416static int
5417ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5418 struct ftrace_probe_ops *ops, void *data)
5419{
5420 long count = (long)data;
5421
5422 seq_printf(m, "%ps:", (void *)ip);
5423
5424 seq_printf(m, "snapshot");
5425
5426 if (count == -1)
5427 seq_printf(m, ":unlimited\n");
5428 else
5429 seq_printf(m, ":count=%ld\n", count);
5430
5431 return 0;
5432}
5433
5434static struct ftrace_probe_ops snapshot_probe_ops = {
5435 .func = ftrace_snapshot,
5436 .print = ftrace_snapshot_print,
5437};
5438
5439static struct ftrace_probe_ops snapshot_count_probe_ops = {
5440 .func = ftrace_count_snapshot,
5441 .print = ftrace_snapshot_print,
5442};
5443
5444static int
5445ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5446 char *glob, char *cmd, char *param, int enable)
5447{
5448 struct ftrace_probe_ops *ops;
5449 void *count = (void *)-1;
5450 char *number;
5451 int ret;
5452
5453 /* hash funcs only work with set_ftrace_filter */
5454 if (!enable)
5455 return -EINVAL;
5456
5457 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5458
5459 if (glob[0] == '!') {
5460 unregister_ftrace_function_probe_func(glob+1, ops);
5461 return 0;
5462 }
5463
5464 if (!param)
5465 goto out_reg;
5466
5467 number = strsep(&param, ":");
5468
5469 if (!strlen(number))
5470 goto out_reg;
5471
5472 /*
5473 * We use the callback data field (which is a pointer)
5474 * as our counter.
5475 */
5476 ret = kstrtoul(number, 0, (unsigned long *)&count);
5477 if (ret)
5478 return ret;
5479
5480 out_reg:
5481 ret = register_ftrace_function_probe(glob, ops, count);
5482
5483 if (ret >= 0)
5484 alloc_snapshot(&global_trace);
5485
5486 return ret < 0 ? ret : 0;
5487}
5488
5489static struct ftrace_func_command ftrace_snapshot_cmd = {
5490 .name = "snapshot",
5491 .func = ftrace_trace_snapshot_callback,
5492};
5493
5494static int register_snapshot_cmd(void)
5495{
5496 return register_ftrace_command(&ftrace_snapshot_cmd);
5497}
5498#else
5499static inline int register_snapshot_cmd(void) { return 0; }
5500#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005501
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005502struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005503{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005504 if (tr->dir)
5505 return tr->dir;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005506
Frederic Weisbecker3e1f60b2009-03-22 23:10:45 +01005507 if (!debugfs_initialized())
5508 return NULL;
5509
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005510 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5511 tr->dir = debugfs_create_dir("tracing", NULL);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005512
zhangwei(Jovi)687c8782013-03-11 15:13:29 +08005513 if (!tr->dir)
5514 pr_warn_once("Could not create debugfs directory 'tracing'\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005515
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005516 return tr->dir;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005517}
5518
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005519struct dentry *tracing_init_dentry(void)
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005520{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005521 return tracing_init_dentry_tr(&global_trace);
5522}
5523
5524static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5525{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005526 struct dentry *d_tracer;
5527
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005528 if (tr->percpu_dir)
5529 return tr->percpu_dir;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005530
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005531 d_tracer = tracing_init_dentry_tr(tr);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005532 if (!d_tracer)
5533 return NULL;
5534
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005535 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005536
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005537 WARN_ONCE(!tr->percpu_dir,
5538 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005539
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005540 return tr->percpu_dir;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005541}
5542
Oleg Nesterov649e9c72013-07-23 17:25:54 +02005543static struct dentry *
5544trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5545 void *data, long cpu, const struct file_operations *fops)
5546{
5547 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5548
5549 if (ret) /* See tracing_get_cpu() */
5550 ret->d_inode->i_cdev = (void *)(cpu + 1);
5551 return ret;
5552}
5553
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005554static void
5555tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005556{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005557 struct trace_array_cpu *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005558 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005559 struct dentry *d_cpu;
Steven Rostedtdd49a382010-10-20 21:51:26 -04005560 char cpu_dir[30]; /* 30 characters should be more than enough */
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005561
Namhyung Kim0a3d7ce2012-04-23 10:11:57 +09005562 if (!d_percpu)
5563 return;
5564
Steven Rostedtdd49a382010-10-20 21:51:26 -04005565 snprintf(cpu_dir, 30, "cpu%ld", cpu);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01005566 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5567 if (!d_cpu) {
5568 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5569 return;
5570 }
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005571
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01005572 /* per cpu trace_pipe */
Oleg Nesterov649e9c72013-07-23 17:25:54 +02005573 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5574 &data->trace_cpu, cpu, &tracing_pipe_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005575
5576 /* per cpu trace */
Oleg Nesterov649e9c72013-07-23 17:25:54 +02005577 trace_create_cpu_file("trace", 0644, d_cpu,
5578 &data->trace_cpu, cpu, &tracing_fops);
Steven Rostedt7f96f932009-03-13 00:37:42 -04005579
Oleg Nesterov649e9c72013-07-23 17:25:54 +02005580 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5581 &data->trace_cpu, cpu, &tracing_buffers_fops);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005582
Oleg Nesterov649e9c72013-07-23 17:25:54 +02005583 trace_create_cpu_file("stats", 0444, d_cpu,
5584 &data->trace_cpu, cpu, &tracing_stats_fops);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08005585
Oleg Nesterov649e9c72013-07-23 17:25:54 +02005586 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5587 &data->trace_cpu, cpu, &tracing_entries_fops);
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005588
5589#ifdef CONFIG_TRACER_SNAPSHOT
Oleg Nesterov649e9c72013-07-23 17:25:54 +02005590 trace_create_cpu_file("snapshot", 0644, d_cpu,
5591 &data->trace_cpu, cpu, &snapshot_fops);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005592
Oleg Nesterov649e9c72013-07-23 17:25:54 +02005593 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5594 &data->trace_cpu, cpu, &snapshot_raw_fops);
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005595#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005596}
5597
Steven Rostedt60a11772008-05-12 21:20:44 +02005598#ifdef CONFIG_FTRACE_SELFTEST
5599/* Let selftest have access to static functions in this file */
5600#include "trace_selftest.c"
5601#endif
5602
Steven Rostedt577b7852009-02-26 23:43:05 -05005603struct trace_option_dentry {
5604 struct tracer_opt *opt;
5605 struct tracer_flags *flags;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005606 struct trace_array *tr;
Steven Rostedt577b7852009-02-26 23:43:05 -05005607 struct dentry *entry;
5608};
5609
5610static ssize_t
5611trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5612 loff_t *ppos)
5613{
5614 struct trace_option_dentry *topt = filp->private_data;
5615 char *buf;
5616
5617 if (topt->flags->val & topt->opt->bit)
5618 buf = "1\n";
5619 else
5620 buf = "0\n";
5621
5622 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5623}
5624
5625static ssize_t
5626trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5627 loff_t *ppos)
5628{
5629 struct trace_option_dentry *topt = filp->private_data;
5630 unsigned long val;
Steven Rostedt577b7852009-02-26 23:43:05 -05005631 int ret;
5632
Peter Huewe22fe9b52011-06-07 21:58:27 +02005633 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5634 if (ret)
Steven Rostedt577b7852009-02-26 23:43:05 -05005635 return ret;
5636
Li Zefan8d18eaa2009-12-08 11:17:06 +08005637 if (val != 0 && val != 1)
Steven Rostedt577b7852009-02-26 23:43:05 -05005638 return -EINVAL;
Li Zefan8d18eaa2009-12-08 11:17:06 +08005639
5640 if (!!(topt->flags->val & topt->opt->bit) != val) {
5641 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005642 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
Steven Rostedtc757bea2009-12-21 22:35:16 -05005643 topt->opt, !val);
Li Zefan8d18eaa2009-12-08 11:17:06 +08005644 mutex_unlock(&trace_types_lock);
5645 if (ret)
5646 return ret;
Steven Rostedt577b7852009-02-26 23:43:05 -05005647 }
5648
5649 *ppos += cnt;
5650
5651 return cnt;
5652}
5653
5654
5655static const struct file_operations trace_options_fops = {
5656 .open = tracing_open_generic,
5657 .read = trace_options_read,
5658 .write = trace_options_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005659 .llseek = generic_file_llseek,
Steven Rostedt577b7852009-02-26 23:43:05 -05005660};
5661
Steven Rostedta8259072009-02-26 22:19:12 -05005662static ssize_t
5663trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5664 loff_t *ppos)
5665{
5666 long index = (long)filp->private_data;
5667 char *buf;
5668
5669 if (trace_flags & (1 << index))
5670 buf = "1\n";
5671 else
5672 buf = "0\n";
5673
5674 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5675}
5676
5677static ssize_t
5678trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5679 loff_t *ppos)
5680{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005681 struct trace_array *tr = &global_trace;
Steven Rostedta8259072009-02-26 22:19:12 -05005682 long index = (long)filp->private_data;
Steven Rostedta8259072009-02-26 22:19:12 -05005683 unsigned long val;
5684 int ret;
5685
Peter Huewe22fe9b52011-06-07 21:58:27 +02005686 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5687 if (ret)
Steven Rostedta8259072009-02-26 22:19:12 -05005688 return ret;
5689
Zhaoleif2d84b62009-08-07 18:55:48 +08005690 if (val != 0 && val != 1)
Steven Rostedta8259072009-02-26 22:19:12 -05005691 return -EINVAL;
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04005692
5693 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005694 ret = set_tracer_flag(tr, 1 << index, val);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04005695 mutex_unlock(&trace_types_lock);
Steven Rostedta8259072009-02-26 22:19:12 -05005696
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04005697 if (ret < 0)
5698 return ret;
5699
Steven Rostedta8259072009-02-26 22:19:12 -05005700 *ppos += cnt;
5701
5702 return cnt;
5703}
5704
Steven Rostedta8259072009-02-26 22:19:12 -05005705static const struct file_operations trace_options_core_fops = {
5706 .open = tracing_open_generic,
5707 .read = trace_options_core_read,
5708 .write = trace_options_core_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005709 .llseek = generic_file_llseek,
Steven Rostedta8259072009-02-26 22:19:12 -05005710};
5711
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005712struct dentry *trace_create_file(const char *name,
Al Virof4ae40a2011-07-24 04:33:43 -04005713 umode_t mode,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005714 struct dentry *parent,
5715 void *data,
5716 const struct file_operations *fops)
5717{
5718 struct dentry *ret;
5719
5720 ret = debugfs_create_file(name, mode, parent, data, fops);
5721 if (!ret)
5722 pr_warning("Could not create debugfs '%s' entry\n", name);
5723
5724 return ret;
5725}
5726
5727
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005728static struct dentry *trace_options_init_dentry(struct trace_array *tr)
Steven Rostedta8259072009-02-26 22:19:12 -05005729{
5730 struct dentry *d_tracer;
Steven Rostedta8259072009-02-26 22:19:12 -05005731
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005732 if (tr->options)
5733 return tr->options;
Steven Rostedta8259072009-02-26 22:19:12 -05005734
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005735 d_tracer = tracing_init_dentry_tr(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05005736 if (!d_tracer)
5737 return NULL;
5738
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005739 tr->options = debugfs_create_dir("options", d_tracer);
5740 if (!tr->options) {
Steven Rostedta8259072009-02-26 22:19:12 -05005741 pr_warning("Could not create debugfs directory 'options'\n");
5742 return NULL;
5743 }
5744
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005745 return tr->options;
Steven Rostedta8259072009-02-26 22:19:12 -05005746}
5747
Steven Rostedt577b7852009-02-26 23:43:05 -05005748static void
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005749create_trace_option_file(struct trace_array *tr,
5750 struct trace_option_dentry *topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05005751 struct tracer_flags *flags,
5752 struct tracer_opt *opt)
5753{
5754 struct dentry *t_options;
Steven Rostedt577b7852009-02-26 23:43:05 -05005755
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005756 t_options = trace_options_init_dentry(tr);
Steven Rostedt577b7852009-02-26 23:43:05 -05005757 if (!t_options)
5758 return;
5759
5760 topt->flags = flags;
5761 topt->opt = opt;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005762 topt->tr = tr;
Steven Rostedt577b7852009-02-26 23:43:05 -05005763
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005764 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05005765 &trace_options_fops);
5766
Steven Rostedt577b7852009-02-26 23:43:05 -05005767}
5768
5769static struct trace_option_dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005770create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
Steven Rostedt577b7852009-02-26 23:43:05 -05005771{
5772 struct trace_option_dentry *topts;
5773 struct tracer_flags *flags;
5774 struct tracer_opt *opts;
5775 int cnt;
5776
5777 if (!tracer)
5778 return NULL;
5779
5780 flags = tracer->flags;
5781
5782 if (!flags || !flags->opts)
5783 return NULL;
5784
5785 opts = flags->opts;
5786
5787 for (cnt = 0; opts[cnt].name; cnt++)
5788 ;
5789
Steven Rostedt0cfe8242009-02-27 10:51:10 -05005790 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
Steven Rostedt577b7852009-02-26 23:43:05 -05005791 if (!topts)
5792 return NULL;
5793
5794 for (cnt = 0; opts[cnt].name; cnt++)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005795 create_trace_option_file(tr, &topts[cnt], flags,
Steven Rostedt577b7852009-02-26 23:43:05 -05005796 &opts[cnt]);
5797
5798 return topts;
5799}
5800
5801static void
5802destroy_trace_option_files(struct trace_option_dentry *topts)
5803{
5804 int cnt;
5805
5806 if (!topts)
5807 return;
5808
5809 for (cnt = 0; topts[cnt].opt; cnt++) {
5810 if (topts[cnt].entry)
5811 debugfs_remove(topts[cnt].entry);
5812 }
5813
5814 kfree(topts);
5815}
5816
Steven Rostedta8259072009-02-26 22:19:12 -05005817static struct dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005818create_trace_option_core_file(struct trace_array *tr,
5819 const char *option, long index)
Steven Rostedta8259072009-02-26 22:19:12 -05005820{
5821 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05005822
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005823 t_options = trace_options_init_dentry(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05005824 if (!t_options)
5825 return NULL;
5826
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005827 return trace_create_file(option, 0644, t_options, (void *)index,
Steven Rostedta8259072009-02-26 22:19:12 -05005828 &trace_options_core_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05005829}
5830
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005831static __init void create_trace_options_dir(struct trace_array *tr)
Steven Rostedta8259072009-02-26 22:19:12 -05005832{
5833 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05005834 int i;
5835
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005836 t_options = trace_options_init_dentry(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05005837 if (!t_options)
5838 return;
5839
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005840 for (i = 0; trace_options[i]; i++)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005841 create_trace_option_core_file(tr, trace_options[i], i);
Steven Rostedta8259072009-02-26 22:19:12 -05005842}
5843
Steven Rostedt499e5472012-02-22 15:50:28 -05005844static ssize_t
5845rb_simple_read(struct file *filp, char __user *ubuf,
5846 size_t cnt, loff_t *ppos)
5847{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04005848 struct trace_array *tr = filp->private_data;
Steven Rostedt499e5472012-02-22 15:50:28 -05005849 char buf[64];
5850 int r;
5851
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04005852 r = tracer_tracing_is_on(tr);
Steven Rostedt499e5472012-02-22 15:50:28 -05005853 r = sprintf(buf, "%d\n", r);
5854
5855 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5856}
5857
5858static ssize_t
5859rb_simple_write(struct file *filp, const char __user *ubuf,
5860 size_t cnt, loff_t *ppos)
5861{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04005862 struct trace_array *tr = filp->private_data;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005863 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt499e5472012-02-22 15:50:28 -05005864 unsigned long val;
5865 int ret;
5866
5867 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5868 if (ret)
5869 return ret;
5870
5871 if (buffer) {
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05005872 mutex_lock(&trace_types_lock);
5873 if (val) {
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04005874 tracer_tracing_on(tr);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005875 if (tr->current_trace->start)
5876 tr->current_trace->start(tr);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05005877 } else {
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04005878 tracer_tracing_off(tr);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005879 if (tr->current_trace->stop)
5880 tr->current_trace->stop(tr);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05005881 }
5882 mutex_unlock(&trace_types_lock);
Steven Rostedt499e5472012-02-22 15:50:28 -05005883 }
5884
5885 (*ppos)++;
5886
5887 return cnt;
5888}
5889
5890static const struct file_operations rb_simple_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005891 .open = tracing_open_generic_tr,
Steven Rostedt499e5472012-02-22 15:50:28 -05005892 .read = rb_simple_read,
5893 .write = rb_simple_write,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005894 .release = tracing_release_generic_tr,
Steven Rostedt499e5472012-02-22 15:50:28 -05005895 .llseek = default_llseek,
5896};
5897
Steven Rostedt277ba042012-08-03 16:10:49 -04005898struct dentry *trace_instance_dir;
5899
5900static void
5901init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5902
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005903static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
5904{
5905 int cpu;
5906
5907 for_each_tracing_cpu(cpu) {
5908 memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
5909 per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
5910 per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
5911 }
5912}
5913
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005914static int
5915allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
Steven Rostedt277ba042012-08-03 16:10:49 -04005916{
5917 enum ring_buffer_flags rb_flags;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005918
5919 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5920
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005921 buf->buffer = ring_buffer_alloc(size, rb_flags);
5922 if (!buf->buffer)
5923 return -ENOMEM;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005924
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005925 buf->data = alloc_percpu(struct trace_array_cpu);
5926 if (!buf->data) {
5927 ring_buffer_free(buf->buffer);
5928 return -ENOMEM;
5929 }
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005930
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005931 init_trace_buffers(tr, buf);
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005932
5933 /* Allocate the first page for all buffers */
5934 set_buffer_entries(&tr->trace_buffer,
5935 ring_buffer_size(tr->trace_buffer.buffer, 0));
5936
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005937 return 0;
5938}
5939
5940static int allocate_trace_buffers(struct trace_array *tr, int size)
5941{
5942 int ret;
5943
5944 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5945 if (ret)
5946 return ret;
5947
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005948#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005949 ret = allocate_trace_buffer(tr, &tr->max_buffer,
5950 allocate_snapshot ? size : 1);
5951 if (WARN_ON(ret)) {
5952 ring_buffer_free(tr->trace_buffer.buffer);
5953 free_percpu(tr->trace_buffer.data);
5954 return -ENOMEM;
5955 }
5956 tr->allocated_snapshot = allocate_snapshot;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005957
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005958 /*
5959 * Only the top level trace array gets its snapshot allocated
5960 * from the kernel command line.
5961 */
5962 allocate_snapshot = false;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005963#endif
5964 return 0;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005965}
5966
5967static int new_instance_create(const char *name)
5968{
Steven Rostedt277ba042012-08-03 16:10:49 -04005969 struct trace_array *tr;
5970 int ret;
Steven Rostedt277ba042012-08-03 16:10:49 -04005971
5972 mutex_lock(&trace_types_lock);
5973
5974 ret = -EEXIST;
5975 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5976 if (tr->name && strcmp(tr->name, name) == 0)
5977 goto out_unlock;
5978 }
5979
5980 ret = -ENOMEM;
5981 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5982 if (!tr)
5983 goto out_unlock;
5984
5985 tr->name = kstrdup(name, GFP_KERNEL);
5986 if (!tr->name)
5987 goto out_free_tr;
5988
5989 raw_spin_lock_init(&tr->start_lock);
5990
5991 tr->current_trace = &nop_trace;
5992
5993 INIT_LIST_HEAD(&tr->systems);
5994 INIT_LIST_HEAD(&tr->events);
5995
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005996 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
Steven Rostedt277ba042012-08-03 16:10:49 -04005997 goto out_free_tr;
5998
Steven Rostedt277ba042012-08-03 16:10:49 -04005999 /* Holder for file callbacks */
6000 tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6001 tr->trace_cpu.tr = tr;
6002
6003 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6004 if (!tr->dir)
6005 goto out_free_tr;
6006
6007 ret = event_trace_add_tracer(tr->dir, tr);
Alexander Z Lam609e85a2013-07-10 17:34:34 -07006008 if (ret) {
6009 debugfs_remove_recursive(tr->dir);
Steven Rostedt277ba042012-08-03 16:10:49 -04006010 goto out_free_tr;
Alexander Z Lam609e85a2013-07-10 17:34:34 -07006011 }
Steven Rostedt277ba042012-08-03 16:10:49 -04006012
6013 init_tracer_debugfs(tr, tr->dir);
6014
6015 list_add(&tr->list, &ftrace_trace_arrays);
6016
6017 mutex_unlock(&trace_types_lock);
6018
6019 return 0;
6020
6021 out_free_tr:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006022 if (tr->trace_buffer.buffer)
6023 ring_buffer_free(tr->trace_buffer.buffer);
Steven Rostedt277ba042012-08-03 16:10:49 -04006024 kfree(tr->name);
6025 kfree(tr);
6026
6027 out_unlock:
6028 mutex_unlock(&trace_types_lock);
6029
6030 return ret;
6031
6032}
6033
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006034static int instance_delete(const char *name)
6035{
6036 struct trace_array *tr;
6037 int found = 0;
6038 int ret;
6039
6040 mutex_lock(&trace_types_lock);
6041
6042 ret = -ENODEV;
6043 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6044 if (tr->name && strcmp(tr->name, name) == 0) {
6045 found = 1;
6046 break;
6047 }
6048 }
6049 if (!found)
6050 goto out_unlock;
6051
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05006052 ret = -EBUSY;
6053 if (tr->ref)
6054 goto out_unlock;
6055
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006056 list_del(&tr->list);
6057
6058 event_trace_del_tracer(tr);
6059 debugfs_remove_recursive(tr->dir);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006060 free_percpu(tr->trace_buffer.data);
6061 ring_buffer_free(tr->trace_buffer.buffer);
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006062
6063 kfree(tr->name);
6064 kfree(tr);
6065
6066 ret = 0;
6067
6068 out_unlock:
6069 mutex_unlock(&trace_types_lock);
6070
6071 return ret;
6072}
6073
Steven Rostedt277ba042012-08-03 16:10:49 -04006074static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6075{
6076 struct dentry *parent;
6077 int ret;
6078
6079 /* Paranoid: Make sure the parent is the "instances" directory */
6080 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6081 if (WARN_ON_ONCE(parent != trace_instance_dir))
6082 return -ENOENT;
6083
6084 /*
6085 * The inode mutex is locked, but debugfs_create_dir() will also
6086 * take the mutex. As the instances directory can not be destroyed
6087 * or changed in any other way, it is safe to unlock it, and
6088 * let the dentry try. If two users try to make the same dir at
6089 * the same time, then the new_instance_create() will determine the
6090 * winner.
6091 */
6092 mutex_unlock(&inode->i_mutex);
6093
6094 ret = new_instance_create(dentry->d_iname);
6095
6096 mutex_lock(&inode->i_mutex);
6097
6098 return ret;
6099}
6100
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006101static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6102{
6103 struct dentry *parent;
6104 int ret;
6105
6106 /* Paranoid: Make sure the parent is the "instances" directory */
6107 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6108 if (WARN_ON_ONCE(parent != trace_instance_dir))
6109 return -ENOENT;
6110
6111 /* The caller did a dget() on dentry */
6112 mutex_unlock(&dentry->d_inode->i_mutex);
6113
6114 /*
6115 * The inode mutex is locked, but debugfs_create_dir() will also
6116 * take the mutex. As the instances directory can not be destroyed
6117 * or changed in any other way, it is safe to unlock it, and
6118 * let the dentry try. If two users try to make the same dir at
6119 * the same time, then the instance_delete() will determine the
6120 * winner.
6121 */
6122 mutex_unlock(&inode->i_mutex);
6123
6124 ret = instance_delete(dentry->d_iname);
6125
6126 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6127 mutex_lock(&dentry->d_inode->i_mutex);
6128
6129 return ret;
6130}
6131
Steven Rostedt277ba042012-08-03 16:10:49 -04006132static const struct inode_operations instance_dir_inode_operations = {
6133 .lookup = simple_lookup,
6134 .mkdir = instance_mkdir,
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006135 .rmdir = instance_rmdir,
Steven Rostedt277ba042012-08-03 16:10:49 -04006136};
6137
6138static __init void create_trace_instances(struct dentry *d_tracer)
6139{
6140 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6141 if (WARN_ON(!trace_instance_dir))
6142 return;
6143
6144 /* Hijack the dir inode operations, to allow mkdir */
6145 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6146}
6147
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006148static void
6149init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6150{
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006151 int cpu;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006152
6153 trace_create_file("trace_options", 0644, d_tracer,
6154 tr, &tracing_iter_fops);
6155
6156 trace_create_file("trace", 0644, d_tracer,
6157 (void *)&tr->trace_cpu, &tracing_fops);
6158
6159 trace_create_file("trace_pipe", 0444, d_tracer,
6160 (void *)&tr->trace_cpu, &tracing_pipe_fops);
6161
6162 trace_create_file("buffer_size_kb", 0644, d_tracer,
6163 (void *)&tr->trace_cpu, &tracing_entries_fops);
6164
6165 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6166 tr, &tracing_total_entries_fops);
6167
Wang YanQing238ae932013-05-26 16:52:01 +08006168 trace_create_file("free_buffer", 0200, d_tracer,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006169 tr, &tracing_free_buffer_fops);
6170
6171 trace_create_file("trace_marker", 0220, d_tracer,
6172 tr, &tracing_mark_fops);
6173
6174 trace_create_file("trace_clock", 0644, d_tracer, tr,
6175 &trace_clock_fops);
6176
6177 trace_create_file("tracing_on", 0644, d_tracer,
6178 tr, &rb_simple_fops);
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05006179
6180#ifdef CONFIG_TRACER_SNAPSHOT
6181 trace_create_file("snapshot", 0644, d_tracer,
6182 (void *)&tr->trace_cpu, &snapshot_fops);
6183#endif
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006184
6185 for_each_tracing_cpu(cpu)
6186 tracing_init_debugfs_percpu(tr, cpu);
6187
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006188}
6189
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01006190static __init int tracer_init_debugfs(void)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006191{
6192 struct dentry *d_tracer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006193
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08006194 trace_access_lock_init();
6195
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006196 d_tracer = tracing_init_dentry();
Namhyung Kimed6f1c92013-04-10 09:18:12 +09006197 if (!d_tracer)
6198 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006199
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006200 init_tracer_debugfs(&global_trace, d_tracer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006201
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006202 trace_create_file("tracing_cpumask", 0644, d_tracer,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006203 &global_trace, &tracing_cpumask_fops);
Ingo Molnarc7078de2008-05-12 21:20:52 +02006204
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006205 trace_create_file("available_tracers", 0444, d_tracer,
6206 &global_trace, &show_traces_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006207
Li Zefan339ae5d2009-04-17 10:34:30 +08006208 trace_create_file("current_tracer", 0644, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006209 &global_trace, &set_tracer_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006210
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04006211#ifdef CONFIG_TRACER_MAX_TRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006212 trace_create_file("tracing_max_latency", 0644, d_tracer,
6213 &tracing_max_latency, &tracing_max_lat_fops);
Tim Bird0e950172010-02-25 15:36:43 -08006214#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006215
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006216 trace_create_file("tracing_thresh", 0644, d_tracer,
6217 &tracing_thresh, &tracing_max_lat_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006218
Li Zefan339ae5d2009-04-17 10:34:30 +08006219 trace_create_file("README", 0444, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006220 NULL, &tracing_readme_fops);
Ingo Molnar7bd2f242008-05-12 21:20:45 +02006221
Avadh Patel69abe6a2009-04-10 16:04:48 -04006222 trace_create_file("saved_cmdlines", 0444, d_tracer,
6223 NULL, &tracing_saved_cmdlines_fops);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03006224
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006225#ifdef CONFIG_DYNAMIC_FTRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006226 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6227 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006228#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006229
Steven Rostedt277ba042012-08-03 16:10:49 -04006230 create_trace_instances(d_tracer);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09006231
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006232 create_trace_options_dir(&global_trace);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006233
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01006234 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006235}
6236
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006237static int trace_panic_handler(struct notifier_block *this,
6238 unsigned long event, void *unused)
6239{
Steven Rostedt944ac422008-10-23 19:26:08 -04006240 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006241 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006242 return NOTIFY_OK;
6243}
6244
6245static struct notifier_block trace_panic_notifier = {
6246 .notifier_call = trace_panic_handler,
6247 .next = NULL,
6248 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6249};
6250
6251static int trace_die_handler(struct notifier_block *self,
6252 unsigned long val,
6253 void *data)
6254{
6255 switch (val) {
6256 case DIE_OOPS:
Steven Rostedt944ac422008-10-23 19:26:08 -04006257 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006258 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006259 break;
6260 default:
6261 break;
6262 }
6263 return NOTIFY_OK;
6264}
6265
6266static struct notifier_block trace_die_notifier = {
6267 .notifier_call = trace_die_handler,
6268 .priority = 200
6269};
6270
6271/*
6272 * printk is set to max of 1024, we really don't need it that big.
6273 * Nothing should be printing 1000 characters anyway.
6274 */
6275#define TRACE_MAX_PRINT 1000
6276
6277/*
6278 * Define here KERN_TRACE so that we have one place to modify
6279 * it if we decide to change what log level the ftrace dump
6280 * should be at.
6281 */
Steven Rostedt428aee12009-01-14 12:24:42 -05006282#define KERN_TRACE KERN_EMERG
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006283
Jason Wessel955b61e2010-08-05 09:22:23 -05006284void
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006285trace_printk_seq(struct trace_seq *s)
6286{
6287 /* Probably should print a warning here. */
zhangwei(Jovi)bd6df182013-03-11 15:13:37 +08006288 if (s->len >= TRACE_MAX_PRINT)
6289 s->len = TRACE_MAX_PRINT;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006290
6291 /* should be zero ended, but we are paranoid. */
6292 s->buffer[s->len] = 0;
6293
6294 printk(KERN_TRACE "%s", s->buffer);
6295
Steven Rostedtf9520752009-03-02 14:04:40 -05006296 trace_seq_init(s);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006297}
6298
Jason Wessel955b61e2010-08-05 09:22:23 -05006299void trace_init_global_iter(struct trace_iterator *iter)
6300{
6301 iter->tr = &global_trace;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006302 iter->trace = iter->tr->current_trace;
Steven Rostedtae3b5092013-01-23 15:22:59 -05006303 iter->cpu_file = RING_BUFFER_ALL_CPUS;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006304 iter->trace_buffer = &global_trace.trace_buffer;
Jason Wessel955b61e2010-08-05 09:22:23 -05006305}
6306
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04006307void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006308{
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006309 /* use static because iter can be a bit big for the stack */
6310 static struct trace_iterator iter;
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04006311 static atomic_t dump_running;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006312 unsigned int old_userobj;
Steven Rostedtd7690412008-10-01 00:29:53 -04006313 unsigned long flags;
6314 int cnt = 0, cpu;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006315
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04006316 /* Only allow one dump user at a time. */
6317 if (atomic_inc_return(&dump_running) != 1) {
6318 atomic_dec(&dump_running);
6319 return;
Steven Rostedte0a413f2011-09-29 21:26:16 -04006320 }
6321
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04006322 /*
6323 * Always turn off tracing when we dump.
6324 * We don't need to show trace output of what happens
6325 * between multiple crashes.
6326 *
6327 * If the user does a sysrq-z, then they can re-enable
6328 * tracing with echo 1 > tracing_on.
6329 */
6330 tracing_off();
6331
6332 local_irq_save(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006333
Jovi Zhang38dbe0b2013-01-25 18:03:07 +08006334 /* Simulate the iterator */
Jason Wessel955b61e2010-08-05 09:22:23 -05006335 trace_init_global_iter(&iter);
6336
Steven Rostedtd7690412008-10-01 00:29:53 -04006337 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006338 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -04006339 }
6340
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006341 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6342
Török Edwinb54d3de2008-11-22 13:28:48 +02006343 /* don't look at user memory in panic mode */
6344 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6345
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006346 switch (oops_dump_mode) {
6347 case DUMP_ALL:
Steven Rostedtae3b5092013-01-23 15:22:59 -05006348 iter.cpu_file = RING_BUFFER_ALL_CPUS;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006349 break;
6350 case DUMP_ORIG:
6351 iter.cpu_file = raw_smp_processor_id();
6352 break;
6353 case DUMP_NONE:
6354 goto out_enable;
6355 default:
6356 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
Steven Rostedtae3b5092013-01-23 15:22:59 -05006357 iter.cpu_file = RING_BUFFER_ALL_CPUS;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006358 }
6359
6360 printk(KERN_TRACE "Dumping ftrace buffer:\n");
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006361
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04006362 /* Did function tracer already get disabled? */
6363 if (ftrace_is_dead()) {
6364 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6365 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6366 }
6367
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006368 /*
6369 * We need to stop all tracing on all CPUS to read the
6370 * the next buffer. This is a bit expensive, but is
6371 * not done often. We fill all what we can read,
6372 * and then release the locks again.
6373 */
6374
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006375 while (!trace_empty(&iter)) {
6376
6377 if (!cnt)
6378 printk(KERN_TRACE "---------------------------------\n");
6379
6380 cnt++;
6381
6382 /* reset all but tr, trace, and overruns */
6383 memset(&iter.seq, 0,
6384 sizeof(struct trace_iterator) -
6385 offsetof(struct trace_iterator, seq));
6386 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6387 iter.pos = -1;
6388
Jason Wessel955b61e2010-08-05 09:22:23 -05006389 if (trace_find_next_entry_inc(&iter) != NULL) {
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08006390 int ret;
6391
6392 ret = print_trace_line(&iter);
6393 if (ret != TRACE_TYPE_NO_CONSUME)
6394 trace_consume(&iter);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006395 }
Steven Rostedtb892e5c2012-03-01 22:06:48 -05006396 touch_nmi_watchdog();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006397
6398 trace_printk_seq(&iter.seq);
6399 }
6400
6401 if (!cnt)
6402 printk(KERN_TRACE " (ftrace buffer empty)\n");
6403 else
6404 printk(KERN_TRACE "---------------------------------\n");
6405
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006406 out_enable:
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04006407 trace_flags |= old_userobj;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006408
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04006409 for_each_tracing_cpu(cpu) {
6410 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006411 }
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04006412 atomic_dec(&dump_running);
Steven Rostedtcd891ae2009-04-28 11:39:34 -04006413 local_irq_restore(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006414}
Paul E. McKenneya8eecf22011-10-02 11:01:15 -07006415EXPORT_SYMBOL_GPL(ftrace_dump);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006416
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006417__init static int tracer_alloc_buffers(void)
6418{
Steven Rostedt73c51622009-03-11 13:42:01 -04006419 int ring_buf_size;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306420 int ret = -ENOMEM;
6421
David Sharp750912f2010-12-08 13:46:47 -08006422
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306423 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6424 goto out;
6425
6426 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
6427 goto out_free_buffer_mask;
6428
Steven Rostedt07d777f2011-09-22 14:01:55 -04006429 /* Only allocate trace_printk buffers if a trace_printk exists */
6430 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
Steven Rostedt81698832012-10-11 10:15:05 -04006431 /* Must be called before global_trace.buffer is allocated */
Steven Rostedt07d777f2011-09-22 14:01:55 -04006432 trace_printk_init_buffers();
6433
Steven Rostedt73c51622009-03-11 13:42:01 -04006434 /* To save memory, keep the ring buffer size to its minimum */
6435 if (ring_buffer_expanded)
6436 ring_buf_size = trace_buf_size;
6437 else
6438 ring_buf_size = 1;
6439
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306440 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6441 cpumask_copy(tracing_cpumask, cpu_all_mask);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006442
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006443 raw_spin_lock_init(&global_trace.start_lock);
6444
Steven Rostedtab464282008-05-12 21:21:00 +02006445 /* TODO: make the number of buffers hot pluggable with CPUS */
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006446 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04006447 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6448 WARN_ON(1);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306449 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04006450 }
Steven Rostedta7603ff2012-08-06 16:24:11 -04006451
Steven Rostedt499e5472012-02-22 15:50:28 -05006452 if (global_trace.buffer_disabled)
6453 tracing_off();
Steven Rostedt3928a8a2008-09-29 23:02:41 -04006454
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006455 trace_init_cmdlines();
6456
Steven Rostedt (Red Hat)ca164312013-05-23 11:51:10 -04006457 /*
6458 * register_tracer() might reference current_trace, so it
6459 * needs to be set before we register anything. This is
6460 * just a bootstrap of current_trace anyway.
6461 */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006462 global_trace.current_trace = &nop_trace;
6463
Steven Rostedt (Red Hat)ca164312013-05-23 11:51:10 -04006464 register_tracer(&nop_trace);
6465
Steven Rostedt60a11772008-05-12 21:20:44 +02006466 /* All seems OK, enable tracing */
6467 tracing_disabled = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04006468
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006469 atomic_notifier_chain_register(&panic_notifier_list,
6470 &trace_panic_notifier);
6471
6472 register_die_notifier(&trace_die_notifier);
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01006473
Steven Rostedtae63b312012-05-03 23:09:03 -04006474 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6475
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006476 /* Holder for file callbacks */
6477 global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6478 global_trace.trace_cpu.tr = &global_trace;
6479
Steven Rostedtae63b312012-05-03 23:09:03 -04006480 INIT_LIST_HEAD(&global_trace.systems);
6481 INIT_LIST_HEAD(&global_trace.events);
6482 list_add(&global_trace.list, &ftrace_trace_arrays);
6483
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04006484 while (trace_boot_options) {
6485 char *option;
6486
6487 option = strsep(&trace_boot_options, ",");
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006488 trace_set_options(&global_trace, option);
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04006489 }
6490
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04006491 register_snapshot_cmd();
6492
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01006493 return 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006494
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306495out_free_cpumask:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006496 free_percpu(global_trace.trace_buffer.data);
6497#ifdef CONFIG_TRACER_MAX_TRACE
6498 free_percpu(global_trace.max_buffer.data);
6499#endif
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306500 free_cpumask_var(tracing_cpumask);
6501out_free_buffer_mask:
6502 free_cpumask_var(tracing_buffer_mask);
6503out:
6504 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006505}
Steven Rostedtb2821ae2009-02-02 21:38:32 -05006506
6507__init static int clear_boot_tracer(void)
6508{
6509 /*
6510 * The default tracer at boot buffer is an init section.
6511 * This function is called in lateinit. If we did not
6512 * find the boot tracer, then clear it out, to prevent
6513 * later registration from accessing the buffer that is
6514 * about to be freed.
6515 */
6516 if (!default_bootup_tracer)
6517 return 0;
6518
6519 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6520 default_bootup_tracer);
6521 default_bootup_tracer = NULL;
6522
6523 return 0;
6524}
6525
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01006526early_initcall(tracer_alloc_buffers);
6527fs_initcall(tracer_init_debugfs);
Steven Rostedtb2821ae2009-02-02 21:38:32 -05006528late_initcall(clear_boot_tracer);