blob: bba1ba958ee8dfc235e7015c6833dc5f26969485 [file] [log] [blame]
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001/*
2 * ring buffer based function tracer
3 *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +010012 * Copyright (C) 2004 Nadia Yvette Chambers
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020013 */
Steven Rostedt2cadf912008-12-01 22:20:19 -050014#include <linux/ring_buffer.h>
Sam Ravnborg273b2812009-10-18 00:52:28 +020015#include <generated/utsrelease.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050016#include <linux/stacktrace.h>
17#include <linux/writeback.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020018#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040020#include <linux/notifier.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050021#include <linux/irqflags.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020022#include <linux/debugfs.h>
Steven Rostedt4c11d7a2008-05-12 21:20:43 +020023#include <linux/pagemap.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020024#include <linux/hardirq.h>
25#include <linux/linkage.h>
26#include <linux/uaccess.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050027#include <linux/kprobes.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020028#include <linux/ftrace.h>
29#include <linux/module.h>
30#include <linux/percpu.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050031#include <linux/splice.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040032#include <linux/kdebug.h>
Frederic Weisbecker5f0c6c02009-03-27 14:22:10 +010033#include <linux/string.h>
Lai Jiangshan7e53bd42010-01-06 20:08:50 +080034#include <linux/rwsem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090035#include <linux/slab.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020036#include <linux/ctype.h>
37#include <linux/init.h>
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +020038#include <linux/poll.h>
Steven Rostedtb892e5c2012-03-01 22:06:48 -050039#include <linux/nmi.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020040#include <linux/fs.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060041#include <linux/sched/rt.h>
Ingo Molnar86387f72008-05-12 21:20:51 +020042
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020043#include "trace.h"
Steven Rostedtf0868d12008-12-23 23:24:12 -050044#include "trace_output.h"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020045
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010046/*
Steven Rostedt73c51622009-03-11 13:42:01 -040047 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
49 */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -050050bool ring_buffer_expanded;
Steven Rostedt73c51622009-03-11 13:42:01 -040051
52/*
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010053 * We need to change this state when a selftest is running.
Frederic Weisbeckerff325042008-12-04 23:47:35 +010054 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
Ingo Molnar5e1607a2009-03-05 10:24:48 +010056 * insertions into the ring-buffer such as trace_printk could occurred
Frederic Weisbeckerff325042008-12-04 23:47:35 +010057 * at the same time, giving false positive or negative results.
58 */
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010059static bool __read_mostly tracing_selftest_running;
Frederic Weisbeckerff325042008-12-04 23:47:35 +010060
Steven Rostedtb2821ae2009-02-02 21:38:32 -050061/*
62 * If a tracer is running, we do not want to run SELFTEST.
63 */
Li Zefan020e5f82009-07-01 10:47:05 +080064bool __read_mostly tracing_selftest_disabled;
Steven Rostedtb2821ae2009-02-02 21:38:32 -050065
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010066/* For tracers that don't implement custom flags */
67static struct tracer_opt dummy_tracer_opt[] = {
68 { }
69};
70
71static struct tracer_flags dummy_tracer_flags = {
72 .val = 0,
73 .opts = dummy_tracer_opt
74};
75
76static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77{
78 return 0;
79}
Steven Rostedt0f048702008-11-05 16:05:44 -050080
81/*
Steven Rostedt7ffbd482012-10-11 12:14:25 -040082 * To prevent the comm cache from being overwritten when no
83 * tracing is active, only save the comm when a trace event
84 * occurred.
85 */
86static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88/*
Steven Rostedt0f048702008-11-05 16:05:44 -050089 * Kill all tracing for good (never come back).
90 * It is initialized to 1 but will turn to zero if the initialization
91 * of the tracer is successful. But that is the only place that sets
92 * this back to zero.
93 */
Hannes Eder4fd27352009-02-10 19:44:12 +010094static int tracing_disabled = 1;
Steven Rostedt0f048702008-11-05 16:05:44 -050095
Christoph Lameter9288f992009-10-07 19:17:45 -040096DEFINE_PER_CPU(int, ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -040097
Jason Wessel955b61e2010-08-05 09:22:23 -050098cpumask_var_t __read_mostly tracing_buffer_mask;
Steven Rostedtab464282008-05-12 21:21:00 +020099
Steven Rostedt944ac422008-10-23 19:26:08 -0400100/*
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102 *
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
107 * serial console.
108 *
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
Steven Rostedt944ac422008-10-23 19:26:08 -0400114 */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200115
116enum ftrace_dump_mode ftrace_dump_on_oops;
Steven Rostedt944ac422008-10-23 19:26:08 -0400117
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500118static int tracing_set_tracer(const char *buf);
119
Li Zefanee6c2c12009-09-18 14:06:47 +0800120#define MAX_TRACER_SIZE 100
121static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500122static char *default_bootup_tracer;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100123
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500124static bool allocate_snapshot;
125
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200126static int __init set_cmdline_ftrace(char *str)
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100127{
Li Zefanee6c2c12009-09-18 14:06:47 +0800128 strncpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500129 default_bootup_tracer = bootup_tracer_buf;
Steven Rostedt73c51622009-03-11 13:42:01 -0400130 /* We are using ftrace early, expand it */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500131 ring_buffer_expanded = true;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100132 return 1;
133}
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200134__setup("ftrace=", set_cmdline_ftrace);
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100135
Steven Rostedt944ac422008-10-23 19:26:08 -0400136static int __init set_ftrace_dump_on_oops(char *str)
137{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200138 if (*str++ != '=' || !*str) {
139 ftrace_dump_on_oops = DUMP_ALL;
140 return 1;
141 }
142
143 if (!strcmp("orig_cpu", str)) {
144 ftrace_dump_on_oops = DUMP_ORIG;
145 return 1;
146 }
147
148 return 0;
Steven Rostedt944ac422008-10-23 19:26:08 -0400149}
150__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
Steven Rostedt60a11772008-05-12 21:20:44 +0200151
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400152static int __init boot_alloc_snapshot(char *str)
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500153{
154 allocate_snapshot = true;
155 /* We also need the main ring buffer expanded */
156 ring_buffer_expanded = true;
157 return 1;
158}
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400159__setup("alloc_snapshot", boot_alloc_snapshot);
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500160
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400161
162static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
163static char *trace_boot_options __initdata;
164
165static int __init set_trace_boot_options(char *str)
166{
167 strncpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
168 trace_boot_options = trace_boot_options_buf;
169 return 0;
170}
171__setup("trace_options=", set_trace_boot_options);
172
Lai Jiangshancf8e3472009-03-30 13:48:00 +0800173unsigned long long ns2usecs(cycle_t nsec)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200174{
175 nsec += 500;
176 do_div(nsec, 1000);
177 return nsec;
178}
179
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200180/*
181 * The global_trace is the descriptor that holds the tracing
182 * buffers for the live tracing. For each CPU, it contains
183 * a link list of pages that will store trace entries. The
184 * page descriptor of the pages in the memory is used to hold
185 * the link list by linking the lru item in the page descriptor
186 * to each of the pages in the buffer per CPU.
187 *
188 * For each active CPU there is a data field that holds the
189 * pages for the buffer for that CPU. Each CPU has the same number
190 * of pages allocated for its buffer.
191 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200192static struct trace_array global_trace;
193
Steven Rostedtae63b312012-05-03 23:09:03 -0400194LIST_HEAD(ftrace_trace_arrays);
195
Steven Rostedte77405a2009-09-02 14:17:06 -0400196int filter_current_check_discard(struct ring_buffer *buffer,
197 struct ftrace_event_call *call, void *rec,
Tom Zanussieb02ce02009-04-08 03:15:54 -0500198 struct ring_buffer_event *event)
199{
Steven Rostedte77405a2009-09-02 14:17:06 -0400200 return filter_check_discard(call, rec, buffer, event);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500201}
Steven Rostedt17c873e2009-04-10 18:12:50 -0400202EXPORT_SYMBOL_GPL(filter_current_check_discard);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500203
Steven Rostedt37886f62009-03-17 17:22:06 -0400204cycle_t ftrace_now(int cpu)
205{
206 u64 ts;
207
208 /* Early boot up does not have a buffer yet */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500209 if (!global_trace.trace_buffer.buffer)
Steven Rostedt37886f62009-03-17 17:22:06 -0400210 return trace_clock_local();
211
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500212 ts = ring_buffer_time_stamp(global_trace.trace_buffer.buffer, cpu);
213 ring_buffer_normalize_time_stamp(global_trace.trace_buffer.buffer, cpu, &ts);
Steven Rostedt37886f62009-03-17 17:22:06 -0400214
215 return ts;
216}
217
Steven Rostedt90369902008-11-05 16:05:44 -0500218int tracing_is_enabled(void)
219{
Steven Rostedt0fb96562012-05-11 14:25:30 -0400220 return tracing_is_on();
Steven Rostedt90369902008-11-05 16:05:44 -0500221}
222
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200223/*
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400224 * trace_buf_size is the size in bytes that is allocated
225 * for a buffer. Note, the number of bytes is always rounded
226 * to page size.
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400227 *
228 * This number is purposely set to a low number of 16384.
229 * If the dump on oops happens, it will be much appreciated
230 * to not have to wait for all that output. Anyway this can be
231 * boot time and run time configurable.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200232 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400233#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400234
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400235static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200236
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200237/* trace_types holds a link list of available tracers. */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200238static struct tracer *trace_types __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200239
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200240/*
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200241 * trace_types_lock is used to protect the trace_types list.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200242 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200243static DEFINE_MUTEX(trace_types_lock);
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200244
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800245/*
246 * serialize the access of the ring buffer
247 *
248 * ring buffer serializes readers, but it is low level protection.
249 * The validity of the events (which returns by ring_buffer_peek() ..etc)
250 * are not protected by ring buffer.
251 *
252 * The content of events may become garbage if we allow other process consumes
253 * these events concurrently:
254 * A) the page of the consumed events may become a normal page
255 * (not reader page) in ring buffer, and this page will be rewrited
256 * by events producer.
257 * B) The page of the consumed events may become a page for splice_read,
258 * and this page will be returned to system.
259 *
260 * These primitives allow multi process access to different cpu ring buffer
261 * concurrently.
262 *
263 * These primitives don't distinguish read-only and read-consume access.
264 * Multi read-only access are also serialized.
265 */
266
267#ifdef CONFIG_SMP
268static DECLARE_RWSEM(all_cpu_access_lock);
269static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
270
271static inline void trace_access_lock(int cpu)
272{
Steven Rostedtae3b5092013-01-23 15:22:59 -0500273 if (cpu == RING_BUFFER_ALL_CPUS) {
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800274 /* gain it for accessing the whole ring buffer. */
275 down_write(&all_cpu_access_lock);
276 } else {
277 /* gain it for accessing a cpu ring buffer. */
278
Steven Rostedtae3b5092013-01-23 15:22:59 -0500279 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800280 down_read(&all_cpu_access_lock);
281
282 /* Secondly block other access to this @cpu ring buffer. */
283 mutex_lock(&per_cpu(cpu_access_lock, cpu));
284 }
285}
286
287static inline void trace_access_unlock(int cpu)
288{
Steven Rostedtae3b5092013-01-23 15:22:59 -0500289 if (cpu == RING_BUFFER_ALL_CPUS) {
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800290 up_write(&all_cpu_access_lock);
291 } else {
292 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
293 up_read(&all_cpu_access_lock);
294 }
295}
296
297static inline void trace_access_lock_init(void)
298{
299 int cpu;
300
301 for_each_possible_cpu(cpu)
302 mutex_init(&per_cpu(cpu_access_lock, cpu));
303}
304
305#else
306
307static DEFINE_MUTEX(access_lock);
308
309static inline void trace_access_lock(int cpu)
310{
311 (void)cpu;
312 mutex_lock(&access_lock);
313}
314
315static inline void trace_access_unlock(int cpu)
316{
317 (void)cpu;
318 mutex_unlock(&access_lock);
319}
320
321static inline void trace_access_lock_init(void)
322{
323}
324
325#endif
326
Steven Rostedtee6bce52008-11-12 17:52:37 -0500327/* trace_flags holds trace_options default values */
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500328unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
Steven Rostedta2a16d62009-03-24 23:17:58 -0400329 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
Steven Rostedt77271ce2011-11-17 09:34:33 -0500330 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
Steven Rostedt (Red Hat)328df472013-03-14 12:10:40 -0400331 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
Ingo Molnar4e655512008-05-12 21:20:52 +0200332
Steven Rostedt0d5c6e12012-11-01 20:54:21 -0400333/**
Steven Rostedt499e5472012-02-22 15:50:28 -0500334 * tracing_on - enable tracing buffers
335 *
336 * This function enables tracing buffers that may have been
337 * disabled with tracing_off.
338 */
339void tracing_on(void)
340{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500341 if (global_trace.trace_buffer.buffer)
342 ring_buffer_record_on(global_trace.trace_buffer.buffer);
Steven Rostedt499e5472012-02-22 15:50:28 -0500343 /*
344 * This flag is only looked at when buffers haven't been
345 * allocated yet. We don't really care about the race
346 * between setting this flag and actually turning
347 * on the buffer.
348 */
349 global_trace.buffer_disabled = 0;
350}
351EXPORT_SYMBOL_GPL(tracing_on);
352
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500353/**
354 * __trace_puts - write a constant string into the trace buffer.
355 * @ip: The address of the caller
356 * @str: The constant string to write
357 * @size: The size of the string.
358 */
359int __trace_puts(unsigned long ip, const char *str, int size)
360{
361 struct ring_buffer_event *event;
362 struct ring_buffer *buffer;
363 struct print_entry *entry;
364 unsigned long irq_flags;
365 int alloc;
366
367 alloc = sizeof(*entry) + size + 2; /* possible \n added */
368
369 local_save_flags(irq_flags);
370 buffer = global_trace.trace_buffer.buffer;
371 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
372 irq_flags, preempt_count());
373 if (!event)
374 return 0;
375
376 entry = ring_buffer_event_data(event);
377 entry->ip = ip;
378
379 memcpy(&entry->buf, str, size);
380
381 /* Add a newline if necessary */
382 if (entry->buf[size - 1] != '\n') {
383 entry->buf[size] = '\n';
384 entry->buf[size + 1] = '\0';
385 } else
386 entry->buf[size] = '\0';
387
388 __buffer_unlock_commit(buffer, event);
389
390 return size;
391}
392EXPORT_SYMBOL_GPL(__trace_puts);
393
394/**
395 * __trace_bputs - write the pointer to a constant string into trace buffer
396 * @ip: The address of the caller
397 * @str: The constant string to write to the buffer to
398 */
399int __trace_bputs(unsigned long ip, const char *str)
400{
401 struct ring_buffer_event *event;
402 struct ring_buffer *buffer;
403 struct bputs_entry *entry;
404 unsigned long irq_flags;
405 int size = sizeof(struct bputs_entry);
406
407 local_save_flags(irq_flags);
408 buffer = global_trace.trace_buffer.buffer;
409 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
410 irq_flags, preempt_count());
411 if (!event)
412 return 0;
413
414 entry = ring_buffer_event_data(event);
415 entry->ip = ip;
416 entry->str = str;
417
418 __buffer_unlock_commit(buffer, event);
419
420 return 1;
421}
422EXPORT_SYMBOL_GPL(__trace_bputs);
423
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500424#ifdef CONFIG_TRACER_SNAPSHOT
425/**
426 * trace_snapshot - take a snapshot of the current buffer.
427 *
428 * This causes a swap between the snapshot buffer and the current live
429 * tracing buffer. You can use this to take snapshots of the live
430 * trace when some condition is triggered, but continue to trace.
431 *
432 * Note, make sure to allocate the snapshot with either
433 * a tracing_snapshot_alloc(), or by doing it manually
434 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
435 *
436 * If the snapshot buffer is not allocated, it will stop tracing.
437 * Basically making a permanent snapshot.
438 */
439void tracing_snapshot(void)
440{
441 struct trace_array *tr = &global_trace;
442 struct tracer *tracer = tr->current_trace;
443 unsigned long flags;
444
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500445 if (in_nmi()) {
446 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
447 internal_trace_puts("*** snapshot is being ignored ***\n");
448 return;
449 }
450
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500451 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)ca268da2013-03-09 00:40:58 -0500452 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
453 internal_trace_puts("*** stopping trace here! ***\n");
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500454 tracing_off();
455 return;
456 }
457
458 /* Note, snapshot can not be used when the tracer uses it */
459 if (tracer->use_max_tr) {
Steven Rostedt (Red Hat)ca268da2013-03-09 00:40:58 -0500460 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
461 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500462 return;
463 }
464
465 local_irq_save(flags);
466 update_max_tr(tr, current, smp_processor_id());
467 local_irq_restore(flags);
468}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500469EXPORT_SYMBOL_GPL(tracing_snapshot);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500470
471static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
472 struct trace_buffer *size_buf, int cpu_id);
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400473static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
474
475static int alloc_snapshot(struct trace_array *tr)
476{
477 int ret;
478
479 if (!tr->allocated_snapshot) {
480
481 /* allocate spare buffer */
482 ret = resize_buffer_duplicate_size(&tr->max_buffer,
483 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
484 if (ret < 0)
485 return ret;
486
487 tr->allocated_snapshot = true;
488 }
489
490 return 0;
491}
492
493void free_snapshot(struct trace_array *tr)
494{
495 /*
496 * We don't free the ring buffer. instead, resize it because
497 * The max_tr ring buffer has some state (e.g. ring->clock) and
498 * we want preserve it.
499 */
500 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
501 set_buffer_entries(&tr->max_buffer, 1);
502 tracing_reset_online_cpus(&tr->max_buffer);
503 tr->allocated_snapshot = false;
504}
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500505
506/**
507 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
508 *
509 * This is similar to trace_snapshot(), but it will allocate the
510 * snapshot buffer if it isn't already allocated. Use this only
511 * where it is safe to sleep, as the allocation may sleep.
512 *
513 * This causes a swap between the snapshot buffer and the current live
514 * tracing buffer. You can use this to take snapshots of the live
515 * trace when some condition is triggered, but continue to trace.
516 */
517void tracing_snapshot_alloc(void)
518{
519 struct trace_array *tr = &global_trace;
520 int ret;
521
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400522 ret = alloc_snapshot(tr);
523 if (WARN_ON(ret < 0))
524 return;
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500525
526 tracing_snapshot();
527}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500528EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500529#else
530void tracing_snapshot(void)
531{
532 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
533}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500534EXPORT_SYMBOL_GPL(tracing_snapshot);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500535void tracing_snapshot_alloc(void)
536{
537 /* Give warning */
538 tracing_snapshot();
539}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500540EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500541#endif /* CONFIG_TRACER_SNAPSHOT */
542
Steven Rostedt499e5472012-02-22 15:50:28 -0500543/**
544 * tracing_off - turn off tracing buffers
545 *
546 * This function stops the tracing buffers from recording data.
547 * It does not disable any overhead the tracers themselves may
548 * be causing. This function simply causes all recording to
549 * the ring buffers to fail.
550 */
551void tracing_off(void)
552{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500553 if (global_trace.trace_buffer.buffer)
554 ring_buffer_record_off(global_trace.trace_buffer.buffer);
Steven Rostedt499e5472012-02-22 15:50:28 -0500555 /*
556 * This flag is only looked at when buffers haven't been
557 * allocated yet. We don't really care about the race
558 * between setting this flag and actually turning
559 * on the buffer.
560 */
561 global_trace.buffer_disabled = 1;
562}
563EXPORT_SYMBOL_GPL(tracing_off);
564
565/**
566 * tracing_is_on - show state of ring buffers enabled
567 */
568int tracing_is_on(void)
569{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500570 if (global_trace.trace_buffer.buffer)
571 return ring_buffer_record_is_on(global_trace.trace_buffer.buffer);
Steven Rostedt499e5472012-02-22 15:50:28 -0500572 return !global_trace.buffer_disabled;
573}
574EXPORT_SYMBOL_GPL(tracing_is_on);
575
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400576static int __init set_buf_size(char *str)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200577{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400578 unsigned long buf_size;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200579
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200580 if (!str)
581 return 0;
Li Zefan9d612be2009-06-24 17:33:15 +0800582 buf_size = memparse(str, &str);
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200583 /* nr_entries can not be zero */
Li Zefan9d612be2009-06-24 17:33:15 +0800584 if (buf_size == 0)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200585 return 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400586 trace_buf_size = buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200587 return 1;
588}
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400589__setup("trace_buf_size=", set_buf_size);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200590
Tim Bird0e950172010-02-25 15:36:43 -0800591static int __init set_tracing_thresh(char *str)
592{
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800593 unsigned long threshold;
Tim Bird0e950172010-02-25 15:36:43 -0800594 int ret;
595
596 if (!str)
597 return 0;
Daniel Walterbcd83ea2012-09-26 22:08:38 +0200598 ret = kstrtoul(str, 0, &threshold);
Tim Bird0e950172010-02-25 15:36:43 -0800599 if (ret < 0)
600 return 0;
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800601 tracing_thresh = threshold * 1000;
Tim Bird0e950172010-02-25 15:36:43 -0800602 return 1;
603}
604__setup("tracing_thresh=", set_tracing_thresh);
605
Steven Rostedt57f50be2008-05-12 21:20:44 +0200606unsigned long nsecs_to_usecs(unsigned long nsecs)
607{
608 return nsecs / 1000;
609}
610
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200611/* These must match the bit postions in trace_iterator_flags */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200612static const char *trace_options[] = {
613 "print-parent",
614 "sym-offset",
615 "sym-addr",
616 "verbose",
Ingo Molnarf9896bf2008-05-12 21:20:47 +0200617 "raw",
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +0200618 "hex",
Ingo Molnarcb0f12a2008-05-12 21:20:47 +0200619 "bin",
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +0200620 "block",
Ingo Molnar86387f72008-05-12 21:20:51 +0200621 "stacktrace",
Ingo Molnar5e1607a2009-03-05 10:24:48 +0100622 "trace_printk",
Steven Rostedtb2a866f2008-11-03 23:15:57 -0500623 "ftrace_preempt",
Steven Rostedt9f029e82008-11-12 15:24:24 -0500624 "branch",
Steven Rostedt12ef7d42008-11-12 17:52:38 -0500625 "annotate",
Török Edwin02b67512008-11-22 13:28:47 +0200626 "userstacktrace",
Török Edwinb54d3de2008-11-22 13:28:48 +0200627 "sym-userobj",
Frederic Weisbecker66896a82008-12-13 20:18:13 +0100628 "printk-msg-only",
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -0200629 "context-info",
Steven Rostedtc032ef642009-03-04 20:34:24 -0500630 "latency-format",
Steven Rostedtbe6f1642009-03-24 11:06:24 -0400631 "sleep-time",
Steven Rostedta2a16d62009-03-24 23:17:58 -0400632 "graph-time",
Li Zefane870e9a2010-07-02 11:07:32 +0800633 "record-cmd",
David Sharp750912f2010-12-08 13:46:47 -0800634 "overwrite",
Steven Rostedtcf30cf62011-06-14 22:44:07 -0400635 "disable_on_free",
Steven Rostedt77271ce2011-11-17 09:34:33 -0500636 "irq-info",
Mandeep Singh Baines5224c3a2012-09-07 18:12:19 -0700637 "markers",
Steven Rostedt (Red Hat)328df472013-03-14 12:10:40 -0400638 "function-trace",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200639 NULL
640};
641
Zhaolei5079f322009-08-25 16:12:56 +0800642static struct {
643 u64 (*func)(void);
644 const char *name;
David Sharp8be07092012-11-13 12:18:22 -0800645 int in_ns; /* is this clock in nanoseconds? */
Zhaolei5079f322009-08-25 16:12:56 +0800646} trace_clocks[] = {
David Sharp8be07092012-11-13 12:18:22 -0800647 { trace_clock_local, "local", 1 },
648 { trace_clock_global, "global", 1 },
649 { trace_clock_counter, "counter", 0 },
Steven Rostedt (Red Hat)8aacf012013-03-14 13:13:45 -0400650 { trace_clock_jiffies, "uptime", 1 },
Steven Rostedt (Red Hat)76f11912013-03-14 17:53:25 -0400651 { trace_clock, "perf", 1 },
David Sharp8cbd9cc2012-11-13 12:18:21 -0800652 ARCH_TRACE_CLOCKS
Zhaolei5079f322009-08-25 16:12:56 +0800653};
654
655int trace_clock_id;
656
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200657/*
658 * trace_parser_get_init - gets the buffer for trace parser
659 */
660int trace_parser_get_init(struct trace_parser *parser, int size)
661{
662 memset(parser, 0, sizeof(*parser));
663
664 parser->buffer = kmalloc(size, GFP_KERNEL);
665 if (!parser->buffer)
666 return 1;
667
668 parser->size = size;
669 return 0;
670}
671
672/*
673 * trace_parser_put - frees the buffer for trace parser
674 */
675void trace_parser_put(struct trace_parser *parser)
676{
677 kfree(parser->buffer);
678}
679
680/*
681 * trace_get_user - reads the user input string separated by space
682 * (matched by isspace(ch))
683 *
684 * For each string found the 'struct trace_parser' is updated,
685 * and the function returns.
686 *
687 * Returns number of bytes read.
688 *
689 * See kernel/trace/trace.h for 'struct trace_parser' details.
690 */
691int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
692 size_t cnt, loff_t *ppos)
693{
694 char ch;
695 size_t read = 0;
696 ssize_t ret;
697
698 if (!*ppos)
699 trace_parser_clear(parser);
700
701 ret = get_user(ch, ubuf++);
702 if (ret)
703 goto out;
704
705 read++;
706 cnt--;
707
708 /*
709 * The parser is not finished with the last write,
710 * continue reading the user input without skipping spaces.
711 */
712 if (!parser->cont) {
713 /* skip white space */
714 while (cnt && isspace(ch)) {
715 ret = get_user(ch, ubuf++);
716 if (ret)
717 goto out;
718 read++;
719 cnt--;
720 }
721
722 /* only spaces were written */
723 if (isspace(ch)) {
724 *ppos += read;
725 ret = read;
726 goto out;
727 }
728
729 parser->idx = 0;
730 }
731
732 /* read the non-space input */
733 while (cnt && !isspace(ch)) {
Li Zefan3c235a32009-09-22 13:51:54 +0800734 if (parser->idx < parser->size - 1)
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200735 parser->buffer[parser->idx++] = ch;
736 else {
737 ret = -EINVAL;
738 goto out;
739 }
740 ret = get_user(ch, ubuf++);
741 if (ret)
742 goto out;
743 read++;
744 cnt--;
745 }
746
747 /* We either got finished input or we have to wait for another call. */
748 if (isspace(ch)) {
749 parser->buffer[parser->idx] = 0;
750 parser->cont = false;
751 } else {
752 parser->cont = true;
753 parser->buffer[parser->idx++] = ch;
754 }
755
756 *ppos += read;
757 ret = read;
758
759out:
760 return ret;
761}
762
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200763ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
764{
765 int len;
766 int ret;
767
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500768 if (!cnt)
769 return 0;
770
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200771 if (s->len <= s->readpos)
772 return -EBUSY;
773
774 len = s->len - s->readpos;
775 if (cnt > len)
776 cnt = len;
777 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500778 if (ret == cnt)
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200779 return -EFAULT;
780
Steven Rostedt2dc5d122009-03-04 19:10:05 -0500781 cnt -= ret;
782
Steven Rostedte74da522009-03-04 20:31:11 -0500783 s->readpos += cnt;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +0200784 return cnt;
Steven Rostedt214023c2008-05-12 21:20:46 +0200785}
786
Dmitri Vorobievb8b94262009-03-22 19:11:11 +0200787static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200788{
789 int len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200790
791 if (s->len <= s->readpos)
792 return -EBUSY;
793
794 len = s->len - s->readpos;
795 if (cnt > len)
796 cnt = len;
Dan Carpenter5a26c8f2012-04-20 09:31:45 +0300797 memcpy(buf, s->buffer + s->readpos, cnt);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200798
Steven Rostedte74da522009-03-04 20:31:11 -0500799 s->readpos += cnt;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +0200800 return cnt;
801}
802
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400803/*
804 * ftrace_max_lock is used to protect the swapping of buffers
805 * when taking a max snapshot. The buffers themselves are
806 * protected by per_cpu spinlocks. But the action of the swap
807 * needs its own lock.
808 *
Thomas Gleixner445c8952009-12-02 19:49:50 +0100809 * This is defined as a arch_spinlock_t in order to help
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400810 * with performance when lockdep debugging is enabled.
811 *
812 * It is also used in other places outside the update_max_tr
813 * so it needs to be defined outside of the
814 * CONFIG_TRACER_MAX_TRACE.
815 */
Thomas Gleixner445c8952009-12-02 19:49:50 +0100816static arch_spinlock_t ftrace_max_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +0100817 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400818
Tim Bird0e950172010-02-25 15:36:43 -0800819unsigned long __read_mostly tracing_thresh;
820
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400821#ifdef CONFIG_TRACER_MAX_TRACE
822unsigned long __read_mostly tracing_max_latency;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400823
824/*
825 * Copy the new maximum trace into the separate maximum-trace
826 * structure. (this way the maximum trace is permanently saved,
827 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
828 */
829static void
830__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
831{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500832 struct trace_buffer *trace_buf = &tr->trace_buffer;
833 struct trace_buffer *max_buf = &tr->max_buffer;
834 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
835 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400836
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500837 max_buf->cpu = cpu;
838 max_buf->time_start = data->preempt_timestamp;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400839
Steven Rostedt8248ac02009-09-02 12:27:41 -0400840 max_data->saved_latency = tracing_max_latency;
841 max_data->critical_start = data->critical_start;
842 max_data->critical_end = data->critical_end;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400843
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -0300844 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
Steven Rostedt8248ac02009-09-02 12:27:41 -0400845 max_data->pid = tsk->pid;
846 max_data->uid = task_uid(tsk);
847 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
848 max_data->policy = tsk->policy;
849 max_data->rt_priority = tsk->rt_priority;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400850
851 /* record this tasks comm */
852 tracing_record_cmdline(tsk);
853}
854
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200855/**
856 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
857 * @tr: tracer
858 * @tsk: the task with the latency
859 * @cpu: The cpu that initiated the trace.
860 *
861 * Flip the buffers between the @tr and the max_tr and record information
862 * about which task was the cause of this latency.
863 */
Ingo Molnare309b412008-05-12 21:20:51 +0200864void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200865update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
866{
Steven Rostedt (Red Hat)2721e722013-03-12 11:32:32 -0400867 struct ring_buffer *buf;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200868
Steven Rostedt2b6080f2012-05-11 13:29:49 -0400869 if (tr->stop_count)
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400870 return;
871
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200872 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt34600f02013-01-22 13:35:11 -0500873
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -0500874 if (!tr->allocated_snapshot) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +0900875 /* Only the nop tracer should hit this when disabling */
Steven Rostedt2b6080f2012-05-11 13:29:49 -0400876 WARN_ON_ONCE(tr->current_trace != &nop_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900877 return;
Hiraku Toyookadebdd572012-12-26 11:53:00 +0900878 }
Steven Rostedt34600f02013-01-22 13:35:11 -0500879
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100880 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400881
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500882 buf = tr->trace_buffer.buffer;
883 tr->trace_buffer.buffer = tr->max_buffer.buffer;
884 tr->max_buffer.buffer = buf;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400885
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200886 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100887 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200888}
889
890/**
891 * update_max_tr_single - only copy one trace over, and reset the rest
892 * @tr - tracer
893 * @tsk - task with the latency
894 * @cpu - the cpu of the buffer to copy.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200895 *
896 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200897 */
Ingo Molnare309b412008-05-12 21:20:51 +0200898void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200899update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
900{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400901 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200902
Steven Rostedt2b6080f2012-05-11 13:29:49 -0400903 if (tr->stop_count)
Steven Rostedtb8de7bd12009-08-31 22:32:27 -0400904 return;
905
Steven Rostedt4c11d7a2008-05-12 21:20:43 +0200906 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -0500907 if (WARN_ON_ONCE(!tr->allocated_snapshot))
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900908 return;
KOSAKI Motohiroef710e12010-07-01 14:34:35 +0900909
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100910 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200911
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500912 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400913
Steven Rostedte8165db2009-09-03 19:13:05 -0400914 if (ret == -EBUSY) {
915 /*
916 * We failed to swap the buffer due to a commit taking
917 * place on this CPU. We fail to record, but we reset
918 * the max trace buffer (no one writes directly to it)
919 * and flag that it failed.
920 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500921 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
Steven Rostedte8165db2009-09-03 19:13:05 -0400922 "Failed to swap buffers due to commit in progress\n");
923 }
924
Steven Rostedte8165db2009-09-03 19:13:05 -0400925 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200926
927 __update_max_tr(tr, tsk, cpu);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +0100928 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200929}
Steven Rostedt5d4a9db2009-08-27 16:52:21 -0400930#endif /* CONFIG_TRACER_MAX_TRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200931
Steven Rostedt0d5c6e12012-11-01 20:54:21 -0400932static void default_wait_pipe(struct trace_iterator *iter)
933{
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -0500934 /* Iterators are static, they should be filled or empty */
935 if (trace_buffer_iter(iter, iter->cpu_file))
936 return;
Steven Rostedt0d5c6e12012-11-01 20:54:21 -0400937
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -0500938 ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt0d5c6e12012-11-01 20:54:21 -0400939}
940
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -0500941#ifdef CONFIG_FTRACE_STARTUP_TEST
942static int run_tracer_selftest(struct tracer *type)
943{
944 struct trace_array *tr = &global_trace;
945 struct tracer *saved_tracer = tr->current_trace;
946 int ret;
947
948 if (!type->selftest || tracing_selftest_disabled)
949 return 0;
950
951 /*
952 * Run a selftest on this tracer.
953 * Here we reset the trace buffer, and set the current
954 * tracer to be this tracer. The tracer can then run some
955 * internal tracing to verify that everything is in order.
956 * If we fail, we do not register this tracer.
957 */
958 tracing_reset_online_cpus(&tr->trace_buffer);
959
960 tr->current_trace = type;
961
962#ifdef CONFIG_TRACER_MAX_TRACE
963 if (type->use_max_tr) {
964 /* If we expanded the buffers, make sure the max is expanded too */
965 if (ring_buffer_expanded)
966 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
967 RING_BUFFER_ALL_CPUS);
968 tr->allocated_snapshot = true;
969 }
970#endif
971
972 /* the test is responsible for initializing and enabling */
973 pr_info("Testing tracer %s: ", type->name);
974 ret = type->selftest(type, tr);
975 /* the test is responsible for resetting too */
976 tr->current_trace = saved_tracer;
977 if (ret) {
978 printk(KERN_CONT "FAILED!\n");
979 /* Add the warning after printing 'FAILED' */
980 WARN_ON(1);
981 return -1;
982 }
983 /* Only reset on passing, to avoid touching corrupted buffers */
984 tracing_reset_online_cpus(&tr->trace_buffer);
985
986#ifdef CONFIG_TRACER_MAX_TRACE
987 if (type->use_max_tr) {
988 tr->allocated_snapshot = false;
989
990 /* Shrink the max buffer again */
991 if (ring_buffer_expanded)
992 ring_buffer_resize(tr->max_buffer.buffer, 1,
993 RING_BUFFER_ALL_CPUS);
994 }
995#endif
996
997 printk(KERN_CONT "PASSED\n");
998 return 0;
999}
1000#else
1001static inline int run_tracer_selftest(struct tracer *type)
1002{
1003 return 0;
1004}
1005#endif /* CONFIG_FTRACE_STARTUP_TEST */
1006
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001007/**
1008 * register_tracer - register a tracer with the ftrace system.
1009 * @type - the plugin for the tracer
1010 *
1011 * Register a new plugin tracer.
1012 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001013int register_tracer(struct tracer *type)
1014{
1015 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001016 int ret = 0;
1017
1018 if (!type->name) {
1019 pr_info("Tracer must have a name\n");
1020 return -1;
1021 }
1022
Dan Carpenter24a461d2010-07-10 12:06:44 +02001023 if (strlen(type->name) >= MAX_TRACER_SIZE) {
Li Zefanee6c2c12009-09-18 14:06:47 +08001024 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1025 return -1;
1026 }
1027
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001028 mutex_lock(&trace_types_lock);
Ingo Molnar86fa2f62008-11-19 10:00:15 +01001029
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +01001030 tracing_selftest_running = true;
1031
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001032 for (t = trace_types; t; t = t->next) {
1033 if (strcmp(type->name, t->name) == 0) {
1034 /* already found */
Li Zefanee6c2c12009-09-18 14:06:47 +08001035 pr_info("Tracer %s already registered\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001036 type->name);
1037 ret = -1;
1038 goto out;
1039 }
1040 }
1041
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01001042 if (!type->set_flag)
1043 type->set_flag = &dummy_set_flag;
1044 if (!type->flags)
1045 type->flags = &dummy_tracer_flags;
1046 else
1047 if (!type->flags->opts)
1048 type->flags->opts = dummy_tracer_opt;
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01001049 if (!type->wait_pipe)
1050 type->wait_pipe = default_wait_pipe;
1051
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001052 ret = run_tracer_selftest(type);
1053 if (ret < 0)
1054 goto out;
Steven Rostedt60a11772008-05-12 21:20:44 +02001055
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001056 type->next = trace_types;
1057 trace_types = type;
Steven Rostedt60a11772008-05-12 21:20:44 +02001058
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001059 out:
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +01001060 tracing_selftest_running = false;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001061 mutex_unlock(&trace_types_lock);
1062
Steven Rostedtdac74942009-02-05 01:13:38 -05001063 if (ret || !default_bootup_tracer)
1064 goto out_unlock;
Steven Rostedtb2821ae2009-02-02 21:38:32 -05001065
Li Zefanee6c2c12009-09-18 14:06:47 +08001066 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
Steven Rostedtdac74942009-02-05 01:13:38 -05001067 goto out_unlock;
1068
1069 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1070 /* Do we want this tracer to start on bootup? */
1071 tracing_set_tracer(type->name);
1072 default_bootup_tracer = NULL;
1073 /* disable other selftests, since this will break it. */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05001074 tracing_selftest_disabled = true;
Steven Rostedtdac74942009-02-05 01:13:38 -05001075#ifdef CONFIG_FTRACE_STARTUP_TEST
1076 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1077 type->name);
1078#endif
1079
1080 out_unlock:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001081 return ret;
1082}
1083
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001084void tracing_reset(struct trace_buffer *buf, int cpu)
Steven Rostedtf6339032009-09-04 12:35:16 -04001085{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001086 struct ring_buffer *buffer = buf->buffer;
Steven Rostedtf6339032009-09-04 12:35:16 -04001087
Hiraku Toyookaa5416412012-12-19 16:02:34 +09001088 if (!buffer)
1089 return;
1090
Steven Rostedtf6339032009-09-04 12:35:16 -04001091 ring_buffer_record_disable(buffer);
1092
1093 /* Make sure all commits have finished */
1094 synchronize_sched();
Steven Rostedt68179682012-05-08 20:57:53 -04001095 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedtf6339032009-09-04 12:35:16 -04001096
1097 ring_buffer_record_enable(buffer);
1098}
1099
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001100void tracing_reset_online_cpus(struct trace_buffer *buf)
Pekka J Enberg213cc062008-12-19 12:08:39 +02001101{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001102 struct ring_buffer *buffer = buf->buffer;
Pekka J Enberg213cc062008-12-19 12:08:39 +02001103 int cpu;
1104
Hiraku Toyookaa5416412012-12-19 16:02:34 +09001105 if (!buffer)
1106 return;
1107
Steven Rostedt621968c2009-09-04 12:02:35 -04001108 ring_buffer_record_disable(buffer);
1109
1110 /* Make sure all commits have finished */
1111 synchronize_sched();
1112
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001113 buf->time_start = ftrace_now(buf->cpu);
Pekka J Enberg213cc062008-12-19 12:08:39 +02001114
1115 for_each_online_cpu(cpu)
Steven Rostedt68179682012-05-08 20:57:53 -04001116 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedt621968c2009-09-04 12:02:35 -04001117
1118 ring_buffer_record_enable(buffer);
Pekka J Enberg213cc062008-12-19 12:08:39 +02001119}
1120
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001121void tracing_reset_current(int cpu)
1122{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001123 tracing_reset(&global_trace.trace_buffer, cpu);
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001124}
1125
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001126void tracing_reset_all_online_cpus(void)
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001127{
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001128 struct trace_array *tr;
1129
1130 mutex_lock(&trace_types_lock);
1131 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001132 tracing_reset_online_cpus(&tr->trace_buffer);
1133#ifdef CONFIG_TRACER_MAX_TRACE
1134 tracing_reset_online_cpus(&tr->max_buffer);
1135#endif
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001136 }
1137 mutex_unlock(&trace_types_lock);
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001138}
1139
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001140#define SAVED_CMDLINES 128
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001141#define NO_CMDLINE_MAP UINT_MAX
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001142static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1143static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1144static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1145static int cmdline_idx;
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001146static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt25b0b442008-05-12 21:21:00 +02001147
Steven Rostedt25b0b442008-05-12 21:21:00 +02001148/* temporary disable recording */
Hannes Eder4fd27352009-02-10 19:44:12 +01001149static atomic_t trace_record_cmdline_disabled __read_mostly;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001150
1151static void trace_init_cmdlines(void)
1152{
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001153 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1154 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001155 cmdline_idx = 0;
1156}
1157
Carsten Emdeb5130b12009-09-13 01:43:07 +02001158int is_tracing_stopped(void)
1159{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001160 return global_trace.stop_count;
Carsten Emdeb5130b12009-09-13 01:43:07 +02001161}
1162
Steven Rostedt0f048702008-11-05 16:05:44 -05001163/**
Steven Rostedt69bb54e2008-11-21 12:59:38 -05001164 * ftrace_off_permanent - disable all ftrace code permanently
1165 *
1166 * This should only be called when a serious anomally has
1167 * been detected. This will turn off the function tracing,
1168 * ring buffers, and other tracing utilites. It takes no
1169 * locks and can be called from any context.
1170 */
1171void ftrace_off_permanent(void)
1172{
1173 tracing_disabled = 1;
1174 ftrace_stop();
1175 tracing_off_permanent();
1176}
1177
1178/**
Steven Rostedt0f048702008-11-05 16:05:44 -05001179 * tracing_start - quick start of the tracer
1180 *
1181 * If tracing is enabled but was stopped by tracing_stop,
1182 * this will start the tracer back up.
1183 */
1184void tracing_start(void)
1185{
1186 struct ring_buffer *buffer;
1187 unsigned long flags;
1188
1189 if (tracing_disabled)
1190 return;
1191
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001192 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1193 if (--global_trace.stop_count) {
1194 if (global_trace.stop_count < 0) {
Steven Rostedtb06a8302009-01-22 14:26:15 -05001195 /* Someone screwed up their debugging */
1196 WARN_ON_ONCE(1);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001197 global_trace.stop_count = 0;
Steven Rostedtb06a8302009-01-22 14:26:15 -05001198 }
Steven Rostedt0f048702008-11-05 16:05:44 -05001199 goto out;
1200 }
1201
Steven Rostedta2f80712010-03-12 19:56:00 -05001202 /* Prevent the buffers from switching */
1203 arch_spin_lock(&ftrace_max_lock);
Steven Rostedt0f048702008-11-05 16:05:44 -05001204
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001205 buffer = global_trace.trace_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001206 if (buffer)
1207 ring_buffer_record_enable(buffer);
1208
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001209#ifdef CONFIG_TRACER_MAX_TRACE
1210 buffer = global_trace.max_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001211 if (buffer)
1212 ring_buffer_record_enable(buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001213#endif
Steven Rostedt0f048702008-11-05 16:05:44 -05001214
Steven Rostedta2f80712010-03-12 19:56:00 -05001215 arch_spin_unlock(&ftrace_max_lock);
1216
Steven Rostedt0f048702008-11-05 16:05:44 -05001217 ftrace_start();
1218 out:
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001219 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1220}
1221
1222static void tracing_start_tr(struct trace_array *tr)
1223{
1224 struct ring_buffer *buffer;
1225 unsigned long flags;
1226
1227 if (tracing_disabled)
1228 return;
1229
1230 /* If global, we need to also start the max tracer */
1231 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1232 return tracing_start();
1233
1234 raw_spin_lock_irqsave(&tr->start_lock, flags);
1235
1236 if (--tr->stop_count) {
1237 if (tr->stop_count < 0) {
1238 /* Someone screwed up their debugging */
1239 WARN_ON_ONCE(1);
1240 tr->stop_count = 0;
1241 }
1242 goto out;
1243 }
1244
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001245 buffer = tr->trace_buffer.buffer;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001246 if (buffer)
1247 ring_buffer_record_enable(buffer);
1248
1249 out:
1250 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001251}
1252
1253/**
1254 * tracing_stop - quick stop of the tracer
1255 *
1256 * Light weight way to stop tracing. Use in conjunction with
1257 * tracing_start.
1258 */
1259void tracing_stop(void)
1260{
1261 struct ring_buffer *buffer;
1262 unsigned long flags;
1263
1264 ftrace_stop();
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001265 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1266 if (global_trace.stop_count++)
Steven Rostedt0f048702008-11-05 16:05:44 -05001267 goto out;
1268
Steven Rostedta2f80712010-03-12 19:56:00 -05001269 /* Prevent the buffers from switching */
1270 arch_spin_lock(&ftrace_max_lock);
1271
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001272 buffer = global_trace.trace_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001273 if (buffer)
1274 ring_buffer_record_disable(buffer);
1275
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001276#ifdef CONFIG_TRACER_MAX_TRACE
1277 buffer = global_trace.max_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001278 if (buffer)
1279 ring_buffer_record_disable(buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001280#endif
Steven Rostedt0f048702008-11-05 16:05:44 -05001281
Steven Rostedta2f80712010-03-12 19:56:00 -05001282 arch_spin_unlock(&ftrace_max_lock);
1283
Steven Rostedt0f048702008-11-05 16:05:44 -05001284 out:
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001285 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1286}
1287
1288static void tracing_stop_tr(struct trace_array *tr)
1289{
1290 struct ring_buffer *buffer;
1291 unsigned long flags;
1292
1293 /* If global, we need to also stop the max tracer */
1294 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1295 return tracing_stop();
1296
1297 raw_spin_lock_irqsave(&tr->start_lock, flags);
1298 if (tr->stop_count++)
1299 goto out;
1300
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001301 buffer = tr->trace_buffer.buffer;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001302 if (buffer)
1303 ring_buffer_record_disable(buffer);
1304
1305 out:
1306 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001307}
1308
Ingo Molnare309b412008-05-12 21:20:51 +02001309void trace_stop_cmdline_recording(void);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001310
Ingo Molnare309b412008-05-12 21:20:51 +02001311static void trace_save_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001312{
Carsten Emdea635cf02009-03-18 09:00:41 +01001313 unsigned pid, idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001314
1315 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1316 return;
1317
1318 /*
1319 * It's not the end of the world if we don't get
1320 * the lock, but we also don't want to spin
1321 * nor do we want to disable interrupts,
1322 * so if we miss here, then better luck next time.
1323 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001324 if (!arch_spin_trylock(&trace_cmdline_lock))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001325 return;
1326
1327 idx = map_pid_to_cmdline[tsk->pid];
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001328 if (idx == NO_CMDLINE_MAP) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001329 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1330
Carsten Emdea635cf02009-03-18 09:00:41 +01001331 /*
1332 * Check whether the cmdline buffer at idx has a pid
1333 * mapped. We are going to overwrite that entry so we
1334 * need to clear the map_pid_to_cmdline. Otherwise we
1335 * would read the new comm for the old pid.
1336 */
1337 pid = map_cmdline_to_pid[idx];
1338 if (pid != NO_CMDLINE_MAP)
1339 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001340
Carsten Emdea635cf02009-03-18 09:00:41 +01001341 map_cmdline_to_pid[idx] = tsk->pid;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001342 map_pid_to_cmdline[tsk->pid] = idx;
1343
1344 cmdline_idx = idx;
1345 }
1346
1347 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1348
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001349 arch_spin_unlock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001350}
1351
Steven Rostedt4ca53082009-03-16 19:20:15 -04001352void trace_find_cmdline(int pid, char comm[])
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001353{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001354 unsigned map;
1355
Steven Rostedt4ca53082009-03-16 19:20:15 -04001356 if (!pid) {
1357 strcpy(comm, "<idle>");
1358 return;
1359 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001360
Steven Rostedt74bf4072010-01-25 15:11:53 -05001361 if (WARN_ON_ONCE(pid < 0)) {
1362 strcpy(comm, "<XXX>");
1363 return;
1364 }
1365
Steven Rostedt4ca53082009-03-16 19:20:15 -04001366 if (pid > PID_MAX_DEFAULT) {
1367 strcpy(comm, "<...>");
1368 return;
1369 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001370
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001371 preempt_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001372 arch_spin_lock(&trace_cmdline_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001373 map = map_pid_to_cmdline[pid];
Thomas Gleixner50d88752009-03-18 08:58:44 +01001374 if (map != NO_CMDLINE_MAP)
1375 strcpy(comm, saved_cmdlines[map]);
1376 else
1377 strcpy(comm, "<...>");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001378
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001379 arch_spin_unlock(&trace_cmdline_lock);
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001380 preempt_enable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001381}
1382
Ingo Molnare309b412008-05-12 21:20:51 +02001383void tracing_record_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001384{
Steven Rostedt0fb96562012-05-11 14:25:30 -04001385 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001386 return;
1387
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001388 if (!__this_cpu_read(trace_cmdline_save))
1389 return;
1390
1391 __this_cpu_write(trace_cmdline_save, false);
1392
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001393 trace_save_cmdline(tsk);
1394}
1395
Pekka Paalanen45dcd8b2008-09-16 21:56:41 +03001396void
Steven Rostedt38697052008-10-01 13:14:09 -04001397tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1398 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001399{
1400 struct task_struct *tsk = current;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001401
Steven Rostedt777e2082008-09-29 23:02:42 -04001402 entry->preempt_count = pc & 0xff;
1403 entry->pid = (tsk) ? tsk->pid : 0;
1404 entry->flags =
Steven Rostedt92444892008-10-24 09:42:59 -04001405#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
Steven Rostedt2e2ca152008-08-01 12:26:40 -04001406 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
Steven Rostedt92444892008-10-24 09:42:59 -04001407#else
1408 TRACE_FLAG_IRQS_NOSUPPORT |
1409#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001410 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1411 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1412 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1413}
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +02001414EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001415
Steven Rostedte77405a2009-09-02 14:17:06 -04001416struct ring_buffer_event *
1417trace_buffer_lock_reserve(struct ring_buffer *buffer,
1418 int type,
1419 unsigned long len,
1420 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001421{
1422 struct ring_buffer_event *event;
1423
Steven Rostedte77405a2009-09-02 14:17:06 -04001424 event = ring_buffer_lock_reserve(buffer, len);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001425 if (event != NULL) {
1426 struct trace_entry *ent = ring_buffer_event_data(event);
1427
1428 tracing_generic_entry_update(ent, flags, pc);
1429 ent->type = type;
1430 }
1431
1432 return event;
1433}
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001434
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001435void
1436__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1437{
1438 __this_cpu_write(trace_cmdline_save, true);
1439 ring_buffer_unlock_commit(buffer, event);
1440}
1441
Steven Rostedte77405a2009-09-02 14:17:06 -04001442static inline void
1443__trace_buffer_unlock_commit(struct ring_buffer *buffer,
1444 struct ring_buffer_event *event,
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001445 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001446{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001447 __buffer_unlock_commit(buffer, event);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001448
Steven Rostedte77405a2009-09-02 14:17:06 -04001449 ftrace_trace_stack(buffer, flags, 6, pc);
1450 ftrace_trace_userstack(buffer, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001451}
1452
Steven Rostedte77405a2009-09-02 14:17:06 -04001453void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1454 struct ring_buffer_event *event,
1455 unsigned long flags, int pc)
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001456{
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001457 __trace_buffer_unlock_commit(buffer, event, flags, pc);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001458}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001459EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001460
Steven Rostedtef5580d2009-02-27 19:38:04 -05001461struct ring_buffer_event *
Steven Rostedtccb469a2012-08-02 10:32:10 -04001462trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1463 struct ftrace_event_file *ftrace_file,
1464 int type, unsigned long len,
1465 unsigned long flags, int pc)
1466{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001467 *current_rb = ftrace_file->tr->trace_buffer.buffer;
Steven Rostedtccb469a2012-08-02 10:32:10 -04001468 return trace_buffer_lock_reserve(*current_rb,
1469 type, len, flags, pc);
1470}
1471EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1472
1473struct ring_buffer_event *
Steven Rostedte77405a2009-09-02 14:17:06 -04001474trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1475 int type, unsigned long len,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001476 unsigned long flags, int pc)
1477{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001478 *current_rb = global_trace.trace_buffer.buffer;
Steven Rostedte77405a2009-09-02 14:17:06 -04001479 return trace_buffer_lock_reserve(*current_rb,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001480 type, len, flags, pc);
1481}
Steven Rostedt94487d62009-05-05 19:22:53 -04001482EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001483
Steven Rostedte77405a2009-09-02 14:17:06 -04001484void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1485 struct ring_buffer_event *event,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001486 unsigned long flags, int pc)
1487{
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001488 __trace_buffer_unlock_commit(buffer, event, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001489}
Steven Rostedt94487d62009-05-05 19:22:53 -04001490EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001491
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001492void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1493 struct ring_buffer_event *event,
1494 unsigned long flags, int pc,
1495 struct pt_regs *regs)
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001496{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001497 __buffer_unlock_commit(buffer, event);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001498
1499 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1500 ftrace_trace_userstack(buffer, flags, pc);
1501}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001502EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001503
Steven Rostedte77405a2009-09-02 14:17:06 -04001504void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1505 struct ring_buffer_event *event)
Steven Rostedt77d9f462009-04-02 01:16:59 -04001506{
Steven Rostedte77405a2009-09-02 14:17:06 -04001507 ring_buffer_discard_commit(buffer, event);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001508}
Steven Rostedt12acd472009-04-17 16:01:56 -04001509EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001510
Ingo Molnare309b412008-05-12 21:20:51 +02001511void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001512trace_function(struct trace_array *tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001513 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1514 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001515{
Tom Zanussie1112b42009-03-31 00:48:49 -05001516 struct ftrace_event_call *call = &event_function;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001517 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001518 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001519 struct ftrace_entry *entry;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001520
Steven Rostedtd7690412008-10-01 00:29:53 -04001521 /* If we are reading the ring buffer, don't trace */
Rusty Russelldd17c8f2009-10-29 22:34:15 +09001522 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
Steven Rostedtd7690412008-10-01 00:29:53 -04001523 return;
1524
Steven Rostedte77405a2009-09-02 14:17:06 -04001525 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001526 flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001527 if (!event)
1528 return;
1529 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001530 entry->ip = ip;
1531 entry->parent_ip = parent_ip;
Tom Zanussie1112b42009-03-31 00:48:49 -05001532
Steven Rostedte77405a2009-09-02 14:17:06 -04001533 if (!filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001534 __buffer_unlock_commit(buffer, event);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001535}
1536
Ingo Molnare309b412008-05-12 21:20:51 +02001537void
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001538ftrace(struct trace_array *tr, struct trace_array_cpu *data,
Steven Rostedt38697052008-10-01 13:14:09 -04001539 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1540 int pc)
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001541{
1542 if (likely(!atomic_read(&data->disabled)))
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001543 trace_function(tr, ip, parent_ip, flags, pc);
Ingo Molnar2e0f5762008-05-12 21:20:49 +02001544}
1545
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001546#ifdef CONFIG_STACKTRACE
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001547
1548#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1549struct ftrace_stack {
1550 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1551};
1552
1553static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1554static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1555
Steven Rostedte77405a2009-09-02 14:17:06 -04001556static void __ftrace_trace_stack(struct ring_buffer *buffer,
Steven Rostedt53614992009-01-15 19:12:40 -05001557 unsigned long flags,
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001558 int skip, int pc, struct pt_regs *regs)
Ingo Molnar86387f72008-05-12 21:20:51 +02001559{
Tom Zanussie1112b42009-03-31 00:48:49 -05001560 struct ftrace_event_call *call = &event_kernel_stack;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001561 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001562 struct stack_entry *entry;
Ingo Molnar86387f72008-05-12 21:20:51 +02001563 struct stack_trace trace;
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001564 int use_stack;
1565 int size = FTRACE_STACK_ENTRIES;
Ingo Molnar86387f72008-05-12 21:20:51 +02001566
1567 trace.nr_entries = 0;
Ingo Molnar86387f72008-05-12 21:20:51 +02001568 trace.skip = skip;
Ingo Molnar86387f72008-05-12 21:20:51 +02001569
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001570 /*
1571 * Since events can happen in NMIs there's no safe way to
1572 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1573 * or NMI comes in, it will just have to use the default
1574 * FTRACE_STACK_SIZE.
1575 */
1576 preempt_disable_notrace();
1577
Shan Wei82146522012-11-19 13:21:01 +08001578 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001579 /*
1580 * We don't need any atomic variables, just a barrier.
1581 * If an interrupt comes in, we don't care, because it would
1582 * have exited and put the counter back to what we want.
1583 * We just need a barrier to keep gcc from moving things
1584 * around.
1585 */
1586 barrier();
1587 if (use_stack == 1) {
1588 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1589 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1590
1591 if (regs)
1592 save_stack_trace_regs(regs, &trace);
1593 else
1594 save_stack_trace(&trace);
1595
1596 if (trace.nr_entries > size)
1597 size = trace.nr_entries;
1598 } else
1599 /* From now on, use_stack is a boolean */
1600 use_stack = 0;
1601
1602 size *= sizeof(unsigned long);
1603
1604 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1605 sizeof(*entry) + size, flags, pc);
1606 if (!event)
1607 goto out;
1608 entry = ring_buffer_event_data(event);
1609
1610 memset(&entry->caller, 0, size);
1611
1612 if (use_stack)
1613 memcpy(&entry->caller, trace.entries,
1614 trace.nr_entries * sizeof(unsigned long));
1615 else {
1616 trace.max_entries = FTRACE_STACK_ENTRIES;
1617 trace.entries = entry->caller;
1618 if (regs)
1619 save_stack_trace_regs(regs, &trace);
1620 else
1621 save_stack_trace(&trace);
1622 }
1623
1624 entry->size = trace.nr_entries;
1625
Steven Rostedte77405a2009-09-02 14:17:06 -04001626 if (!filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001627 __buffer_unlock_commit(buffer, event);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001628
1629 out:
1630 /* Again, don't let gcc optimize things here */
1631 barrier();
Shan Wei82146522012-11-19 13:21:01 +08001632 __this_cpu_dec(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001633 preempt_enable_notrace();
1634
Ingo Molnarf0a920d2008-05-12 21:20:47 +02001635}
1636
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001637void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1638 int skip, int pc, struct pt_regs *regs)
1639{
1640 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1641 return;
1642
1643 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1644}
1645
Steven Rostedte77405a2009-09-02 14:17:06 -04001646void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1647 int skip, int pc)
Steven Rostedt53614992009-01-15 19:12:40 -05001648{
1649 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1650 return;
1651
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001652 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
Steven Rostedt53614992009-01-15 19:12:40 -05001653}
1654
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001655void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1656 int pc)
Steven Rostedt38697052008-10-01 13:14:09 -04001657{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001658 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
Steven Rostedt38697052008-10-01 13:14:09 -04001659}
1660
Steven Rostedt03889382009-12-11 09:48:22 -05001661/**
1662 * trace_dump_stack - record a stack back trace in the trace buffer
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001663 * @skip: Number of functions to skip (helper handlers)
Steven Rostedt03889382009-12-11 09:48:22 -05001664 */
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001665void trace_dump_stack(int skip)
Steven Rostedt03889382009-12-11 09:48:22 -05001666{
1667 unsigned long flags;
1668
1669 if (tracing_disabled || tracing_selftest_running)
Steven Rostedte36c5452009-12-14 15:58:33 -05001670 return;
Steven Rostedt03889382009-12-11 09:48:22 -05001671
1672 local_save_flags(flags);
1673
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001674 /*
1675 * Skip 3 more, seems to get us at the caller of
1676 * this function.
1677 */
1678 skip += 3;
1679 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1680 flags, skip, preempt_count(), NULL);
Steven Rostedt03889382009-12-11 09:48:22 -05001681}
1682
Steven Rostedt91e86e52010-11-10 12:56:12 +01001683static DEFINE_PER_CPU(int, user_stack_count);
1684
Steven Rostedte77405a2009-09-02 14:17:06 -04001685void
1686ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
Török Edwin02b67512008-11-22 13:28:47 +02001687{
Tom Zanussie1112b42009-03-31 00:48:49 -05001688 struct ftrace_event_call *call = &event_user_stack;
Török Edwin8d7c6a92008-11-23 12:39:06 +02001689 struct ring_buffer_event *event;
Török Edwin02b67512008-11-22 13:28:47 +02001690 struct userstack_entry *entry;
1691 struct stack_trace trace;
Török Edwin02b67512008-11-22 13:28:47 +02001692
1693 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1694 return;
1695
Steven Rostedtb6345872010-03-12 20:03:30 -05001696 /*
1697 * NMIs can not handle page faults, even with fix ups.
1698 * The save user stack can (and often does) fault.
1699 */
1700 if (unlikely(in_nmi()))
1701 return;
1702
Steven Rostedt91e86e52010-11-10 12:56:12 +01001703 /*
1704 * prevent recursion, since the user stack tracing may
1705 * trigger other kernel events.
1706 */
1707 preempt_disable();
1708 if (__this_cpu_read(user_stack_count))
1709 goto out;
1710
1711 __this_cpu_inc(user_stack_count);
1712
Steven Rostedte77405a2009-09-02 14:17:06 -04001713 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001714 sizeof(*entry), flags, pc);
Török Edwin02b67512008-11-22 13:28:47 +02001715 if (!event)
Li Zefan1dbd1952010-12-09 15:47:56 +08001716 goto out_drop_count;
Török Edwin02b67512008-11-22 13:28:47 +02001717 entry = ring_buffer_event_data(event);
Török Edwin02b67512008-11-22 13:28:47 +02001718
Steven Rostedt48659d32009-09-11 11:36:23 -04001719 entry->tgid = current->tgid;
Török Edwin02b67512008-11-22 13:28:47 +02001720 memset(&entry->caller, 0, sizeof(entry->caller));
1721
1722 trace.nr_entries = 0;
1723 trace.max_entries = FTRACE_STACK_ENTRIES;
1724 trace.skip = 0;
1725 trace.entries = entry->caller;
1726
1727 save_stack_trace_user(&trace);
Steven Rostedte77405a2009-09-02 14:17:06 -04001728 if (!filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001729 __buffer_unlock_commit(buffer, event);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001730
Li Zefan1dbd1952010-12-09 15:47:56 +08001731 out_drop_count:
Steven Rostedt91e86e52010-11-10 12:56:12 +01001732 __this_cpu_dec(user_stack_count);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001733 out:
1734 preempt_enable();
Török Edwin02b67512008-11-22 13:28:47 +02001735}
1736
Hannes Eder4fd27352009-02-10 19:44:12 +01001737#ifdef UNUSED
1738static void __trace_userstack(struct trace_array *tr, unsigned long flags)
Török Edwin02b67512008-11-22 13:28:47 +02001739{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001740 ftrace_trace_userstack(tr, flags, preempt_count());
Török Edwin02b67512008-11-22 13:28:47 +02001741}
Hannes Eder4fd27352009-02-10 19:44:12 +01001742#endif /* UNUSED */
Török Edwin02b67512008-11-22 13:28:47 +02001743
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001744#endif /* CONFIG_STACKTRACE */
1745
Steven Rostedt07d777f2011-09-22 14:01:55 -04001746/* created for use with alloc_percpu */
1747struct trace_buffer_struct {
1748 char buffer[TRACE_BUF_SIZE];
1749};
1750
1751static struct trace_buffer_struct *trace_percpu_buffer;
1752static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1753static struct trace_buffer_struct *trace_percpu_irq_buffer;
1754static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1755
1756/*
1757 * The buffer used is dependent on the context. There is a per cpu
1758 * buffer for normal context, softirq contex, hard irq context and
1759 * for NMI context. Thise allows for lockless recording.
1760 *
1761 * Note, if the buffers failed to be allocated, then this returns NULL
1762 */
1763static char *get_trace_buf(void)
1764{
1765 struct trace_buffer_struct *percpu_buffer;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001766
1767 /*
1768 * If we have allocated per cpu buffers, then we do not
1769 * need to do any locking.
1770 */
1771 if (in_nmi())
1772 percpu_buffer = trace_percpu_nmi_buffer;
1773 else if (in_irq())
1774 percpu_buffer = trace_percpu_irq_buffer;
1775 else if (in_softirq())
1776 percpu_buffer = trace_percpu_sirq_buffer;
1777 else
1778 percpu_buffer = trace_percpu_buffer;
1779
1780 if (!percpu_buffer)
1781 return NULL;
1782
Shan Weid8a03492012-11-13 09:53:04 +08001783 return this_cpu_ptr(&percpu_buffer->buffer[0]);
Steven Rostedt07d777f2011-09-22 14:01:55 -04001784}
1785
1786static int alloc_percpu_trace_buffer(void)
1787{
1788 struct trace_buffer_struct *buffers;
1789 struct trace_buffer_struct *sirq_buffers;
1790 struct trace_buffer_struct *irq_buffers;
1791 struct trace_buffer_struct *nmi_buffers;
1792
1793 buffers = alloc_percpu(struct trace_buffer_struct);
1794 if (!buffers)
1795 goto err_warn;
1796
1797 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1798 if (!sirq_buffers)
1799 goto err_sirq;
1800
1801 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1802 if (!irq_buffers)
1803 goto err_irq;
1804
1805 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1806 if (!nmi_buffers)
1807 goto err_nmi;
1808
1809 trace_percpu_buffer = buffers;
1810 trace_percpu_sirq_buffer = sirq_buffers;
1811 trace_percpu_irq_buffer = irq_buffers;
1812 trace_percpu_nmi_buffer = nmi_buffers;
1813
1814 return 0;
1815
1816 err_nmi:
1817 free_percpu(irq_buffers);
1818 err_irq:
1819 free_percpu(sirq_buffers);
1820 err_sirq:
1821 free_percpu(buffers);
1822 err_warn:
1823 WARN(1, "Could not allocate percpu trace_printk buffer");
1824 return -ENOMEM;
1825}
1826
Steven Rostedt81698832012-10-11 10:15:05 -04001827static int buffers_allocated;
1828
Steven Rostedt07d777f2011-09-22 14:01:55 -04001829void trace_printk_init_buffers(void)
1830{
Steven Rostedt07d777f2011-09-22 14:01:55 -04001831 if (buffers_allocated)
1832 return;
1833
1834 if (alloc_percpu_trace_buffer())
1835 return;
1836
1837 pr_info("ftrace: Allocated trace_printk buffers\n");
1838
Steven Rostedtb382ede62012-10-10 21:44:34 -04001839 /* Expand the buffers to set size */
1840 tracing_update_buffers();
1841
Steven Rostedt07d777f2011-09-22 14:01:55 -04001842 buffers_allocated = 1;
Steven Rostedt81698832012-10-11 10:15:05 -04001843
1844 /*
1845 * trace_printk_init_buffers() can be called by modules.
1846 * If that happens, then we need to start cmdline recording
1847 * directly here. If the global_trace.buffer is already
1848 * allocated here, then this was called by module code.
1849 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001850 if (global_trace.trace_buffer.buffer)
Steven Rostedt81698832012-10-11 10:15:05 -04001851 tracing_start_cmdline_record();
1852}
1853
1854void trace_printk_start_comm(void)
1855{
1856 /* Start tracing comms if trace printk is set */
1857 if (!buffers_allocated)
1858 return;
1859 tracing_start_cmdline_record();
1860}
1861
1862static void trace_printk_start_stop_comm(int enabled)
1863{
1864 if (!buffers_allocated)
1865 return;
1866
1867 if (enabled)
1868 tracing_start_cmdline_record();
1869 else
1870 tracing_stop_cmdline_record();
Steven Rostedt07d777f2011-09-22 14:01:55 -04001871}
1872
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001873/**
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001874 * trace_vbprintk - write binary msg to tracing buffer
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001875 *
1876 */
Steven Rostedt40ce74f2009-03-19 14:03:53 -04001877int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001878{
Tom Zanussie1112b42009-03-31 00:48:49 -05001879 struct ftrace_event_call *call = &event_bprint;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001880 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04001881 struct ring_buffer *buffer;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001882 struct trace_array *tr = &global_trace;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001883 struct bprint_entry *entry;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001884 unsigned long flags;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001885 char *tbuffer;
1886 int len = 0, size, pc;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001887
1888 if (unlikely(tracing_selftest_running || tracing_disabled))
1889 return 0;
1890
1891 /* Don't pollute graph traces with trace_vprintk internals */
1892 pause_graph_tracing();
1893
1894 pc = preempt_count();
Steven Rostedt5168ae52010-06-03 09:36:50 -04001895 preempt_disable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001896
Steven Rostedt07d777f2011-09-22 14:01:55 -04001897 tbuffer = get_trace_buf();
1898 if (!tbuffer) {
1899 len = 0;
1900 goto out;
1901 }
1902
1903 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1904
1905 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001906 goto out;
1907
Steven Rostedt07d777f2011-09-22 14:01:55 -04001908 local_save_flags(flags);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001909 size = sizeof(*entry) + sizeof(u32) * len;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001910 buffer = tr->trace_buffer.buffer;
Steven Rostedte77405a2009-09-02 14:17:06 -04001911 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1912 flags, pc);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001913 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04001914 goto out;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001915 entry = ring_buffer_event_data(event);
1916 entry->ip = ip;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001917 entry->fmt = fmt;
1918
Steven Rostedt07d777f2011-09-22 14:01:55 -04001919 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
Steven Rostedtd9313692010-01-06 17:27:11 -05001920 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001921 __buffer_unlock_commit(buffer, event);
Steven Rostedtd9313692010-01-06 17:27:11 -05001922 ftrace_trace_stack(buffer, flags, 6, pc);
1923 }
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001924
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001925out:
Steven Rostedt5168ae52010-06-03 09:36:50 -04001926 preempt_enable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01001927 unpause_graph_tracing();
1928
1929 return len;
1930}
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001931EXPORT_SYMBOL_GPL(trace_vbprintk);
1932
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001933static int
1934__trace_array_vprintk(struct ring_buffer *buffer,
1935 unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001936{
Tom Zanussie1112b42009-03-31 00:48:49 -05001937 struct ftrace_event_call *call = &event_print;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001938 struct ring_buffer_event *event;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001939 int len = 0, size, pc;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001940 struct print_entry *entry;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001941 unsigned long flags;
1942 char *tbuffer;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001943
1944 if (tracing_disabled || tracing_selftest_running)
1945 return 0;
1946
Steven Rostedt07d777f2011-09-22 14:01:55 -04001947 /* Don't pollute graph traces with trace_vprintk internals */
1948 pause_graph_tracing();
1949
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001950 pc = preempt_count();
1951 preempt_disable_notrace();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001952
Steven Rostedt07d777f2011-09-22 14:01:55 -04001953
1954 tbuffer = get_trace_buf();
1955 if (!tbuffer) {
1956 len = 0;
1957 goto out;
1958 }
1959
1960 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1961 if (len > TRACE_BUF_SIZE)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001962 goto out;
1963
Steven Rostedt07d777f2011-09-22 14:01:55 -04001964 local_save_flags(flags);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001965 size = sizeof(*entry) + len + 1;
Steven Rostedte77405a2009-09-02 14:17:06 -04001966 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
Steven Rostedt07d777f2011-09-22 14:01:55 -04001967 flags, pc);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001968 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04001969 goto out;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001970 entry = ring_buffer_event_data(event);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001971 entry->ip = ip;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001972
Steven Rostedt07d777f2011-09-22 14:01:55 -04001973 memcpy(&entry->buf, tbuffer, len);
Carsten Emdec13d2f72009-11-16 20:56:13 +01001974 entry->buf[len] = '\0';
Steven Rostedtd9313692010-01-06 17:27:11 -05001975 if (!filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001976 __buffer_unlock_commit(buffer, event);
Steven Rostedt07d777f2011-09-22 14:01:55 -04001977 ftrace_trace_stack(buffer, flags, 6, pc);
Steven Rostedtd9313692010-01-06 17:27:11 -05001978 }
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001979 out:
1980 preempt_enable_notrace();
Steven Rostedt07d777f2011-09-22 14:01:55 -04001981 unpause_graph_tracing();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01001982
1983 return len;
1984}
Steven Rostedt659372d2009-09-03 19:11:07 -04001985
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001986int trace_array_vprintk(struct trace_array *tr,
1987 unsigned long ip, const char *fmt, va_list args)
1988{
1989 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
1990}
1991
1992int trace_array_printk(struct trace_array *tr,
1993 unsigned long ip, const char *fmt, ...)
1994{
1995 int ret;
1996 va_list ap;
1997
1998 if (!(trace_flags & TRACE_ITER_PRINTK))
1999 return 0;
2000
2001 va_start(ap, fmt);
2002 ret = trace_array_vprintk(tr, ip, fmt, ap);
2003 va_end(ap);
2004 return ret;
2005}
2006
2007int trace_array_printk_buf(struct ring_buffer *buffer,
2008 unsigned long ip, const char *fmt, ...)
2009{
2010 int ret;
2011 va_list ap;
2012
2013 if (!(trace_flags & TRACE_ITER_PRINTK))
2014 return 0;
2015
2016 va_start(ap, fmt);
2017 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2018 va_end(ap);
2019 return ret;
2020}
2021
Steven Rostedt659372d2009-09-03 19:11:07 -04002022int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2023{
Steven Rostedta813a152009-10-09 01:41:35 -04002024 return trace_array_vprintk(&global_trace, ip, fmt, args);
Steven Rostedt659372d2009-09-03 19:11:07 -04002025}
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002026EXPORT_SYMBOL_GPL(trace_vprintk);
2027
Robert Richtere2ac8ef2008-11-12 12:59:32 +01002028static void trace_iterator_increment(struct trace_iterator *iter)
Steven Rostedt5a90f572008-09-03 17:42:51 -04002029{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002030 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2031
Steven Rostedt5a90f572008-09-03 17:42:51 -04002032 iter->idx++;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002033 if (buf_iter)
2034 ring_buffer_read(buf_iter, NULL);
Steven Rostedt5a90f572008-09-03 17:42:51 -04002035}
2036
Ingo Molnare309b412008-05-12 21:20:51 +02002037static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04002038peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2039 unsigned long *lost_events)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002040{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002041 struct ring_buffer_event *event;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002042 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002043
Steven Rostedtd7690412008-10-01 00:29:53 -04002044 if (buf_iter)
2045 event = ring_buffer_iter_peek(buf_iter, ts);
2046 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002047 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
Steven Rostedtbc21b472010-03-31 19:49:26 -04002048 lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04002049
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04002050 if (event) {
2051 iter->ent_size = ring_buffer_event_length(event);
2052 return ring_buffer_event_data(event);
2053 }
2054 iter->ent_size = 0;
2055 return NULL;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002056}
Steven Rostedtd7690412008-10-01 00:29:53 -04002057
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002058static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04002059__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2060 unsigned long *missing_events, u64 *ent_ts)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002061{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002062 struct ring_buffer *buffer = iter->trace_buffer->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002063 struct trace_entry *ent, *next = NULL;
Lai Jiangshanaa274972010-04-05 17:11:05 +08002064 unsigned long lost_events = 0, next_lost = 0;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002065 int cpu_file = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002066 u64 next_ts = 0, ts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002067 int next_cpu = -1;
Steven Rostedt12b5da32012-03-27 10:43:28 -04002068 int next_size = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002069 int cpu;
2070
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002071 /*
2072 * If we are in a per_cpu trace file, don't bother by iterating over
2073 * all cpu and peek directly.
2074 */
Steven Rostedtae3b5092013-01-23 15:22:59 -05002075 if (cpu_file > RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002076 if (ring_buffer_empty_cpu(buffer, cpu_file))
2077 return NULL;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002078 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002079 if (ent_cpu)
2080 *ent_cpu = cpu_file;
2081
2082 return ent;
2083 }
2084
Steven Rostedtab464282008-05-12 21:21:00 +02002085 for_each_tracing_cpu(cpu) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002086
2087 if (ring_buffer_empty_cpu(buffer, cpu))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002088 continue;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002089
Steven Rostedtbc21b472010-03-31 19:49:26 -04002090 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002091
Ingo Molnarcdd31cd2008-05-12 21:20:46 +02002092 /*
2093 * Pick the entry with the smallest timestamp:
2094 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002095 if (ent && (!next || ts < next_ts)) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002096 next = ent;
2097 next_cpu = cpu;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002098 next_ts = ts;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002099 next_lost = lost_events;
Steven Rostedt12b5da32012-03-27 10:43:28 -04002100 next_size = iter->ent_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002101 }
2102 }
2103
Steven Rostedt12b5da32012-03-27 10:43:28 -04002104 iter->ent_size = next_size;
2105
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002106 if (ent_cpu)
2107 *ent_cpu = next_cpu;
2108
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002109 if (ent_ts)
2110 *ent_ts = next_ts;
2111
Steven Rostedtbc21b472010-03-31 19:49:26 -04002112 if (missing_events)
2113 *missing_events = next_lost;
2114
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002115 return next;
2116}
2117
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002118/* Find the next real entry, without updating the iterator itself */
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002119struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2120 int *ent_cpu, u64 *ent_ts)
Steven Rostedtb3806b42008-05-12 21:20:46 +02002121{
Steven Rostedtbc21b472010-03-31 19:49:26 -04002122 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002123}
Ingo Molnar8c523a92008-05-12 21:20:46 +02002124
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002125/* Find the next real entry, and increment the iterator to the next entry */
Jason Wessel955b61e2010-08-05 09:22:23 -05002126void *trace_find_next_entry_inc(struct trace_iterator *iter)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002127{
Steven Rostedtbc21b472010-03-31 19:49:26 -04002128 iter->ent = __find_next_entry(iter, &iter->cpu,
2129 &iter->lost_events, &iter->ts);
Steven Rostedtb3806b42008-05-12 21:20:46 +02002130
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002131 if (iter->ent)
Robert Richtere2ac8ef2008-11-12 12:59:32 +01002132 trace_iterator_increment(iter);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002133
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002134 return iter->ent ? iter : NULL;
Steven Rostedtb3806b42008-05-12 21:20:46 +02002135}
2136
Ingo Molnare309b412008-05-12 21:20:51 +02002137static void trace_consume(struct trace_iterator *iter)
Steven Rostedtb3806b42008-05-12 21:20:46 +02002138{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002139 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
Steven Rostedtbc21b472010-03-31 19:49:26 -04002140 &iter->lost_events);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002141}
2142
Ingo Molnare309b412008-05-12 21:20:51 +02002143static void *s_next(struct seq_file *m, void *v, loff_t *pos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002144{
2145 struct trace_iterator *iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002146 int i = (int)*pos;
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002147 void *ent;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002148
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002149 WARN_ON_ONCE(iter->leftover);
2150
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002151 (*pos)++;
2152
2153 /* can't go backwards */
2154 if (iter->idx > i)
2155 return NULL;
2156
2157 if (iter->idx < 0)
Jason Wessel955b61e2010-08-05 09:22:23 -05002158 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002159 else
2160 ent = iter;
2161
2162 while (ent && iter->idx < i)
Jason Wessel955b61e2010-08-05 09:22:23 -05002163 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002164
2165 iter->pos = *pos;
2166
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002167 return ent;
2168}
2169
Jason Wessel955b61e2010-08-05 09:22:23 -05002170void tracing_iter_reset(struct trace_iterator *iter, int cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002171{
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002172 struct ring_buffer_event *event;
2173 struct ring_buffer_iter *buf_iter;
2174 unsigned long entries = 0;
2175 u64 ts;
2176
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002177 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002178
Steven Rostedt6d158a82012-06-27 20:46:14 -04002179 buf_iter = trace_buffer_iter(iter, cpu);
2180 if (!buf_iter)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002181 return;
2182
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002183 ring_buffer_iter_reset(buf_iter);
2184
2185 /*
2186 * We could have the case with the max latency tracers
2187 * that a reset never took place on a cpu. This is evident
2188 * by the timestamp being before the start of the buffer.
2189 */
2190 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002191 if (ts >= iter->trace_buffer->time_start)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002192 break;
2193 entries++;
2194 ring_buffer_read(buf_iter, NULL);
2195 }
2196
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002197 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002198}
2199
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002200/*
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002201 * The current tracer is copied to avoid a global locking
2202 * all around.
2203 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002204static void *s_start(struct seq_file *m, loff_t *pos)
2205{
2206 struct trace_iterator *iter = m->private;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002207 struct trace_array *tr = iter->tr;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002208 int cpu_file = iter->cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002209 void *p = NULL;
2210 loff_t l = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002211 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002212
Hiraku Toyooka2fd196e2012-12-26 11:52:52 +09002213 /*
2214 * copy the tracer to avoid using a global lock all around.
2215 * iter->trace is a copy of current_trace, the pointer to the
2216 * name may be used instead of a strcmp(), as iter->trace->name
2217 * will point to the same string as current_trace->name.
2218 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002219 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002220 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2221 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002222 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002223
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002224#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002225 if (iter->snapshot && iter->trace->use_max_tr)
2226 return ERR_PTR(-EBUSY);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002227#endif
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002228
2229 if (!iter->snapshot)
2230 atomic_inc(&trace_record_cmdline_disabled);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002231
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002232 if (*pos != iter->pos) {
2233 iter->ent = NULL;
2234 iter->cpu = 0;
2235 iter->idx = -1;
2236
Steven Rostedtae3b5092013-01-23 15:22:59 -05002237 if (cpu_file == RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002238 for_each_tracing_cpu(cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002239 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002240 } else
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002241 tracing_iter_reset(iter, cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002242
Lai Jiangshanac91d852010-03-02 17:54:50 +08002243 iter->leftover = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002244 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2245 ;
2246
2247 } else {
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002248 /*
2249 * If we overflowed the seq_file before, then we want
2250 * to just reuse the trace_seq buffer again.
2251 */
2252 if (iter->leftover)
2253 p = iter;
2254 else {
2255 l = *pos - 1;
2256 p = s_next(m, p, &l);
2257 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002258 }
2259
Lai Jiangshan4f535962009-05-18 19:35:34 +08002260 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002261 trace_access_lock(cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002262 return p;
2263}
2264
2265static void s_stop(struct seq_file *m, void *p)
2266{
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002267 struct trace_iterator *iter = m->private;
2268
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002269#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002270 if (iter->snapshot && iter->trace->use_max_tr)
2271 return;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002272#endif
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002273
2274 if (!iter->snapshot)
2275 atomic_dec(&trace_record_cmdline_disabled);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002276
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002277 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08002278 trace_event_read_unlock();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002279}
2280
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002281static void
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002282get_total_entries(struct trace_buffer *buf,
2283 unsigned long *total, unsigned long *entries)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002284{
2285 unsigned long count;
2286 int cpu;
2287
2288 *total = 0;
2289 *entries = 0;
2290
2291 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002292 count = ring_buffer_entries_cpu(buf->buffer, cpu);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002293 /*
2294 * If this buffer has skipped entries, then we hold all
2295 * entries for the trace and we need to ignore the
2296 * ones before the time stamp.
2297 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002298 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2299 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002300 /* total is the same as the entries */
2301 *total += count;
2302 } else
2303 *total += count +
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002304 ring_buffer_overrun_cpu(buf->buffer, cpu);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002305 *entries += count;
2306 }
2307}
2308
Ingo Molnare309b412008-05-12 21:20:51 +02002309static void print_lat_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002310{
Michael Ellermana6168352008-08-20 16:36:11 -07002311 seq_puts(m, "# _------=> CPU# \n");
2312 seq_puts(m, "# / _-----=> irqs-off \n");
2313 seq_puts(m, "# | / _----=> need-resched \n");
2314 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2315 seq_puts(m, "# ||| / _--=> preempt-depth \n");
Steven Rostedte6e1e252011-03-09 10:41:56 -05002316 seq_puts(m, "# |||| / delay \n");
2317 seq_puts(m, "# cmd pid ||||| time | caller \n");
2318 seq_puts(m, "# \\ / ||||| \\ | / \n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002319}
2320
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002321static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002322{
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002323 unsigned long total;
2324 unsigned long entries;
2325
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002326 get_total_entries(buf, &total, &entries);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002327 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2328 entries, total, num_online_cpus());
2329 seq_puts(m, "#\n");
2330}
2331
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002332static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002333{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002334 print_event_info(buf, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002335 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
Michael Ellermana6168352008-08-20 16:36:11 -07002336 seq_puts(m, "# | | | | |\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002337}
2338
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002339static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedt77271ce2011-11-17 09:34:33 -05002340{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002341 print_event_info(buf, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002342 seq_puts(m, "# _-----=> irqs-off\n");
2343 seq_puts(m, "# / _----=> need-resched\n");
2344 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2345 seq_puts(m, "# || / _--=> preempt-depth\n");
2346 seq_puts(m, "# ||| / delay\n");
2347 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2348 seq_puts(m, "# | | | |||| | |\n");
2349}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002350
Jiri Olsa62b915f2010-04-02 19:01:22 +02002351void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002352print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2353{
2354 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002355 struct trace_buffer *buf = iter->trace_buffer;
2356 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002357 struct tracer *type = iter->trace;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002358 unsigned long entries;
2359 unsigned long total;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002360 const char *name = "preemption";
2361
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05002362 name = type->name;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002363
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002364 get_total_entries(buf, &total, &entries);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002365
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002366 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002367 name, UTS_RELEASE);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002368 seq_puts(m, "# -----------------------------------"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002369 "---------------------------------\n");
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002370 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002371 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
Steven Rostedt57f50be2008-05-12 21:20:44 +02002372 nsecs_to_usecs(data->saved_latency),
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002373 entries,
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02002374 total,
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002375 buf->cpu,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002376#if defined(CONFIG_PREEMPT_NONE)
2377 "server",
2378#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2379 "desktop",
Steven Rostedtb5c21b42008-07-10 20:58:12 -04002380#elif defined(CONFIG_PREEMPT)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002381 "preempt",
2382#else
2383 "unknown",
2384#endif
2385 /* These are reserved for later use */
2386 0, 0, 0, 0);
2387#ifdef CONFIG_SMP
2388 seq_printf(m, " #P:%d)\n", num_online_cpus());
2389#else
2390 seq_puts(m, ")\n");
2391#endif
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002392 seq_puts(m, "# -----------------\n");
2393 seq_printf(m, "# | task: %.16s-%d "
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002394 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
Eric W. Biedermand20b92a2012-03-13 16:02:19 -07002395 data->comm, data->pid,
2396 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002397 data->policy, data->rt_priority);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002398 seq_puts(m, "# -----------------\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002399
2400 if (data->critical_start) {
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002401 seq_puts(m, "# => started at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002402 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2403 trace_print_seq(m, &iter->seq);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002404 seq_puts(m, "\n# => ended at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002405 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2406 trace_print_seq(m, &iter->seq);
Steven Rostedt8248ac02009-09-02 12:27:41 -04002407 seq_puts(m, "\n#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002408 }
2409
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002410 seq_puts(m, "#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002411}
2412
Steven Rostedta3097202008-11-07 22:36:02 -05002413static void test_cpu_buff_start(struct trace_iterator *iter)
2414{
2415 struct trace_seq *s = &iter->seq;
2416
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002417 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2418 return;
2419
2420 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2421 return;
2422
Rusty Russell44623442009-01-01 10:12:23 +10302423 if (cpumask_test_cpu(iter->cpu, iter->started))
Steven Rostedta3097202008-11-07 22:36:02 -05002424 return;
2425
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002426 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002427 return;
2428
Rusty Russell44623442009-01-01 10:12:23 +10302429 cpumask_set_cpu(iter->cpu, iter->started);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002430
2431 /* Don't print started cpu buffer for the first entry of the trace */
2432 if (iter->idx > 1)
2433 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2434 iter->cpu);
Steven Rostedta3097202008-11-07 22:36:02 -05002435}
2436
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002437static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002438{
Steven Rostedt214023c2008-05-12 21:20:46 +02002439 struct trace_seq *s = &iter->seq;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002440 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002441 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002442 struct trace_event *event;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002443
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002444 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002445
Steven Rostedta3097202008-11-07 22:36:02 -05002446 test_cpu_buff_start(iter);
2447
Steven Rostedtf633cef2008-12-23 23:24:13 -05002448 event = ftrace_find_event(entry->type);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002449
2450 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt27d48be2009-03-04 21:57:29 -05002451 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2452 if (!trace_print_lat_context(iter))
2453 goto partial;
2454 } else {
2455 if (!trace_print_context(iter))
2456 goto partial;
2457 }
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002458 }
2459
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002460 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002461 return event->funcs->trace(iter, sym_flags, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002462
2463 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2464 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04002465
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002466 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002467partial:
2468 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002469}
2470
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002471static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002472{
2473 struct trace_seq *s = &iter->seq;
2474 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002475 struct trace_event *event;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002476
2477 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002478
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002479 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002480 if (!trace_seq_printf(s, "%d %d %llu ",
2481 entry->pid, iter->cpu, iter->ts))
2482 goto partial;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002483 }
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002484
Steven Rostedtf633cef2008-12-23 23:24:13 -05002485 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002486 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002487 return event->funcs->raw(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002488
2489 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2490 goto partial;
Steven Rostedt7104f302008-10-01 10:52:51 -04002491
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002492 return TRACE_TYPE_HANDLED;
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002493partial:
2494 return TRACE_TYPE_PARTIAL_LINE;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002495}
2496
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002497static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002498{
2499 struct trace_seq *s = &iter->seq;
2500 unsigned char newline = '\n';
2501 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002502 struct trace_event *event;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002503
2504 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002505
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002506 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2507 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2508 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2509 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2510 }
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002511
Steven Rostedtf633cef2008-12-23 23:24:13 -05002512 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002513 if (event) {
Steven Rostedta9a57762010-04-22 18:46:14 -04002514 enum print_line_t ret = event->funcs->hex(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002515 if (ret != TRACE_TYPE_HANDLED)
2516 return ret;
2517 }
Steven Rostedt7104f302008-10-01 10:52:51 -04002518
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002519 SEQ_PUT_FIELD_RET(s, newline);
2520
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002521 return TRACE_TYPE_HANDLED;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002522}
2523
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002524static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002525{
2526 struct trace_seq *s = &iter->seq;
2527 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002528 struct trace_event *event;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002529
2530 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002531
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002532 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2533 SEQ_PUT_FIELD_RET(s, entry->pid);
Steven Rostedt1830b52d2009-02-07 19:38:43 -05002534 SEQ_PUT_FIELD_RET(s, iter->cpu);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002535 SEQ_PUT_FIELD_RET(s, iter->ts);
2536 }
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002537
Steven Rostedtf633cef2008-12-23 23:24:13 -05002538 event = ftrace_find_event(entry->type);
Steven Rostedta9a57762010-04-22 18:46:14 -04002539 return event ? event->funcs->binary(iter, 0, event) :
2540 TRACE_TYPE_HANDLED;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002541}
2542
Jiri Olsa62b915f2010-04-02 19:01:22 +02002543int trace_empty(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002544{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002545 struct ring_buffer_iter *buf_iter;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002546 int cpu;
2547
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002548 /* If we are looking at one CPU buffer, only check that one */
Steven Rostedtae3b5092013-01-23 15:22:59 -05002549 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002550 cpu = iter->cpu_file;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002551 buf_iter = trace_buffer_iter(iter, cpu);
2552 if (buf_iter) {
2553 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002554 return 0;
2555 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002556 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002557 return 0;
2558 }
2559 return 1;
2560 }
2561
Steven Rostedtab464282008-05-12 21:21:00 +02002562 for_each_tracing_cpu(cpu) {
Steven Rostedt6d158a82012-06-27 20:46:14 -04002563 buf_iter = trace_buffer_iter(iter, cpu);
2564 if (buf_iter) {
2565 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedtd7690412008-10-01 00:29:53 -04002566 return 0;
2567 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002568 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
Steven Rostedtd7690412008-10-01 00:29:53 -04002569 return 0;
2570 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002571 }
Steven Rostedtd7690412008-10-01 00:29:53 -04002572
Frederic Weisbecker797d3712008-09-30 18:13:45 +02002573 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002574}
2575
Lai Jiangshan4f535962009-05-18 19:35:34 +08002576/* Called with trace_event_read_lock() held. */
Jason Wessel955b61e2010-08-05 09:22:23 -05002577enum print_line_t print_trace_line(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002578{
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002579 enum print_line_t ret;
2580
Jiri Olsaee5e51f2011-03-25 12:05:18 +01002581 if (iter->lost_events &&
2582 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2583 iter->cpu, iter->lost_events))
2584 return TRACE_TYPE_PARTIAL_LINE;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002585
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002586 if (iter->trace && iter->trace->print_line) {
2587 ret = iter->trace->print_line(iter);
2588 if (ret != TRACE_TYPE_UNHANDLED)
2589 return ret;
2590 }
Thomas Gleixner72829bc2008-05-23 21:37:28 +02002591
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -05002592 if (iter->ent->type == TRACE_BPUTS &&
2593 trace_flags & TRACE_ITER_PRINTK &&
2594 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2595 return trace_print_bputs_msg_only(iter);
2596
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002597 if (iter->ent->type == TRACE_BPRINT &&
2598 trace_flags & TRACE_ITER_PRINTK &&
2599 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002600 return trace_print_bprintk_msg_only(iter);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002601
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002602 if (iter->ent->type == TRACE_PRINT &&
2603 trace_flags & TRACE_ITER_PRINTK &&
2604 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002605 return trace_print_printk_msg_only(iter);
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002606
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002607 if (trace_flags & TRACE_ITER_BIN)
2608 return print_bin_fmt(iter);
2609
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002610 if (trace_flags & TRACE_ITER_HEX)
2611 return print_hex_fmt(iter);
2612
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002613 if (trace_flags & TRACE_ITER_RAW)
2614 return print_raw_fmt(iter);
2615
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002616 return print_trace_fmt(iter);
2617}
2618
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002619void trace_latency_header(struct seq_file *m)
2620{
2621 struct trace_iterator *iter = m->private;
2622
2623 /* print nothing if the buffers are empty */
2624 if (trace_empty(iter))
2625 return;
2626
2627 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2628 print_trace_header(m, iter);
2629
2630 if (!(trace_flags & TRACE_ITER_VERBOSE))
2631 print_lat_help_header(m);
2632}
2633
Jiri Olsa62b915f2010-04-02 19:01:22 +02002634void trace_default_header(struct seq_file *m)
2635{
2636 struct trace_iterator *iter = m->private;
2637
Jiri Olsaf56e7f82011-06-03 16:58:49 +02002638 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2639 return;
2640
Jiri Olsa62b915f2010-04-02 19:01:22 +02002641 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2642 /* print nothing if the buffers are empty */
2643 if (trace_empty(iter))
2644 return;
2645 print_trace_header(m, iter);
2646 if (!(trace_flags & TRACE_ITER_VERBOSE))
2647 print_lat_help_header(m);
2648 } else {
Steven Rostedt77271ce2011-11-17 09:34:33 -05002649 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2650 if (trace_flags & TRACE_ITER_IRQ_INFO)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002651 print_func_help_header_irq(iter->trace_buffer, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002652 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002653 print_func_help_header(iter->trace_buffer, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002654 }
Jiri Olsa62b915f2010-04-02 19:01:22 +02002655 }
2656}
2657
Steven Rostedte0a413f2011-09-29 21:26:16 -04002658static void test_ftrace_alive(struct seq_file *m)
2659{
2660 if (!ftrace_is_dead())
2661 return;
2662 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2663 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2664}
2665
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002666#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002667static void show_snapshot_main_help(struct seq_file *m)
2668{
2669 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2670 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2671 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2672 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2673 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2674 seq_printf(m, "# is not a '0' or '1')\n");
2675}
2676
2677static void show_snapshot_percpu_help(struct seq_file *m)
2678{
2679 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2680#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2681 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2682 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2683#else
2684 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2685 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2686#endif
2687 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2688 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2689 seq_printf(m, "# is not a '0' or '1')\n");
2690}
2691
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002692static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2693{
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05002694 if (iter->tr->allocated_snapshot)
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002695 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2696 else
2697 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2698
2699 seq_printf(m, "# Snapshot commands:\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002700 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2701 show_snapshot_main_help(m);
2702 else
2703 show_snapshot_percpu_help(m);
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002704}
2705#else
2706/* Should never be called */
2707static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2708#endif
2709
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002710static int s_show(struct seq_file *m, void *v)
2711{
2712 struct trace_iterator *iter = v;
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002713 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002714
2715 if (iter->ent == NULL) {
2716 if (iter->tr) {
2717 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2718 seq_puts(m, "#\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002719 test_ftrace_alive(m);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002720 }
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002721 if (iter->snapshot && trace_empty(iter))
2722 print_snapshot_help(m, iter);
2723 else if (iter->trace && iter->trace->print_header)
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002724 iter->trace->print_header(m);
Jiri Olsa62b915f2010-04-02 19:01:22 +02002725 else
2726 trace_default_header(m);
2727
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002728 } else if (iter->leftover) {
2729 /*
2730 * If we filled the seq_file buffer earlier, we
2731 * want to just show it now.
2732 */
2733 ret = trace_print_seq(m, &iter->seq);
2734
2735 /* ret should this time be zero, but you never know */
2736 iter->leftover = ret;
2737
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002738 } else {
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002739 print_trace_line(iter);
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002740 ret = trace_print_seq(m, &iter->seq);
2741 /*
2742 * If we overflow the seq_file buffer, then it will
2743 * ask us for this data again at start up.
2744 * Use that instead.
2745 * ret is 0 if seq_file write succeeded.
2746 * -1 otherwise.
2747 */
2748 iter->leftover = ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002749 }
2750
2751 return 0;
2752}
2753
James Morris88e9d342009-09-22 16:43:43 -07002754static const struct seq_operations tracer_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002755 .start = s_start,
2756 .next = s_next,
2757 .stop = s_stop,
2758 .show = s_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002759};
2760
Ingo Molnare309b412008-05-12 21:20:51 +02002761static struct trace_iterator *
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002762__tracing_open(struct inode *inode, struct file *file, bool snapshot)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002763{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002764 struct trace_cpu *tc = inode->i_private;
2765 struct trace_array *tr = tc->tr;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002766 struct trace_iterator *iter;
Jiri Olsa50e18b92012-04-25 10:23:39 +02002767 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002768
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002769 if (tracing_disabled)
2770 return ERR_PTR(-ENODEV);
Steven Rostedt60a11772008-05-12 21:20:44 +02002771
Jiri Olsa50e18b92012-04-25 10:23:39 +02002772 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002773 if (!iter)
2774 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002775
Steven Rostedt6d158a82012-06-27 20:46:14 -04002776 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2777 GFP_KERNEL);
Dan Carpenter93574fc2012-07-11 09:35:08 +03002778 if (!iter->buffer_iter)
2779 goto release;
2780
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002781 /*
2782 * We make a copy of the current tracer to avoid concurrent
2783 * changes on it while we are reading.
2784 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002785 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002786 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002787 if (!iter->trace)
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002788 goto fail;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002789
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002790 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002791
Li Zefan79f55992009-06-15 14:58:26 +08002792 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002793 goto fail;
2794
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002795 iter->tr = tr;
2796
2797#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002798 /* Currently only the top directory has a snapshot */
2799 if (tr->current_trace->print_max || snapshot)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002800 iter->trace_buffer = &tr->max_buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002801 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002802#endif
2803 iter->trace_buffer = &tr->trace_buffer;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002804 iter->snapshot = snapshot;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002805 iter->pos = -1;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002806 mutex_init(&iter->mutex);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002807 iter->cpu_file = tc->cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002808
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002809 /* Notify the tracer early; before we stop tracing. */
2810 if (iter->trace && iter->trace->open)
Markus Metzgera93751c2008-12-11 13:53:26 +01002811 iter->trace->open(iter);
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002812
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002813 /* Annotate start of buffers if we had overruns */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002814 if (ring_buffer_overruns(iter->trace_buffer->buffer))
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002815 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2816
David Sharp8be07092012-11-13 12:18:22 -08002817 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2818 if (trace_clocks[trace_clock_id].in_ns)
2819 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2820
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002821 /* stop the trace while dumping if we are not opening "snapshot" */
2822 if (!iter->snapshot)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002823 tracing_stop_tr(tr);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002824
Steven Rostedtae3b5092013-01-23 15:22:59 -05002825 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002826 for_each_tracing_cpu(cpu) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002827 iter->buffer_iter[cpu] =
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002828 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
David Miller72c9ddf2010-04-20 15:47:11 -07002829 }
2830 ring_buffer_read_prepare_sync();
2831 for_each_tracing_cpu(cpu) {
2832 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002833 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002834 }
2835 } else {
2836 cpu = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002837 iter->buffer_iter[cpu] =
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002838 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
David Miller72c9ddf2010-04-20 15:47:11 -07002839 ring_buffer_read_prepare_sync();
2840 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002841 tracing_iter_reset(iter, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002842 }
2843
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05002844 tr->ref++;
2845
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002846 mutex_unlock(&trace_types_lock);
2847
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002848 return iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002849
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002850 fail:
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002851 mutex_unlock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002852 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04002853 kfree(iter->buffer_iter);
Dan Carpenter93574fc2012-07-11 09:35:08 +03002854release:
Jiri Olsa50e18b92012-04-25 10:23:39 +02002855 seq_release_private(inode, file);
2856 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002857}
2858
2859int tracing_open_generic(struct inode *inode, struct file *filp)
2860{
Steven Rostedt60a11772008-05-12 21:20:44 +02002861 if (tracing_disabled)
2862 return -ENODEV;
2863
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002864 filp->private_data = inode->i_private;
2865 return 0;
2866}
2867
Hannes Eder4fd27352009-02-10 19:44:12 +01002868static int tracing_release(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002869{
matt mooney907f2782010-09-27 19:04:53 -07002870 struct seq_file *m = file->private_data;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002871 struct trace_iterator *iter;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002872 struct trace_array *tr;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002873 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002874
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002875 if (!(file->f_mode & FMODE_READ))
2876 return 0;
2877
2878 iter = m->private;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002879 tr = iter->tr;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002880
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002881 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05002882
2883 WARN_ON(!tr->ref);
2884 tr->ref--;
2885
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002886 for_each_tracing_cpu(cpu) {
2887 if (iter->buffer_iter[cpu])
2888 ring_buffer_read_finish(iter->buffer_iter[cpu]);
2889 }
2890
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002891 if (iter->trace && iter->trace->close)
2892 iter->trace->close(iter);
2893
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002894 if (!iter->snapshot)
2895 /* reenable tracing if it was previously enabled */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002896 tracing_start_tr(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002897 mutex_unlock(&trace_types_lock);
2898
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002899 mutex_destroy(&iter->mutex);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002900 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002901 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04002902 kfree(iter->buffer_iter);
Jiri Olsa50e18b92012-04-25 10:23:39 +02002903 seq_release_private(inode, file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002904 return 0;
2905}
2906
2907static int tracing_open(struct inode *inode, struct file *file)
2908{
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05002909 struct trace_iterator *iter;
2910 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002911
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002912 /* If this file was open for write, then erase contents */
2913 if ((file->f_mode & FMODE_WRITE) &&
Steven Rostedt8650ae32009-07-22 23:29:30 -04002914 (file->f_flags & O_TRUNC)) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002915 struct trace_cpu *tc = inode->i_private;
2916 struct trace_array *tr = tc->tr;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002917
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002918 if (tc->cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002919 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002920 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002921 tracing_reset(&tr->trace_buffer, tc->cpu);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002922 }
2923
2924 if (file->f_mode & FMODE_READ) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002925 iter = __tracing_open(inode, file, false);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002926 if (IS_ERR(iter))
2927 ret = PTR_ERR(iter);
2928 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2929 iter->iter_flags |= TRACE_FILE_LAT_FMT;
2930 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002931 return ret;
2932}
2933
Ingo Molnare309b412008-05-12 21:20:51 +02002934static void *
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002935t_next(struct seq_file *m, void *v, loff_t *pos)
2936{
Li Zefanf129e962009-06-24 09:53:44 +08002937 struct tracer *t = v;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002938
2939 (*pos)++;
2940
2941 if (t)
2942 t = t->next;
2943
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002944 return t;
2945}
2946
2947static void *t_start(struct seq_file *m, loff_t *pos)
2948{
Li Zefanf129e962009-06-24 09:53:44 +08002949 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002950 loff_t l = 0;
2951
2952 mutex_lock(&trace_types_lock);
Li Zefanf129e962009-06-24 09:53:44 +08002953 for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002954 ;
2955
2956 return t;
2957}
2958
2959static void t_stop(struct seq_file *m, void *p)
2960{
2961 mutex_unlock(&trace_types_lock);
2962}
2963
2964static int t_show(struct seq_file *m, void *v)
2965{
2966 struct tracer *t = v;
2967
2968 if (!t)
2969 return 0;
2970
2971 seq_printf(m, "%s", t->name);
2972 if (t->next)
2973 seq_putc(m, ' ');
2974 else
2975 seq_putc(m, '\n');
2976
2977 return 0;
2978}
2979
James Morris88e9d342009-09-22 16:43:43 -07002980static const struct seq_operations show_traces_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02002981 .start = t_start,
2982 .next = t_next,
2983 .stop = t_stop,
2984 .show = t_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002985};
2986
2987static int show_traces_open(struct inode *inode, struct file *file)
2988{
Steven Rostedt60a11772008-05-12 21:20:44 +02002989 if (tracing_disabled)
2990 return -ENODEV;
2991
Li Zefanf129e962009-06-24 09:53:44 +08002992 return seq_open(file, &show_traces_seq_ops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002993}
2994
Steven Rostedt4acd4d02009-03-18 10:40:24 -04002995static ssize_t
2996tracing_write_stub(struct file *filp, const char __user *ubuf,
2997 size_t count, loff_t *ppos)
2998{
2999 return count;
3000}
3001
Slava Pestov364829b2010-11-24 15:13:16 -08003002static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
3003{
3004 if (file->f_mode & FMODE_READ)
3005 return seq_lseek(file, offset, origin);
3006 else
3007 return 0;
3008}
3009
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003010static const struct file_operations tracing_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003011 .open = tracing_open,
3012 .read = seq_read,
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003013 .write = tracing_write_stub,
Slava Pestov364829b2010-11-24 15:13:16 -08003014 .llseek = tracing_seek,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003015 .release = tracing_release,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003016};
3017
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003018static const struct file_operations show_traces_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003019 .open = show_traces_open,
3020 .read = seq_read,
3021 .release = seq_release,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003022 .llseek = seq_lseek,
Ingo Molnarc7078de2008-05-12 21:20:52 +02003023};
3024
Ingo Molnar36dfe922008-05-12 21:20:52 +02003025/*
3026 * Only trace on a CPU if the bitmask is set:
3027 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303028static cpumask_var_t tracing_cpumask;
Ingo Molnar36dfe922008-05-12 21:20:52 +02003029
3030/*
3031 * The tracer itself will not take this lock, but still we want
3032 * to provide a consistent cpumask to user-space:
3033 */
3034static DEFINE_MUTEX(tracing_cpumask_update_lock);
3035
3036/*
3037 * Temporary storage for the character representation of the
3038 * CPU bitmask (and one more byte for the newline):
3039 */
3040static char mask_str[NR_CPUS + 1];
3041
Ingo Molnarc7078de2008-05-12 21:20:52 +02003042static ssize_t
3043tracing_cpumask_read(struct file *filp, char __user *ubuf,
3044 size_t count, loff_t *ppos)
3045{
Ingo Molnar36dfe922008-05-12 21:20:52 +02003046 int len;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003047
3048 mutex_lock(&tracing_cpumask_update_lock);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003049
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303050 len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003051 if (count - len < 2) {
3052 count = -EINVAL;
3053 goto out_err;
3054 }
3055 len += sprintf(mask_str + len, "\n");
3056 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3057
3058out_err:
Ingo Molnarc7078de2008-05-12 21:20:52 +02003059 mutex_unlock(&tracing_cpumask_update_lock);
3060
3061 return count;
3062}
3063
3064static ssize_t
3065tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3066 size_t count, loff_t *ppos)
3067{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003068 struct trace_array *tr = filp->private_data;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303069 cpumask_var_t tracing_cpumask_new;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003070 int err, cpu;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303071
3072 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3073 return -ENOMEM;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003074
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303075 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003076 if (err)
3077 goto err_unlock;
3078
Li Zefan215368e2009-06-15 10:56:42 +08003079 mutex_lock(&tracing_cpumask_update_lock);
3080
Steven Rostedta5e25882008-12-02 15:34:05 -05003081 local_irq_disable();
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01003082 arch_spin_lock(&ftrace_max_lock);
Steven Rostedtab464282008-05-12 21:21:00 +02003083 for_each_tracing_cpu(cpu) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02003084 /*
3085 * Increase/decrease the disabled counter if we are
3086 * about to flip a bit in the cpumask:
3087 */
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303088 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
3089 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003090 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3091 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003092 }
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303093 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
3094 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003095 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3096 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003097 }
3098 }
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01003099 arch_spin_unlock(&ftrace_max_lock);
Steven Rostedta5e25882008-12-02 15:34:05 -05003100 local_irq_enable();
Ingo Molnar36dfe922008-05-12 21:20:52 +02003101
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303102 cpumask_copy(tracing_cpumask, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003103
Ingo Molnarc7078de2008-05-12 21:20:52 +02003104 mutex_unlock(&tracing_cpumask_update_lock);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303105 free_cpumask_var(tracing_cpumask_new);
Ingo Molnarc7078de2008-05-12 21:20:52 +02003106
Ingo Molnarc7078de2008-05-12 21:20:52 +02003107 return count;
Ingo Molnar36dfe922008-05-12 21:20:52 +02003108
3109err_unlock:
Li Zefan215368e2009-06-15 10:56:42 +08003110 free_cpumask_var(tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003111
3112 return err;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003113}
3114
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003115static const struct file_operations tracing_cpumask_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003116 .open = tracing_open_generic,
3117 .read = tracing_cpumask_read,
3118 .write = tracing_cpumask_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003119 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003120};
3121
Li Zefanfdb372e2009-12-08 11:15:59 +08003122static int tracing_trace_options_show(struct seq_file *m, void *v)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003123{
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003124 struct tracer_opt *trace_opts;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003125 struct trace_array *tr = m->private;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003126 u32 tracer_flags;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003127 int i;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003128
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003129 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003130 tracer_flags = tr->current_trace->flags->val;
3131 trace_opts = tr->current_trace->flags->opts;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003132
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003133 for (i = 0; trace_options[i]; i++) {
3134 if (trace_flags & (1 << i))
Li Zefanfdb372e2009-12-08 11:15:59 +08003135 seq_printf(m, "%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003136 else
Li Zefanfdb372e2009-12-08 11:15:59 +08003137 seq_printf(m, "no%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003138 }
3139
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003140 for (i = 0; trace_opts[i].name; i++) {
3141 if (tracer_flags & trace_opts[i].bit)
Li Zefanfdb372e2009-12-08 11:15:59 +08003142 seq_printf(m, "%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003143 else
Li Zefanfdb372e2009-12-08 11:15:59 +08003144 seq_printf(m, "no%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003145 }
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003146 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003147
Li Zefanfdb372e2009-12-08 11:15:59 +08003148 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003149}
3150
Li Zefan8d18eaa2009-12-08 11:17:06 +08003151static int __set_tracer_option(struct tracer *trace,
3152 struct tracer_flags *tracer_flags,
3153 struct tracer_opt *opts, int neg)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003154{
Li Zefan8d18eaa2009-12-08 11:17:06 +08003155 int ret;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003156
Li Zefan8d18eaa2009-12-08 11:17:06 +08003157 ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003158 if (ret)
3159 return ret;
3160
3161 if (neg)
Zhaolei77708412009-08-07 18:53:21 +08003162 tracer_flags->val &= ~opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003163 else
Zhaolei77708412009-08-07 18:53:21 +08003164 tracer_flags->val |= opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003165 return 0;
3166}
3167
Li Zefan8d18eaa2009-12-08 11:17:06 +08003168/* Try to assign a tracer specific option */
3169static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3170{
3171 struct tracer_flags *tracer_flags = trace->flags;
3172 struct tracer_opt *opts = NULL;
3173 int i;
3174
3175 for (i = 0; tracer_flags->opts[i].name; i++) {
3176 opts = &tracer_flags->opts[i];
3177
3178 if (strcmp(cmp, opts->name) == 0)
3179 return __set_tracer_option(trace, trace->flags,
3180 opts, neg);
3181 }
3182
3183 return -EINVAL;
3184}
3185
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003186/* Some tracers require overwrite to stay enabled */
3187int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3188{
3189 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3190 return -1;
3191
3192 return 0;
3193}
3194
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003195int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003196{
3197 /* do nothing if flag is already set */
3198 if (!!(trace_flags & mask) == !!enabled)
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003199 return 0;
3200
3201 /* Give the tracer a chance to approve the change */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003202 if (tr->current_trace->flag_changed)
3203 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003204 return -EINVAL;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003205
3206 if (enabled)
3207 trace_flags |= mask;
3208 else
3209 trace_flags &= ~mask;
Li Zefane870e9a2010-07-02 11:07:32 +08003210
3211 if (mask == TRACE_ITER_RECORD_CMD)
3212 trace_event_enable_cmd_record(enabled);
David Sharp750912f2010-12-08 13:46:47 -08003213
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003214 if (mask == TRACE_ITER_OVERWRITE) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003215 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003216#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003217 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003218#endif
3219 }
Steven Rostedt81698832012-10-11 10:15:05 -04003220
3221 if (mask == TRACE_ITER_PRINTK)
3222 trace_printk_start_stop_comm(enabled);
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003223
3224 return 0;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003225}
3226
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003227static int trace_set_options(struct trace_array *tr, char *option)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003228{
Li Zefan8d18eaa2009-12-08 11:17:06 +08003229 char *cmp;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003230 int neg = 0;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003231 int ret = -ENODEV;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003232 int i;
3233
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003234 cmp = strstrip(option);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003235
Li Zefan8d18eaa2009-12-08 11:17:06 +08003236 if (strncmp(cmp, "no", 2) == 0) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003237 neg = 1;
3238 cmp += 2;
3239 }
3240
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003241 mutex_lock(&trace_types_lock);
3242
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003243 for (i = 0; trace_options[i]; i++) {
Li Zefan8d18eaa2009-12-08 11:17:06 +08003244 if (strcmp(cmp, trace_options[i]) == 0) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003245 ret = set_tracer_flag(tr, 1 << i, !neg);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003246 break;
3247 }
3248 }
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003249
3250 /* If no option could be set, test the specific tracer options */
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003251 if (!trace_options[i])
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003252 ret = set_tracer_option(tr->current_trace, cmp, neg);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003253
3254 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003255
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003256 return ret;
3257}
3258
3259static ssize_t
3260tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3261 size_t cnt, loff_t *ppos)
3262{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003263 struct seq_file *m = filp->private_data;
3264 struct trace_array *tr = m->private;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003265 char buf[64];
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003266 int ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003267
3268 if (cnt >= sizeof(buf))
3269 return -EINVAL;
3270
3271 if (copy_from_user(&buf, ubuf, cnt))
3272 return -EFAULT;
3273
Steven Rostedta8dd2172013-01-09 20:54:17 -05003274 buf[cnt] = 0;
3275
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003276 ret = trace_set_options(tr, buf);
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003277 if (ret < 0)
3278 return ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003279
Jiri Olsacf8517c2009-10-23 19:36:16 -04003280 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003281
3282 return cnt;
3283}
3284
Li Zefanfdb372e2009-12-08 11:15:59 +08003285static int tracing_trace_options_open(struct inode *inode, struct file *file)
3286{
3287 if (tracing_disabled)
3288 return -ENODEV;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003289
3290 return single_open(file, tracing_trace_options_show, inode->i_private);
Li Zefanfdb372e2009-12-08 11:15:59 +08003291}
3292
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003293static const struct file_operations tracing_iter_fops = {
Li Zefanfdb372e2009-12-08 11:15:59 +08003294 .open = tracing_trace_options_open,
3295 .read = seq_read,
3296 .llseek = seq_lseek,
3297 .release = single_release,
Steven Rostedtee6bce52008-11-12 17:52:37 -05003298 .write = tracing_trace_options_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003299};
3300
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003301static const char readme_msg[] =
3302 "tracing mini-HOWTO:\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003303 "# mount -t debugfs nodev /sys/kernel/debug\n\n"
3304 "# cat /sys/kernel/debug/tracing/available_tracers\n"
Geunsik Lim1e42e832012-02-08 19:05:36 +09003305 "wakeup wakeup_rt preemptirqsoff preemptoff irqsoff function nop\n\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003306 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Nikanth Karthikesanbc2b6872009-03-23 11:58:31 +05303307 "nop\n"
Geunsik Lim1e42e832012-02-08 19:05:36 +09003308 "# echo wakeup > /sys/kernel/debug/tracing/current_tracer\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003309 "# cat /sys/kernel/debug/tracing/current_tracer\n"
Geunsik Lim1e42e832012-02-08 19:05:36 +09003310 "wakeup\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003311 "# cat /sys/kernel/debug/tracing/trace_options\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003312 "noprint-parent nosym-offset nosym-addr noverbose\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003313 "# echo print-parent > /sys/kernel/debug/tracing/trace_options\n"
Geunsik Lim9b5f8b32011-08-12 14:30:22 +09003314 "# echo 1 > /sys/kernel/debug/tracing/tracing_on\n"
GeunSik Lim156f5a72009-06-02 15:01:37 +09003315 "# cat /sys/kernel/debug/tracing/trace > /tmp/trace.txt\n"
Geunsik Lim9b5f8b32011-08-12 14:30:22 +09003316 "# echo 0 > /sys/kernel/debug/tracing/tracing_on\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003317;
3318
3319static ssize_t
3320tracing_readme_read(struct file *filp, char __user *ubuf,
3321 size_t cnt, loff_t *ppos)
3322{
3323 return simple_read_from_buffer(ubuf, cnt, ppos,
3324 readme_msg, strlen(readme_msg));
3325}
3326
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003327static const struct file_operations tracing_readme_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003328 .open = tracing_open_generic,
3329 .read = tracing_readme_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003330 .llseek = generic_file_llseek,
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003331};
3332
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003333static ssize_t
Avadh Patel69abe6a2009-04-10 16:04:48 -04003334tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3335 size_t cnt, loff_t *ppos)
3336{
3337 char *buf_comm;
3338 char *file_buf;
3339 char *buf;
3340 int len = 0;
3341 int pid;
3342 int i;
3343
3344 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3345 if (!file_buf)
3346 return -ENOMEM;
3347
3348 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3349 if (!buf_comm) {
3350 kfree(file_buf);
3351 return -ENOMEM;
3352 }
3353
3354 buf = file_buf;
3355
3356 for (i = 0; i < SAVED_CMDLINES; i++) {
3357 int r;
3358
3359 pid = map_cmdline_to_pid[i];
3360 if (pid == -1 || pid == NO_CMDLINE_MAP)
3361 continue;
3362
3363 trace_find_cmdline(pid, buf_comm);
3364 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3365 buf += r;
3366 len += r;
3367 }
3368
3369 len = simple_read_from_buffer(ubuf, cnt, ppos,
3370 file_buf, len);
3371
3372 kfree(file_buf);
3373 kfree(buf_comm);
3374
3375 return len;
3376}
3377
3378static const struct file_operations tracing_saved_cmdlines_fops = {
3379 .open = tracing_open_generic,
3380 .read = tracing_saved_cmdlines_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003381 .llseek = generic_file_llseek,
Avadh Patel69abe6a2009-04-10 16:04:48 -04003382};
3383
3384static ssize_t
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003385tracing_set_trace_read(struct file *filp, char __user *ubuf,
3386 size_t cnt, loff_t *ppos)
3387{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003388 struct trace_array *tr = filp->private_data;
Li Zefanee6c2c12009-09-18 14:06:47 +08003389 char buf[MAX_TRACER_SIZE+2];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003390 int r;
3391
3392 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003393 r = sprintf(buf, "%s\n", tr->current_trace->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003394 mutex_unlock(&trace_types_lock);
3395
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003396 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003397}
3398
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003399int tracer_init(struct tracer *t, struct trace_array *tr)
3400{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003401 tracing_reset_online_cpus(&tr->trace_buffer);
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003402 return t->init(tr);
3403}
3404
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003405static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003406{
3407 int cpu;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05003408
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003409 for_each_tracing_cpu(cpu)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003410 per_cpu_ptr(buf->data, cpu)->entries = val;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003411}
3412
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003413#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookad60da502012-10-17 11:56:16 +09003414/* resize @tr's buffer to the size of @size_tr's entries */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003415static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3416 struct trace_buffer *size_buf, int cpu_id)
Hiraku Toyookad60da502012-10-17 11:56:16 +09003417{
3418 int cpu, ret = 0;
3419
3420 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3421 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003422 ret = ring_buffer_resize(trace_buf->buffer,
3423 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
Hiraku Toyookad60da502012-10-17 11:56:16 +09003424 if (ret < 0)
3425 break;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003426 per_cpu_ptr(trace_buf->data, cpu)->entries =
3427 per_cpu_ptr(size_buf->data, cpu)->entries;
Hiraku Toyookad60da502012-10-17 11:56:16 +09003428 }
3429 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003430 ret = ring_buffer_resize(trace_buf->buffer,
3431 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
Hiraku Toyookad60da502012-10-17 11:56:16 +09003432 if (ret == 0)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003433 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3434 per_cpu_ptr(size_buf->data, cpu_id)->entries;
Hiraku Toyookad60da502012-10-17 11:56:16 +09003435 }
3436
3437 return ret;
3438}
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003439#endif /* CONFIG_TRACER_MAX_TRACE */
Hiraku Toyookad60da502012-10-17 11:56:16 +09003440
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003441static int __tracing_resize_ring_buffer(struct trace_array *tr,
3442 unsigned long size, int cpu)
Steven Rostedt73c51622009-03-11 13:42:01 -04003443{
3444 int ret;
3445
3446 /*
3447 * If kernel or user changes the size of the ring buffer
Steven Rostedta123c522009-03-12 11:21:08 -04003448 * we use the size that was given, and we can forget about
3449 * expanding it later.
Steven Rostedt73c51622009-03-11 13:42:01 -04003450 */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05003451 ring_buffer_expanded = true;
Steven Rostedt73c51622009-03-11 13:42:01 -04003452
Steven Rostedtb382ede62012-10-10 21:44:34 -04003453 /* May be called before buffers are initialized */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003454 if (!tr->trace_buffer.buffer)
Steven Rostedtb382ede62012-10-10 21:44:34 -04003455 return 0;
3456
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003457 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003458 if (ret < 0)
3459 return ret;
3460
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003461#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003462 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3463 !tr->current_trace->use_max_tr)
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003464 goto out;
3465
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003466 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003467 if (ret < 0) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003468 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3469 &tr->trace_buffer, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04003470 if (r < 0) {
Steven Rostedta123c522009-03-12 11:21:08 -04003471 /*
3472 * AARGH! We are left with different
3473 * size max buffer!!!!
3474 * The max buffer is our "snapshot" buffer.
3475 * When a tracer needs a snapshot (one of the
3476 * latency tracers), it swaps the max buffer
3477 * with the saved snap shot. We succeeded to
3478 * update the size of the main buffer, but failed to
3479 * update the size of the max buffer. But when we tried
3480 * to reset the main buffer to the original size, we
3481 * failed there too. This is very unlikely to
3482 * happen, but if it does, warn and kill all
3483 * tracing.
3484 */
Steven Rostedt73c51622009-03-11 13:42:01 -04003485 WARN_ON(1);
3486 tracing_disabled = 1;
3487 }
3488 return ret;
3489 }
3490
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003491 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003492 set_buffer_entries(&tr->max_buffer, size);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003493 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003494 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003495
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003496 out:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003497#endif /* CONFIG_TRACER_MAX_TRACE */
3498
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003499 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003500 set_buffer_entries(&tr->trace_buffer, size);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003501 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003502 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
Steven Rostedt73c51622009-03-11 13:42:01 -04003503
3504 return ret;
3505}
3506
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003507static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3508 unsigned long size, int cpu_id)
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003509{
Vaibhav Nagarnaik83f40312012-05-03 18:59:50 -07003510 int ret = size;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003511
3512 mutex_lock(&trace_types_lock);
3513
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003514 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3515 /* make sure, this cpu is enabled in the mask */
3516 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3517 ret = -EINVAL;
3518 goto out;
3519 }
3520 }
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003521
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003522 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003523 if (ret < 0)
3524 ret = -ENOMEM;
3525
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003526out:
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07003527 mutex_unlock(&trace_types_lock);
3528
3529 return ret;
3530}
3531
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003532
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003533/**
3534 * tracing_update_buffers - used by tracing facility to expand ring buffers
3535 *
3536 * To save on memory when the tracing is never used on a system with it
3537 * configured in. The ring buffers are set to a minimum size. But once
3538 * a user starts to use the tracing facility, then they need to grow
3539 * to their default size.
3540 *
3541 * This function is to be called when a tracer is about to be used.
3542 */
3543int tracing_update_buffers(void)
3544{
3545 int ret = 0;
3546
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003547 mutex_lock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003548 if (!ring_buffer_expanded)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003549 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003550 RING_BUFFER_ALL_CPUS);
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003551 mutex_unlock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04003552
3553 return ret;
3554}
3555
Steven Rostedt577b7852009-02-26 23:43:05 -05003556struct trace_option_dentry;
3557
3558static struct trace_option_dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003559create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
Steven Rostedt577b7852009-02-26 23:43:05 -05003560
3561static void
3562destroy_trace_option_files(struct trace_option_dentry *topts);
3563
Steven Rostedtb2821ae2009-02-02 21:38:32 -05003564static int tracing_set_tracer(const char *buf)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003565{
Steven Rostedt577b7852009-02-26 23:43:05 -05003566 static struct trace_option_dentry *topts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003567 struct trace_array *tr = &global_trace;
3568 struct tracer *t;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003569#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt34600f02013-01-22 13:35:11 -05003570 bool had_max_tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003571#endif
Peter Zijlstrad9e54072008-11-01 19:57:37 +01003572 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003573
Steven Rostedt1027fcb2009-03-12 11:33:20 -04003574 mutex_lock(&trace_types_lock);
3575
Steven Rostedt73c51622009-03-11 13:42:01 -04003576 if (!ring_buffer_expanded) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003577 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08003578 RING_BUFFER_ALL_CPUS);
Steven Rostedt73c51622009-03-11 13:42:01 -04003579 if (ret < 0)
Frederic Weisbecker59f586d2009-03-15 22:10:39 +01003580 goto out;
Steven Rostedt73c51622009-03-11 13:42:01 -04003581 ret = 0;
3582 }
3583
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003584 for (t = trace_types; t; t = t->next) {
3585 if (strcmp(t->name, buf) == 0)
3586 break;
3587 }
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02003588 if (!t) {
3589 ret = -EINVAL;
3590 goto out;
3591 }
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003592 if (t == tr->current_trace)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003593 goto out;
3594
Steven Rostedt9f029e82008-11-12 15:24:24 -05003595 trace_branch_disable();
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003596
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003597 tr->current_trace->enabled = false;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003598
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003599 if (tr->current_trace->reset)
3600 tr->current_trace->reset(tr);
Steven Rostedt34600f02013-01-22 13:35:11 -05003601
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003602 /* Current trace needs to be nop_trace before synchronize_sched */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003603 tr->current_trace = &nop_trace;
Steven Rostedt34600f02013-01-22 13:35:11 -05003604
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05003605#ifdef CONFIG_TRACER_MAX_TRACE
3606 had_max_tr = tr->allocated_snapshot;
3607
Steven Rostedt34600f02013-01-22 13:35:11 -05003608 if (had_max_tr && !t->use_max_tr) {
3609 /*
3610 * We need to make sure that the update_max_tr sees that
3611 * current_trace changed to nop_trace to keep it from
3612 * swapping the buffers after we resize it.
3613 * The update_max_tr is called from interrupts disabled
3614 * so a synchronized_sched() is sufficient.
3615 */
3616 synchronize_sched();
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04003617 free_snapshot(tr);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003618 }
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003619#endif
Steven Rostedt577b7852009-02-26 23:43:05 -05003620 destroy_trace_option_files(topts);
3621
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003622 topts = create_trace_option_files(tr, t);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003623
3624#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt34600f02013-01-22 13:35:11 -05003625 if (t->use_max_tr && !had_max_tr) {
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04003626 ret = alloc_snapshot(tr);
Hiraku Toyookad60da502012-10-17 11:56:16 +09003627 if (ret < 0)
3628 goto out;
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09003629 }
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003630#endif
Steven Rostedt577b7852009-02-26 23:43:05 -05003631
Frederic Weisbecker1c800252008-11-16 05:57:26 +01003632 if (t->init) {
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02003633 ret = tracer_init(t, tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +01003634 if (ret)
3635 goto out;
3636 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003637
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003638 tr->current_trace = t;
3639 tr->current_trace->enabled = true;
Steven Rostedt9f029e82008-11-12 15:24:24 -05003640 trace_branch_enable(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003641 out:
3642 mutex_unlock(&trace_types_lock);
3643
Peter Zijlstrad9e54072008-11-01 19:57:37 +01003644 return ret;
3645}
3646
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003647static ssize_t
3648tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3649 size_t cnt, loff_t *ppos)
3650{
Li Zefanee6c2c12009-09-18 14:06:47 +08003651 char buf[MAX_TRACER_SIZE+1];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003652 int i;
3653 size_t ret;
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01003654 int err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003655
Steven Rostedt60063a62008-10-28 10:44:24 -04003656 ret = cnt;
3657
Li Zefanee6c2c12009-09-18 14:06:47 +08003658 if (cnt > MAX_TRACER_SIZE)
3659 cnt = MAX_TRACER_SIZE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003660
3661 if (copy_from_user(&buf, ubuf, cnt))
3662 return -EFAULT;
3663
3664 buf[cnt] = 0;
3665
3666 /* strip ending whitespace. */
3667 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3668 buf[i] = 0;
3669
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01003670 err = tracing_set_tracer(buf);
3671 if (err)
3672 return err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003673
Jiri Olsacf8517c2009-10-23 19:36:16 -04003674 *ppos += ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003675
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02003676 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003677}
3678
3679static ssize_t
3680tracing_max_lat_read(struct file *filp, char __user *ubuf,
3681 size_t cnt, loff_t *ppos)
3682{
3683 unsigned long *ptr = filp->private_data;
3684 char buf[64];
3685 int r;
3686
Steven Rostedtcffae432008-05-12 21:21:00 +02003687 r = snprintf(buf, sizeof(buf), "%ld\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003688 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
Steven Rostedtcffae432008-05-12 21:21:00 +02003689 if (r > sizeof(buf))
3690 r = sizeof(buf);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003691 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003692}
3693
3694static ssize_t
3695tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3696 size_t cnt, loff_t *ppos)
3697{
Hannes Eder5e398412009-02-10 19:44:34 +01003698 unsigned long *ptr = filp->private_data;
Hannes Eder5e398412009-02-10 19:44:34 +01003699 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003700 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003701
Peter Huewe22fe9b52011-06-07 21:58:27 +02003702 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3703 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02003704 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003705
3706 *ptr = val * 1000;
3707
3708 return cnt;
3709}
3710
Steven Rostedtb3806b42008-05-12 21:20:46 +02003711static int tracing_open_pipe(struct inode *inode, struct file *filp)
3712{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003713 struct trace_cpu *tc = inode->i_private;
3714 struct trace_array *tr = tc->tr;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003715 struct trace_iterator *iter;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003716 int ret = 0;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003717
3718 if (tracing_disabled)
3719 return -ENODEV;
3720
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003721 mutex_lock(&trace_types_lock);
3722
Steven Rostedtb3806b42008-05-12 21:20:46 +02003723 /* create a buffer to store the information to pass to userspace */
3724 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003725 if (!iter) {
3726 ret = -ENOMEM;
3727 goto out;
3728 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003729
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003730 /*
3731 * We make a copy of the current tracer to avoid concurrent
3732 * changes on it while we are reading.
3733 */
3734 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3735 if (!iter->trace) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003736 ret = -ENOMEM;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003737 goto fail;
3738 }
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003739 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003740
3741 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3742 ret = -ENOMEM;
3743 goto fail;
Rusty Russell44623442009-01-01 10:12:23 +10303744 }
3745
Steven Rostedta3097202008-11-07 22:36:02 -05003746 /* trace pipe does not show start of buffer */
Rusty Russell44623442009-01-01 10:12:23 +10303747 cpumask_setall(iter->started);
Steven Rostedta3097202008-11-07 22:36:02 -05003748
Steven Rostedt112f38a72009-06-01 15:16:05 -04003749 if (trace_flags & TRACE_ITER_LATENCY_FMT)
3750 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3751
David Sharp8be07092012-11-13 12:18:22 -08003752 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3753 if (trace_clocks[trace_clock_id].in_ns)
3754 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3755
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003756 iter->cpu_file = tc->cpu;
3757 iter->tr = tc->tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003758 iter->trace_buffer = &tc->tr->trace_buffer;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003759 mutex_init(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003760 filp->private_data = iter;
3761
Steven Rostedt107bad82008-05-12 21:21:01 +02003762 if (iter->trace->pipe_open)
3763 iter->trace->pipe_open(iter);
Steven Rostedt107bad82008-05-12 21:21:01 +02003764
Arnd Bergmannb4447862010-07-07 23:40:11 +02003765 nonseekable_open(inode, filp);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003766out:
3767 mutex_unlock(&trace_types_lock);
3768 return ret;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003769
3770fail:
3771 kfree(iter->trace);
3772 kfree(iter);
3773 mutex_unlock(&trace_types_lock);
3774 return ret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003775}
3776
3777static int tracing_release_pipe(struct inode *inode, struct file *file)
3778{
3779 struct trace_iterator *iter = file->private_data;
3780
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003781 mutex_lock(&trace_types_lock);
3782
Steven Rostedt29bf4a52009-12-09 12:37:43 -05003783 if (iter->trace->pipe_close)
Steven Rostedtc521efd2009-12-07 09:06:24 -05003784 iter->trace->pipe_close(iter);
3785
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003786 mutex_unlock(&trace_types_lock);
3787
Rusty Russell44623442009-01-01 10:12:23 +10303788 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003789 mutex_destroy(&iter->mutex);
3790 kfree(iter->trace);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003791 kfree(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003792
3793 return 0;
3794}
3795
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003796static unsigned int
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05003797trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003798{
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05003799 /* Iterators are static, they should be filled or empty */
3800 if (trace_buffer_iter(iter, iter->cpu_file))
3801 return POLLIN | POLLRDNORM;
3802
3803 if (trace_flags & TRACE_ITER_BLOCK)
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003804 /*
3805 * Always select as readable when in blocking mode
3806 */
3807 return POLLIN | POLLRDNORM;
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05003808 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003809 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05003810 filp, poll_table);
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02003811}
3812
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05003813static unsigned int
3814tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3815{
3816 struct trace_iterator *iter = filp->private_data;
3817
3818 return trace_poll(iter, filp, poll_table);
3819}
3820
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003821/*
3822 * This is a make-shift waitqueue.
3823 * A tracer might use this callback on some rare cases:
3824 *
3825 * 1) the current tracer might hold the runqueue lock when it wakes up
3826 * a reader, hence a deadlock (sched, function, and function graph tracers)
3827 * 2) the function tracers, trace all functions, we don't want
3828 * the overhead of calling wake_up and friends
3829 * (and tracing them too)
3830 *
3831 * Anyway, this is really very primitive wakeup.
3832 */
3833void poll_wait_pipe(struct trace_iterator *iter)
3834{
3835 set_current_state(TASK_INTERRUPTIBLE);
3836 /* sleep for 100 msecs, and try again. */
3837 schedule_timeout(HZ / 10);
3838}
3839
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003840/* Must be called with trace_types_lock mutex held. */
3841static int tracing_wait_pipe(struct file *filp)
3842{
3843 struct trace_iterator *iter = filp->private_data;
3844
3845 while (trace_empty(iter)) {
3846
3847 if ((filp->f_flags & O_NONBLOCK)) {
3848 return -EAGAIN;
3849 }
3850
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003851 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003852
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003853 iter->trace->wait_pipe(iter);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003854
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003855 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003856
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01003857 if (signal_pending(current))
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003858 return -EINTR;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003859
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003860 /*
Liu Bo250bfd32013-01-14 10:54:11 +08003861 * We block until we read something and tracing is disabled.
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003862 * We still block if tracing is disabled, but we have never
3863 * read anything. This allows a user to cat this file, and
3864 * then enable tracing. But after we have read something,
3865 * we give an EOF when tracing is again disabled.
3866 *
3867 * iter->pos will be 0 if we haven't read anything.
3868 */
Liu Bo250bfd32013-01-14 10:54:11 +08003869 if (!tracing_is_enabled() && iter->pos)
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003870 break;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003871 }
3872
3873 return 1;
3874}
3875
Steven Rostedtb3806b42008-05-12 21:20:46 +02003876/*
3877 * Consumer reader.
3878 */
3879static ssize_t
3880tracing_read_pipe(struct file *filp, char __user *ubuf,
3881 size_t cnt, loff_t *ppos)
3882{
3883 struct trace_iterator *iter = filp->private_data;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003884 struct trace_array *tr = iter->tr;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003885 ssize_t sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003886
3887 /* return any leftover data */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003888 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3889 if (sret != -EBUSY)
3890 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003891
Steven Rostedtf9520752009-03-02 14:04:40 -05003892 trace_seq_init(&iter->seq);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003893
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003894 /* copy the tracer to avoid using a global lock all around */
Steven Rostedt107bad82008-05-12 21:21:01 +02003895 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003896 if (unlikely(iter->trace->name != tr->current_trace->name))
3897 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003898 mutex_unlock(&trace_types_lock);
3899
3900 /*
3901 * Avoid more than one consumer on a single file descriptor
3902 * This is just a matter of traces coherency, the ring buffer itself
3903 * is protected.
3904 */
3905 mutex_lock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003906 if (iter->trace->read) {
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003907 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3908 if (sret)
Steven Rostedt107bad82008-05-12 21:21:01 +02003909 goto out;
Steven Rostedt107bad82008-05-12 21:21:01 +02003910 }
3911
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003912waitagain:
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003913 sret = tracing_wait_pipe(filp);
3914 if (sret <= 0)
3915 goto out;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003916
3917 /* stop when tracing is finished */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003918 if (trace_empty(iter)) {
3919 sret = 0;
Steven Rostedt107bad82008-05-12 21:21:01 +02003920 goto out;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02003921 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02003922
3923 if (cnt >= PAGE_SIZE)
3924 cnt = PAGE_SIZE - 1;
3925
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003926 /* reset all but tr, trace, and overruns */
Steven Rostedt53d0aa72008-05-12 21:21:01 +02003927 memset(&iter->seq, 0,
3928 sizeof(struct trace_iterator) -
3929 offsetof(struct trace_iterator, seq));
Steven Rostedt4823ed72008-05-12 21:21:01 +02003930 iter->pos = -1;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003931
Lai Jiangshan4f535962009-05-18 19:35:34 +08003932 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003933 trace_access_lock(iter->cpu_file);
Jason Wessel955b61e2010-08-05 09:22:23 -05003934 while (trace_find_next_entry_inc(iter) != NULL) {
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003935 enum print_line_t ret;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003936 int len = iter->seq.len;
3937
Ingo Molnarf9896bf2008-05-12 21:20:47 +02003938 ret = print_trace_line(iter);
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02003939 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt088b1e422008-05-12 21:20:48 +02003940 /* don't print partial lines */
3941 iter->seq.len = len;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003942 break;
Steven Rostedt088b1e422008-05-12 21:20:48 +02003943 }
Frederic Weisbeckerb91facc2009-02-06 18:30:44 +01003944 if (ret != TRACE_TYPE_NO_CONSUME)
3945 trace_consume(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003946
3947 if (iter->seq.len >= cnt)
3948 break;
Jiri Olsaee5e51f2011-03-25 12:05:18 +01003949
3950 /*
3951 * Setting the full flag means we reached the trace_seq buffer
3952 * size and we should leave by partial output condition above.
3953 * One of the trace_seq_* functions is not used properly.
3954 */
3955 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
3956 iter->ent->type);
Steven Rostedtb3806b42008-05-12 21:20:46 +02003957 }
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08003958 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08003959 trace_event_read_unlock();
Steven Rostedtb3806b42008-05-12 21:20:46 +02003960
Steven Rostedtb3806b42008-05-12 21:20:46 +02003961 /* Now copy what we have to the user */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003962 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3963 if (iter->seq.readpos >= iter->seq.len)
Steven Rostedtf9520752009-03-02 14:04:40 -05003964 trace_seq_init(&iter->seq);
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003965
3966 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003967 * If there was nothing to send to user, in spite of consuming trace
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003968 * entries, go back to wait for more entries.
3969 */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003970 if (sret == -EBUSY)
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02003971 goto waitagain;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003972
Steven Rostedt107bad82008-05-12 21:21:01 +02003973out:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003974 mutex_unlock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02003975
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02003976 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02003977}
3978
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003979static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
3980 struct pipe_buffer *buf)
3981{
3982 __free_page(buf->page);
3983}
3984
3985static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
3986 unsigned int idx)
3987{
3988 __free_page(spd->pages[idx]);
3989}
3990
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08003991static const struct pipe_buf_operations tracing_pipe_buf_ops = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05003992 .can_merge = 0,
3993 .map = generic_pipe_buf_map,
3994 .unmap = generic_pipe_buf_unmap,
3995 .confirm = generic_pipe_buf_confirm,
3996 .release = tracing_pipe_buf_release,
3997 .steal = generic_pipe_buf_steal,
3998 .get = generic_pipe_buf_get,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02003999};
4000
Steven Rostedt34cd4992009-02-09 12:06:29 -05004001static size_t
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01004002tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004003{
4004 size_t count;
4005 int ret;
4006
4007 /* Seq buffer is page-sized, exactly what we need. */
4008 for (;;) {
4009 count = iter->seq.len;
4010 ret = print_trace_line(iter);
4011 count = iter->seq.len - count;
4012 if (rem < count) {
4013 rem = 0;
4014 iter->seq.len -= count;
4015 break;
4016 }
4017 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4018 iter->seq.len -= count;
4019 break;
4020 }
4021
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08004022 if (ret != TRACE_TYPE_NO_CONSUME)
4023 trace_consume(iter);
Steven Rostedt34cd4992009-02-09 12:06:29 -05004024 rem -= count;
Jason Wessel955b61e2010-08-05 09:22:23 -05004025 if (!trace_find_next_entry_inc(iter)) {
Steven Rostedt34cd4992009-02-09 12:06:29 -05004026 rem = 0;
4027 iter->ent = NULL;
4028 break;
4029 }
4030 }
4031
4032 return rem;
4033}
4034
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004035static ssize_t tracing_splice_read_pipe(struct file *filp,
4036 loff_t *ppos,
4037 struct pipe_inode_info *pipe,
4038 size_t len,
4039 unsigned int flags)
4040{
Jens Axboe35f3d142010-05-20 10:43:18 +02004041 struct page *pages_def[PIPE_DEF_BUFFERS];
4042 struct partial_page partial_def[PIPE_DEF_BUFFERS];
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004043 struct trace_iterator *iter = filp->private_data;
4044 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02004045 .pages = pages_def,
4046 .partial = partial_def,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004047 .nr_pages = 0, /* This gets updated below. */
Eric Dumazet047fe362012-06-12 15:24:40 +02004048 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004049 .flags = flags,
4050 .ops = &tracing_pipe_buf_ops,
4051 .spd_release = tracing_spd_release_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004052 };
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004053 struct trace_array *tr = iter->tr;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004054 ssize_t ret;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004055 size_t rem;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004056 unsigned int i;
4057
Jens Axboe35f3d142010-05-20 10:43:18 +02004058 if (splice_grow_spd(pipe, &spd))
4059 return -ENOMEM;
4060
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004061 /* copy the tracer to avoid using a global lock all around */
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004062 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004063 if (unlikely(iter->trace->name != tr->current_trace->name))
4064 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004065 mutex_unlock(&trace_types_lock);
4066
4067 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004068
4069 if (iter->trace->splice_read) {
4070 ret = iter->trace->splice_read(iter, filp,
4071 ppos, pipe, len, flags);
4072 if (ret)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004073 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004074 }
4075
4076 ret = tracing_wait_pipe(filp);
4077 if (ret <= 0)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004078 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004079
Jason Wessel955b61e2010-08-05 09:22:23 -05004080 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004081 ret = -EFAULT;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004082 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004083 }
4084
Lai Jiangshan4f535962009-05-18 19:35:34 +08004085 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004086 trace_access_lock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004087
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004088 /* Fill as many pages as possible. */
Jens Axboe35f3d142010-05-20 10:43:18 +02004089 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4090 spd.pages[i] = alloc_page(GFP_KERNEL);
4091 if (!spd.pages[i])
Steven Rostedt34cd4992009-02-09 12:06:29 -05004092 break;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004093
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01004094 rem = tracing_fill_pipe_page(rem, iter);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004095
4096 /* Copy the data into the page, so we can start over. */
4097 ret = trace_seq_to_buffer(&iter->seq,
Jens Axboe35f3d142010-05-20 10:43:18 +02004098 page_address(spd.pages[i]),
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004099 iter->seq.len);
4100 if (ret < 0) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004101 __free_page(spd.pages[i]);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004102 break;
4103 }
Jens Axboe35f3d142010-05-20 10:43:18 +02004104 spd.partial[i].offset = 0;
4105 spd.partial[i].len = iter->seq.len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004106
Steven Rostedtf9520752009-03-02 14:04:40 -05004107 trace_seq_init(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004108 }
4109
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004110 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004111 trace_event_read_unlock();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004112 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004113
4114 spd.nr_pages = i;
4115
Jens Axboe35f3d142010-05-20 10:43:18 +02004116 ret = splice_to_pipe(pipe, &spd);
4117out:
Eric Dumazet047fe362012-06-12 15:24:40 +02004118 splice_shrink_spd(&spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02004119 return ret;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004120
Steven Rostedt34cd4992009-02-09 12:06:29 -05004121out_err:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004122 mutex_unlock(&iter->mutex);
Jens Axboe35f3d142010-05-20 10:43:18 +02004123 goto out;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004124}
4125
Steven Rostedta98a3c32008-05-12 21:20:59 +02004126static ssize_t
4127tracing_entries_read(struct file *filp, char __user *ubuf,
4128 size_t cnt, loff_t *ppos)
4129{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004130 struct trace_cpu *tc = filp->private_data;
4131 struct trace_array *tr = tc->tr;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004132 char buf[64];
4133 int r = 0;
4134 ssize_t ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004135
Steven Rostedtdb526ca2009-03-12 13:53:25 -04004136 mutex_lock(&trace_types_lock);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004137
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004138 if (tc->cpu == RING_BUFFER_ALL_CPUS) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004139 int cpu, buf_size_same;
4140 unsigned long size;
4141
4142 size = 0;
4143 buf_size_same = 1;
4144 /* check if all cpu sizes are same */
4145 for_each_tracing_cpu(cpu) {
4146 /* fill in the size from first enabled cpu */
4147 if (size == 0)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004148 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4149 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004150 buf_size_same = 0;
4151 break;
4152 }
4153 }
4154
4155 if (buf_size_same) {
4156 if (!ring_buffer_expanded)
4157 r = sprintf(buf, "%lu (expanded: %lu)\n",
4158 size >> 10,
4159 trace_buf_size >> 10);
4160 else
4161 r = sprintf(buf, "%lu\n", size >> 10);
4162 } else
4163 r = sprintf(buf, "X\n");
4164 } else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004165 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, tc->cpu)->entries >> 10);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004166
Steven Rostedtdb526ca2009-03-12 13:53:25 -04004167 mutex_unlock(&trace_types_lock);
4168
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004169 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4170 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004171}
4172
4173static ssize_t
4174tracing_entries_write(struct file *filp, const char __user *ubuf,
4175 size_t cnt, loff_t *ppos)
4176{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004177 struct trace_cpu *tc = filp->private_data;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004178 unsigned long val;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004179 int ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004180
Peter Huewe22fe9b52011-06-07 21:58:27 +02004181 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4182 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02004183 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004184
4185 /* must have at least 1 entry */
4186 if (!val)
4187 return -EINVAL;
4188
Steven Rostedt1696b2b2008-11-13 00:09:35 -05004189 /* value is in KB */
4190 val <<= 10;
4191
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004192 ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004193 if (ret < 0)
4194 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004195
Jiri Olsacf8517c2009-10-23 19:36:16 -04004196 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004197
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004198 return cnt;
4199}
Steven Rostedtbf5e6512008-11-10 21:46:00 -05004200
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004201static ssize_t
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004202tracing_total_entries_read(struct file *filp, char __user *ubuf,
4203 size_t cnt, loff_t *ppos)
4204{
4205 struct trace_array *tr = filp->private_data;
4206 char buf[64];
4207 int r, cpu;
4208 unsigned long size = 0, expanded_size = 0;
4209
4210 mutex_lock(&trace_types_lock);
4211 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004212 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004213 if (!ring_buffer_expanded)
4214 expanded_size += trace_buf_size >> 10;
4215 }
4216 if (ring_buffer_expanded)
4217 r = sprintf(buf, "%lu\n", size);
4218 else
4219 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4220 mutex_unlock(&trace_types_lock);
4221
4222 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4223}
4224
4225static ssize_t
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004226tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4227 size_t cnt, loff_t *ppos)
4228{
4229 /*
4230 * There is no need to read what the user has written, this function
4231 * is just to make sure that there is no error when "echo" is used
4232 */
4233
4234 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004235
4236 return cnt;
4237}
4238
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004239static int
4240tracing_free_buffer_release(struct inode *inode, struct file *filp)
4241{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004242 struct trace_array *tr = inode->i_private;
4243
Steven Rostedtcf30cf62011-06-14 22:44:07 -04004244 /* disable tracing ? */
4245 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4246 tracing_off();
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004247 /* resize the ring buffer to 0 */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004248 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004249
4250 return 0;
4251}
4252
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004253static ssize_t
4254tracing_mark_write(struct file *filp, const char __user *ubuf,
4255 size_t cnt, loff_t *fpos)
4256{
Steven Rostedtd696b582011-09-22 11:50:27 -04004257 unsigned long addr = (unsigned long)ubuf;
4258 struct ring_buffer_event *event;
4259 struct ring_buffer *buffer;
4260 struct print_entry *entry;
4261 unsigned long irq_flags;
4262 struct page *pages[2];
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004263 void *map_page[2];
Steven Rostedtd696b582011-09-22 11:50:27 -04004264 int nr_pages = 1;
4265 ssize_t written;
Steven Rostedtd696b582011-09-22 11:50:27 -04004266 int offset;
4267 int size;
4268 int len;
4269 int ret;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004270 int i;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004271
Steven Rostedtc76f0692008-11-07 22:36:02 -05004272 if (tracing_disabled)
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004273 return -EINVAL;
4274
Mandeep Singh Baines5224c3a2012-09-07 18:12:19 -07004275 if (!(trace_flags & TRACE_ITER_MARKERS))
4276 return -EINVAL;
4277
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004278 if (cnt > TRACE_BUF_SIZE)
4279 cnt = TRACE_BUF_SIZE;
4280
Steven Rostedtd696b582011-09-22 11:50:27 -04004281 /*
4282 * Userspace is injecting traces into the kernel trace buffer.
4283 * We want to be as non intrusive as possible.
4284 * To do so, we do not want to allocate any special buffers
4285 * or take any locks, but instead write the userspace data
4286 * straight into the ring buffer.
4287 *
4288 * First we need to pin the userspace buffer into memory,
4289 * which, most likely it is, because it just referenced it.
4290 * But there's no guarantee that it is. By using get_user_pages_fast()
4291 * and kmap_atomic/kunmap_atomic() we can get access to the
4292 * pages directly. We then write the data directly into the
4293 * ring buffer.
4294 */
4295 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004296
Steven Rostedtd696b582011-09-22 11:50:27 -04004297 /* check if we cross pages */
4298 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4299 nr_pages = 2;
4300
4301 offset = addr & (PAGE_SIZE - 1);
4302 addr &= PAGE_MASK;
4303
4304 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4305 if (ret < nr_pages) {
4306 while (--ret >= 0)
4307 put_page(pages[ret]);
4308 written = -EFAULT;
4309 goto out;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004310 }
4311
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004312 for (i = 0; i < nr_pages; i++)
4313 map_page[i] = kmap_atomic(pages[i]);
Steven Rostedtd696b582011-09-22 11:50:27 -04004314
4315 local_save_flags(irq_flags);
4316 size = sizeof(*entry) + cnt + 2; /* possible \n added */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004317 buffer = global_trace.trace_buffer.buffer;
Steven Rostedtd696b582011-09-22 11:50:27 -04004318 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4319 irq_flags, preempt_count());
4320 if (!event) {
4321 /* Ring buffer disabled, return as if not open for write */
4322 written = -EBADF;
4323 goto out_unlock;
4324 }
4325
4326 entry = ring_buffer_event_data(event);
4327 entry->ip = _THIS_IP_;
4328
4329 if (nr_pages == 2) {
4330 len = PAGE_SIZE - offset;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004331 memcpy(&entry->buf, map_page[0] + offset, len);
4332 memcpy(&entry->buf[len], map_page[1], cnt - len);
Steven Rostedtd696b582011-09-22 11:50:27 -04004333 } else
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004334 memcpy(&entry->buf, map_page[0] + offset, cnt);
Steven Rostedtd696b582011-09-22 11:50:27 -04004335
4336 if (entry->buf[cnt - 1] != '\n') {
4337 entry->buf[cnt] = '\n';
4338 entry->buf[cnt + 1] = '\0';
4339 } else
4340 entry->buf[cnt] = '\0';
4341
Steven Rostedt7ffbd482012-10-11 12:14:25 -04004342 __buffer_unlock_commit(buffer, event);
Steven Rostedtd696b582011-09-22 11:50:27 -04004343
4344 written = cnt;
4345
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02004346 *fpos += written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004347
Steven Rostedtd696b582011-09-22 11:50:27 -04004348 out_unlock:
Steven Rostedt6edb2a82012-05-11 23:28:49 -04004349 for (i = 0; i < nr_pages; i++){
4350 kunmap_atomic(map_page[i]);
4351 put_page(pages[i]);
4352 }
Steven Rostedtd696b582011-09-22 11:50:27 -04004353 out:
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02004354 return written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004355}
4356
Li Zefan13f16d22009-12-08 11:16:11 +08004357static int tracing_clock_show(struct seq_file *m, void *v)
Zhaolei5079f322009-08-25 16:12:56 +08004358{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004359 struct trace_array *tr = m->private;
Zhaolei5079f322009-08-25 16:12:56 +08004360 int i;
4361
4362 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
Li Zefan13f16d22009-12-08 11:16:11 +08004363 seq_printf(m,
Zhaolei5079f322009-08-25 16:12:56 +08004364 "%s%s%s%s", i ? " " : "",
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004365 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4366 i == tr->clock_id ? "]" : "");
Li Zefan13f16d22009-12-08 11:16:11 +08004367 seq_putc(m, '\n');
Zhaolei5079f322009-08-25 16:12:56 +08004368
Li Zefan13f16d22009-12-08 11:16:11 +08004369 return 0;
Zhaolei5079f322009-08-25 16:12:56 +08004370}
4371
4372static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4373 size_t cnt, loff_t *fpos)
4374{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004375 struct seq_file *m = filp->private_data;
4376 struct trace_array *tr = m->private;
Zhaolei5079f322009-08-25 16:12:56 +08004377 char buf[64];
4378 const char *clockstr;
4379 int i;
4380
4381 if (cnt >= sizeof(buf))
4382 return -EINVAL;
4383
4384 if (copy_from_user(&buf, ubuf, cnt))
4385 return -EFAULT;
4386
4387 buf[cnt] = 0;
4388
4389 clockstr = strstrip(buf);
4390
4391 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4392 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4393 break;
4394 }
4395 if (i == ARRAY_SIZE(trace_clocks))
4396 return -EINVAL;
4397
Zhaolei5079f322009-08-25 16:12:56 +08004398 mutex_lock(&trace_types_lock);
4399
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004400 tr->clock_id = i;
4401
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004402 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
Zhaolei5079f322009-08-25 16:12:56 +08004403
David Sharp60303ed2012-10-11 16:27:52 -07004404 /*
4405 * New clock may not be consistent with the previous clock.
4406 * Reset the buffer so that it doesn't have incomparable timestamps.
4407 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004408 tracing_reset_online_cpus(&global_trace.trace_buffer);
4409
4410#ifdef CONFIG_TRACER_MAX_TRACE
4411 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4412 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4413 tracing_reset_online_cpus(&global_trace.max_buffer);
4414#endif
David Sharp60303ed2012-10-11 16:27:52 -07004415
Zhaolei5079f322009-08-25 16:12:56 +08004416 mutex_unlock(&trace_types_lock);
4417
4418 *fpos += cnt;
4419
4420 return cnt;
4421}
4422
Li Zefan13f16d22009-12-08 11:16:11 +08004423static int tracing_clock_open(struct inode *inode, struct file *file)
4424{
4425 if (tracing_disabled)
4426 return -ENODEV;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004427
4428 return single_open(file, tracing_clock_show, inode->i_private);
Li Zefan13f16d22009-12-08 11:16:11 +08004429}
4430
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004431struct ftrace_buffer_info {
4432 struct trace_iterator iter;
4433 void *spare;
4434 unsigned int read;
4435};
4436
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004437#ifdef CONFIG_TRACER_SNAPSHOT
4438static int tracing_snapshot_open(struct inode *inode, struct file *file)
4439{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004440 struct trace_cpu *tc = inode->i_private;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004441 struct trace_iterator *iter;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004442 struct seq_file *m;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004443 int ret = 0;
4444
4445 if (file->f_mode & FMODE_READ) {
4446 iter = __tracing_open(inode, file, true);
4447 if (IS_ERR(iter))
4448 ret = PTR_ERR(iter);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004449 } else {
4450 /* Writes still need the seq_file to hold the private data */
4451 m = kzalloc(sizeof(*m), GFP_KERNEL);
4452 if (!m)
4453 return -ENOMEM;
4454 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4455 if (!iter) {
4456 kfree(m);
4457 return -ENOMEM;
4458 }
4459 iter->tr = tc->tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004460 iter->trace_buffer = &tc->tr->max_buffer;
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004461 iter->cpu_file = tc->cpu;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004462 m->private = iter;
4463 file->private_data = m;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004464 }
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004465
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004466 return ret;
4467}
4468
4469static ssize_t
4470tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4471 loff_t *ppos)
4472{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004473 struct seq_file *m = filp->private_data;
4474 struct trace_iterator *iter = m->private;
4475 struct trace_array *tr = iter->tr;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004476 unsigned long val;
4477 int ret;
4478
4479 ret = tracing_update_buffers();
4480 if (ret < 0)
4481 return ret;
4482
4483 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4484 if (ret)
4485 return ret;
4486
4487 mutex_lock(&trace_types_lock);
4488
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004489 if (tr->current_trace->use_max_tr) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004490 ret = -EBUSY;
4491 goto out;
4492 }
4493
4494 switch (val) {
4495 case 0:
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004496 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4497 ret = -EINVAL;
4498 break;
4499 }
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04004500 if (tr->allocated_snapshot)
4501 free_snapshot(tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004502 break;
4503 case 1:
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004504/* Only allow per-cpu swap if the ring buffer supports it */
4505#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4506 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4507 ret = -EINVAL;
4508 break;
4509 }
4510#endif
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05004511 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04004512 ret = alloc_snapshot(tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004513 if (ret < 0)
4514 break;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004515 }
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004516 local_irq_disable();
4517 /* Now, we're going to swap */
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004518 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05004519 update_max_tr(tr, current, smp_processor_id());
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004520 else
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05004521 update_max_tr_single(tr, current, iter->cpu_file);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004522 local_irq_enable();
4523 break;
4524 default:
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05004525 if (tr->allocated_snapshot) {
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05004526 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4527 tracing_reset_online_cpus(&tr->max_buffer);
4528 else
4529 tracing_reset(&tr->max_buffer, iter->cpu_file);
4530 }
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004531 break;
4532 }
4533
4534 if (ret >= 0) {
4535 *ppos += cnt;
4536 ret = cnt;
4537 }
4538out:
4539 mutex_unlock(&trace_types_lock);
4540 return ret;
4541}
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004542
4543static int tracing_snapshot_release(struct inode *inode, struct file *file)
4544{
4545 struct seq_file *m = file->private_data;
4546
4547 if (file->f_mode & FMODE_READ)
4548 return tracing_release(inode, file);
4549
4550 /* If write only, the seq_file is just a stub */
4551 if (m)
4552 kfree(m->private);
4553 kfree(m);
4554
4555 return 0;
4556}
4557
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004558static int tracing_buffers_open(struct inode *inode, struct file *filp);
4559static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4560 size_t count, loff_t *ppos);
4561static int tracing_buffers_release(struct inode *inode, struct file *file);
4562static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4563 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4564
4565static int snapshot_raw_open(struct inode *inode, struct file *filp)
4566{
4567 struct ftrace_buffer_info *info;
4568 int ret;
4569
4570 ret = tracing_buffers_open(inode, filp);
4571 if (ret < 0)
4572 return ret;
4573
4574 info = filp->private_data;
4575
4576 if (info->iter.trace->use_max_tr) {
4577 tracing_buffers_release(inode, filp);
4578 return -EBUSY;
4579 }
4580
4581 info->iter.snapshot = true;
4582 info->iter.trace_buffer = &info->iter.tr->max_buffer;
4583
4584 return ret;
4585}
4586
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004587#endif /* CONFIG_TRACER_SNAPSHOT */
4588
4589
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004590static const struct file_operations tracing_max_lat_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004591 .open = tracing_open_generic,
4592 .read = tracing_max_lat_read,
4593 .write = tracing_max_lat_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004594 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004595};
4596
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004597static const struct file_operations set_tracer_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004598 .open = tracing_open_generic,
4599 .read = tracing_set_trace_read,
4600 .write = tracing_set_trace_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004601 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004602};
4603
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004604static const struct file_operations tracing_pipe_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004605 .open = tracing_open_pipe,
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004606 .poll = tracing_poll_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004607 .read = tracing_read_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004608 .splice_read = tracing_splice_read_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004609 .release = tracing_release_pipe,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004610 .llseek = no_llseek,
Steven Rostedtb3806b42008-05-12 21:20:46 +02004611};
4612
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004613static const struct file_operations tracing_entries_fops = {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004614 .open = tracing_open_generic,
Steven Rostedta98a3c32008-05-12 21:20:59 +02004615 .read = tracing_entries_read,
4616 .write = tracing_entries_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004617 .llseek = generic_file_llseek,
Steven Rostedta98a3c32008-05-12 21:20:59 +02004618};
4619
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07004620static const struct file_operations tracing_total_entries_fops = {
4621 .open = tracing_open_generic,
4622 .read = tracing_total_entries_read,
4623 .llseek = generic_file_llseek,
4624};
4625
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004626static const struct file_operations tracing_free_buffer_fops = {
4627 .write = tracing_free_buffer_write,
4628 .release = tracing_free_buffer_release,
4629};
4630
Steven Rostedt5e2336a2009-03-05 21:44:55 -05004631static const struct file_operations tracing_mark_fops = {
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02004632 .open = tracing_open_generic,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004633 .write = tracing_mark_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02004634 .llseek = generic_file_llseek,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03004635};
4636
Zhaolei5079f322009-08-25 16:12:56 +08004637static const struct file_operations trace_clock_fops = {
Li Zefan13f16d22009-12-08 11:16:11 +08004638 .open = tracing_clock_open,
4639 .read = seq_read,
4640 .llseek = seq_lseek,
4641 .release = single_release,
Zhaolei5079f322009-08-25 16:12:56 +08004642 .write = tracing_clock_write,
4643};
4644
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004645#ifdef CONFIG_TRACER_SNAPSHOT
4646static const struct file_operations snapshot_fops = {
4647 .open = tracing_snapshot_open,
4648 .read = seq_read,
4649 .write = tracing_snapshot_write,
4650 .llseek = tracing_seek,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004651 .release = tracing_snapshot_release,
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004652};
Hiraku Toyookadebdd572012-12-26 11:53:00 +09004653
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004654static const struct file_operations snapshot_raw_fops = {
4655 .open = snapshot_raw_open,
4656 .read = tracing_buffers_read,
4657 .release = tracing_buffers_release,
4658 .splice_read = tracing_buffers_splice_read,
4659 .llseek = no_llseek,
Steven Rostedt2cadf912008-12-01 22:20:19 -05004660};
4661
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004662#endif /* CONFIG_TRACER_SNAPSHOT */
4663
Steven Rostedt2cadf912008-12-01 22:20:19 -05004664static int tracing_buffers_open(struct inode *inode, struct file *filp)
4665{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004666 struct trace_cpu *tc = inode->i_private;
4667 struct trace_array *tr = tc->tr;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004668 struct ftrace_buffer_info *info;
4669
4670 if (tracing_disabled)
4671 return -ENODEV;
4672
4673 info = kzalloc(sizeof(*info), GFP_KERNEL);
4674 if (!info)
4675 return -ENOMEM;
4676
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05004677 mutex_lock(&trace_types_lock);
4678
4679 tr->ref++;
4680
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004681 info->iter.tr = tr;
4682 info->iter.cpu_file = tc->cpu;
Steven Rostedtb6273442013-02-28 13:44:11 -05004683 info->iter.trace = tr->current_trace;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004684 info->iter.trace_buffer = &tr->trace_buffer;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004685 info->spare = NULL;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004686 /* Force reading ring buffer for first read */
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004687 info->read = (unsigned int)-1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004688
4689 filp->private_data = info;
4690
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05004691 mutex_unlock(&trace_types_lock);
4692
Lai Jiangshand1e7e022009-04-02 15:16:56 +08004693 return nonseekable_open(inode, filp);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004694}
4695
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004696static unsigned int
4697tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4698{
4699 struct ftrace_buffer_info *info = filp->private_data;
4700 struct trace_iterator *iter = &info->iter;
4701
4702 return trace_poll(iter, filp, poll_table);
4703}
4704
Steven Rostedt2cadf912008-12-01 22:20:19 -05004705static ssize_t
4706tracing_buffers_read(struct file *filp, char __user *ubuf,
4707 size_t count, loff_t *ppos)
4708{
4709 struct ftrace_buffer_info *info = filp->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004710 struct trace_iterator *iter = &info->iter;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004711 ssize_t ret;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004712 ssize_t size;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004713
Steven Rostedt2dc5d122009-03-04 19:10:05 -05004714 if (!count)
4715 return 0;
4716
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004717 mutex_lock(&trace_types_lock);
4718
4719#ifdef CONFIG_TRACER_MAX_TRACE
4720 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4721 size = -EBUSY;
4722 goto out_unlock;
4723 }
4724#endif
4725
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004726 if (!info->spare)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004727 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4728 iter->cpu_file);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004729 size = -ENOMEM;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004730 if (!info->spare)
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004731 goto out_unlock;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004732
Steven Rostedt2cadf912008-12-01 22:20:19 -05004733 /* Do we have previous read data to read? */
4734 if (info->read < PAGE_SIZE)
4735 goto read;
4736
Steven Rostedtb6273442013-02-28 13:44:11 -05004737 again:
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004738 trace_access_lock(iter->cpu_file);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004739 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
Steven Rostedt2cadf912008-12-01 22:20:19 -05004740 &info->spare,
4741 count,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004742 iter->cpu_file, 0);
4743 trace_access_unlock(iter->cpu_file);
Steven Rostedtb6273442013-02-28 13:44:11 -05004744
4745 if (ret < 0) {
4746 if (trace_empty(iter)) {
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004747 if ((filp->f_flags & O_NONBLOCK)) {
4748 size = -EAGAIN;
4749 goto out_unlock;
4750 }
4751 mutex_unlock(&trace_types_lock);
Steven Rostedtb6273442013-02-28 13:44:11 -05004752 iter->trace->wait_pipe(iter);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004753 mutex_lock(&trace_types_lock);
4754 if (signal_pending(current)) {
4755 size = -EINTR;
4756 goto out_unlock;
4757 }
Steven Rostedtb6273442013-02-28 13:44:11 -05004758 goto again;
4759 }
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004760 size = 0;
4761 goto out_unlock;
Steven Rostedtb6273442013-02-28 13:44:11 -05004762 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05004763
Steven Rostedt436fc282011-10-14 10:44:25 -04004764 info->read = 0;
Steven Rostedtb6273442013-02-28 13:44:11 -05004765 read:
Steven Rostedt2cadf912008-12-01 22:20:19 -05004766 size = PAGE_SIZE - info->read;
4767 if (size > count)
4768 size = count;
4769
4770 ret = copy_to_user(ubuf, info->spare + info->read, size);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004771 if (ret == size) {
4772 size = -EFAULT;
4773 goto out_unlock;
4774 }
Steven Rostedt2dc5d122009-03-04 19:10:05 -05004775 size -= ret;
4776
Steven Rostedt2cadf912008-12-01 22:20:19 -05004777 *ppos += size;
4778 info->read += size;
4779
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004780 out_unlock:
4781 mutex_unlock(&trace_types_lock);
4782
Steven Rostedt2cadf912008-12-01 22:20:19 -05004783 return size;
4784}
4785
4786static int tracing_buffers_release(struct inode *inode, struct file *file)
4787{
4788 struct ftrace_buffer_info *info = file->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004789 struct trace_iterator *iter = &info->iter;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004790
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05004791 mutex_lock(&trace_types_lock);
4792
4793 WARN_ON(!iter->tr->ref);
4794 iter->tr->ref--;
4795
Lai Jiangshanddd538f2009-04-02 15:16:59 +08004796 if (info->spare)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004797 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004798 kfree(info);
4799
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05004800 mutex_unlock(&trace_types_lock);
4801
Steven Rostedt2cadf912008-12-01 22:20:19 -05004802 return 0;
4803}
4804
4805struct buffer_ref {
4806 struct ring_buffer *buffer;
4807 void *page;
4808 int ref;
4809};
4810
4811static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4812 struct pipe_buffer *buf)
4813{
4814 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4815
4816 if (--ref->ref)
4817 return;
4818
4819 ring_buffer_free_read_page(ref->buffer, ref->page);
4820 kfree(ref);
4821 buf->private = 0;
4822}
4823
Steven Rostedt2cadf912008-12-01 22:20:19 -05004824static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4825 struct pipe_buffer *buf)
4826{
4827 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4828
4829 ref->ref++;
4830}
4831
4832/* Pipe buffer operations for a buffer. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08004833static const struct pipe_buf_operations buffer_pipe_buf_ops = {
Steven Rostedt2cadf912008-12-01 22:20:19 -05004834 .can_merge = 0,
4835 .map = generic_pipe_buf_map,
4836 .unmap = generic_pipe_buf_unmap,
4837 .confirm = generic_pipe_buf_confirm,
4838 .release = buffer_pipe_buf_release,
Masami Hiramatsud55cb6c2012-08-09 21:31:10 +09004839 .steal = generic_pipe_buf_steal,
Steven Rostedt2cadf912008-12-01 22:20:19 -05004840 .get = buffer_pipe_buf_get,
4841};
4842
4843/*
4844 * Callback from splice_to_pipe(), if we need to release some pages
4845 * at the end of the spd in case we error'ed out in filling the pipe.
4846 */
4847static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4848{
4849 struct buffer_ref *ref =
4850 (struct buffer_ref *)spd->partial[i].private;
4851
4852 if (--ref->ref)
4853 return;
4854
4855 ring_buffer_free_read_page(ref->buffer, ref->page);
4856 kfree(ref);
4857 spd->partial[i].private = 0;
4858}
4859
4860static ssize_t
4861tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4862 struct pipe_inode_info *pipe, size_t len,
4863 unsigned int flags)
4864{
4865 struct ftrace_buffer_info *info = file->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004866 struct trace_iterator *iter = &info->iter;
Jens Axboe35f3d142010-05-20 10:43:18 +02004867 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4868 struct page *pages_def[PIPE_DEF_BUFFERS];
Steven Rostedt2cadf912008-12-01 22:20:19 -05004869 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02004870 .pages = pages_def,
4871 .partial = partial_def,
Eric Dumazet047fe362012-06-12 15:24:40 +02004872 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt2cadf912008-12-01 22:20:19 -05004873 .flags = flags,
4874 .ops = &buffer_pipe_buf_ops,
4875 .spd_release = buffer_spd_release,
4876 };
4877 struct buffer_ref *ref;
Steven Rostedt93459c62009-04-29 00:23:13 -04004878 int entries, size, i;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004879 ssize_t ret;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004880
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004881 mutex_lock(&trace_types_lock);
4882
4883#ifdef CONFIG_TRACER_MAX_TRACE
4884 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4885 ret = -EBUSY;
4886 goto out;
4887 }
4888#endif
4889
4890 if (splice_grow_spd(pipe, &spd)) {
4891 ret = -ENOMEM;
4892 goto out;
4893 }
Jens Axboe35f3d142010-05-20 10:43:18 +02004894
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004895 if (*ppos & (PAGE_SIZE - 1)) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004896 ret = -EINVAL;
4897 goto out;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004898 }
4899
4900 if (len & (PAGE_SIZE - 1)) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004901 if (len < PAGE_SIZE) {
4902 ret = -EINVAL;
4903 goto out;
4904 }
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004905 len &= PAGE_MASK;
4906 }
4907
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004908 again:
4909 trace_access_lock(iter->cpu_file);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004910 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt93459c62009-04-29 00:23:13 -04004911
Jens Axboe35f3d142010-05-20 10:43:18 +02004912 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05004913 struct page *page;
4914 int r;
4915
4916 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4917 if (!ref)
4918 break;
4919
Steven Rostedt7267fa62009-04-29 00:16:21 -04004920 ref->ref = 1;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004921 ref->buffer = iter->trace_buffer->buffer;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004922 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004923 if (!ref->page) {
4924 kfree(ref);
4925 break;
4926 }
4927
4928 r = ring_buffer_read_page(ref->buffer, &ref->page,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004929 len, iter->cpu_file, 1);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004930 if (r < 0) {
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07004931 ring_buffer_free_read_page(ref->buffer, ref->page);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004932 kfree(ref);
4933 break;
4934 }
4935
4936 /*
4937 * zero out any left over data, this is going to
4938 * user land.
4939 */
4940 size = ring_buffer_page_len(ref->page);
4941 if (size < PAGE_SIZE)
4942 memset(ref->page + size, 0, PAGE_SIZE - size);
4943
4944 page = virt_to_page(ref->page);
4945
4946 spd.pages[i] = page;
4947 spd.partial[i].len = PAGE_SIZE;
4948 spd.partial[i].offset = 0;
4949 spd.partial[i].private = (unsigned long)ref;
4950 spd.nr_pages++;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08004951 *ppos += PAGE_SIZE;
Steven Rostedt93459c62009-04-29 00:23:13 -04004952
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004953 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004954 }
4955
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004956 trace_access_unlock(iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05004957 spd.nr_pages = i;
4958
4959 /* did we read anything? */
4960 if (!spd.nr_pages) {
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004961 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05004962 ret = -EAGAIN;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004963 goto out;
4964 }
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004965 mutex_unlock(&trace_types_lock);
Steven Rostedtb6273442013-02-28 13:44:11 -05004966 iter->trace->wait_pipe(iter);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004967 mutex_lock(&trace_types_lock);
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004968 if (signal_pending(current)) {
4969 ret = -EINTR;
4970 goto out;
4971 }
4972 goto again;
Steven Rostedt2cadf912008-12-01 22:20:19 -05004973 }
4974
4975 ret = splice_to_pipe(pipe, &spd);
Eric Dumazet047fe362012-06-12 15:24:40 +02004976 splice_shrink_spd(&spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02004977out:
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05004978 mutex_unlock(&trace_types_lock);
4979
Steven Rostedt2cadf912008-12-01 22:20:19 -05004980 return ret;
4981}
4982
4983static const struct file_operations tracing_buffers_fops = {
4984 .open = tracing_buffers_open,
4985 .read = tracing_buffers_read,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004986 .poll = tracing_buffers_poll,
Steven Rostedt2cadf912008-12-01 22:20:19 -05004987 .release = tracing_buffers_release,
4988 .splice_read = tracing_buffers_splice_read,
4989 .llseek = no_llseek,
4990};
4991
Steven Rostedtc8d77182009-04-29 18:03:45 -04004992static ssize_t
4993tracing_stats_read(struct file *filp, char __user *ubuf,
4994 size_t count, loff_t *ppos)
4995{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004996 struct trace_cpu *tc = filp->private_data;
4997 struct trace_array *tr = tc->tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004998 struct trace_buffer *trace_buf = &tr->trace_buffer;
Steven Rostedtc8d77182009-04-29 18:03:45 -04004999 struct trace_seq *s;
5000 unsigned long cnt;
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005001 unsigned long long t;
5002 unsigned long usec_rem;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005003 int cpu = tc->cpu;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005004
Li Zefane4f2d102009-06-15 10:57:28 +08005005 s = kmalloc(sizeof(*s), GFP_KERNEL);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005006 if (!s)
Roel Kluina6463652009-11-11 22:26:35 +01005007 return -ENOMEM;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005008
5009 trace_seq_init(s);
5010
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005011 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005012 trace_seq_printf(s, "entries: %ld\n", cnt);
5013
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005014 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005015 trace_seq_printf(s, "overrun: %ld\n", cnt);
5016
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005017 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005018 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5019
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005020 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005021 trace_seq_printf(s, "bytes: %ld\n", cnt);
5022
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005023 if (trace_clocks[trace_clock_id].in_ns) {
5024 /* local or global for trace_clock */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005025 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005026 usec_rem = do_div(t, USEC_PER_SEC);
5027 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5028 t, usec_rem);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005029
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005030 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005031 usec_rem = do_div(t, USEC_PER_SEC);
5032 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5033 } else {
5034 /* counter or tsc mode for trace_clock */
5035 trace_seq_printf(s, "oldest event ts: %llu\n",
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005036 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005037
5038 trace_seq_printf(s, "now ts: %llu\n",
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005039 ring_buffer_time_stamp(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005040 }
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005041
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005042 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
Slava Pestov884bfe82011-07-15 14:23:58 -07005043 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5044
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005045 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
Steven Rostedt (Red Hat)ad964702013-01-29 17:45:49 -05005046 trace_seq_printf(s, "read events: %ld\n", cnt);
5047
Steven Rostedtc8d77182009-04-29 18:03:45 -04005048 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5049
5050 kfree(s);
5051
5052 return count;
5053}
5054
5055static const struct file_operations tracing_stats_fops = {
5056 .open = tracing_open_generic,
5057 .read = tracing_stats_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005058 .llseek = generic_file_llseek,
Steven Rostedtc8d77182009-04-29 18:03:45 -04005059};
5060
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005061#ifdef CONFIG_DYNAMIC_FTRACE
5062
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005063int __weak ftrace_arch_read_dyn_info(char *buf, int size)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005064{
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005065 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005066}
5067
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005068static ssize_t
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005069tracing_read_dyn_info(struct file *filp, char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005070 size_t cnt, loff_t *ppos)
5071{
Steven Rostedta26a2a22008-10-31 00:03:22 -04005072 static char ftrace_dyn_info_buffer[1024];
5073 static DEFINE_MUTEX(dyn_info_mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005074 unsigned long *p = filp->private_data;
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005075 char *buf = ftrace_dyn_info_buffer;
Steven Rostedta26a2a22008-10-31 00:03:22 -04005076 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005077 int r;
5078
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005079 mutex_lock(&dyn_info_mutex);
5080 r = sprintf(buf, "%ld ", *p);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005081
Steven Rostedta26a2a22008-10-31 00:03:22 -04005082 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005083 buf[r++] = '\n';
5084
5085 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5086
5087 mutex_unlock(&dyn_info_mutex);
5088
5089 return r;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005090}
5091
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005092static const struct file_operations tracing_dyn_info_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005093 .open = tracing_open_generic,
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005094 .read = tracing_read_dyn_info,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005095 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005096};
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005097#endif /* CONFIG_DYNAMIC_FTRACE */
5098
5099#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5100static void
5101ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5102{
5103 tracing_snapshot();
5104}
5105
5106static void
5107ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5108{
5109 unsigned long *count = (long *)data;
5110
5111 if (!*count)
5112 return;
5113
5114 if (*count != -1)
5115 (*count)--;
5116
5117 tracing_snapshot();
5118}
5119
5120static int
5121ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5122 struct ftrace_probe_ops *ops, void *data)
5123{
5124 long count = (long)data;
5125
5126 seq_printf(m, "%ps:", (void *)ip);
5127
5128 seq_printf(m, "snapshot");
5129
5130 if (count == -1)
5131 seq_printf(m, ":unlimited\n");
5132 else
5133 seq_printf(m, ":count=%ld\n", count);
5134
5135 return 0;
5136}
5137
5138static struct ftrace_probe_ops snapshot_probe_ops = {
5139 .func = ftrace_snapshot,
5140 .print = ftrace_snapshot_print,
5141};
5142
5143static struct ftrace_probe_ops snapshot_count_probe_ops = {
5144 .func = ftrace_count_snapshot,
5145 .print = ftrace_snapshot_print,
5146};
5147
5148static int
5149ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5150 char *glob, char *cmd, char *param, int enable)
5151{
5152 struct ftrace_probe_ops *ops;
5153 void *count = (void *)-1;
5154 char *number;
5155 int ret;
5156
5157 /* hash funcs only work with set_ftrace_filter */
5158 if (!enable)
5159 return -EINVAL;
5160
5161 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5162
5163 if (glob[0] == '!') {
5164 unregister_ftrace_function_probe_func(glob+1, ops);
5165 return 0;
5166 }
5167
5168 if (!param)
5169 goto out_reg;
5170
5171 number = strsep(&param, ":");
5172
5173 if (!strlen(number))
5174 goto out_reg;
5175
5176 /*
5177 * We use the callback data field (which is a pointer)
5178 * as our counter.
5179 */
5180 ret = kstrtoul(number, 0, (unsigned long *)&count);
5181 if (ret)
5182 return ret;
5183
5184 out_reg:
5185 ret = register_ftrace_function_probe(glob, ops, count);
5186
5187 if (ret >= 0)
5188 alloc_snapshot(&global_trace);
5189
5190 return ret < 0 ? ret : 0;
5191}
5192
5193static struct ftrace_func_command ftrace_snapshot_cmd = {
5194 .name = "snapshot",
5195 .func = ftrace_trace_snapshot_callback,
5196};
5197
5198static int register_snapshot_cmd(void)
5199{
5200 return register_ftrace_command(&ftrace_snapshot_cmd);
5201}
5202#else
5203static inline int register_snapshot_cmd(void) { return 0; }
5204#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005205
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005206struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005207{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005208 if (tr->dir)
5209 return tr->dir;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005210
Frederic Weisbecker3e1f60b2009-03-22 23:10:45 +01005211 if (!debugfs_initialized())
5212 return NULL;
5213
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005214 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5215 tr->dir = debugfs_create_dir("tracing", NULL);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005216
zhangwei(Jovi)687c8782013-03-11 15:13:29 +08005217 if (!tr->dir)
5218 pr_warn_once("Could not create debugfs directory 'tracing'\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005219
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005220 return tr->dir;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005221}
5222
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005223struct dentry *tracing_init_dentry(void)
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005224{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005225 return tracing_init_dentry_tr(&global_trace);
5226}
5227
5228static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5229{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005230 struct dentry *d_tracer;
5231
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005232 if (tr->percpu_dir)
5233 return tr->percpu_dir;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005234
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005235 d_tracer = tracing_init_dentry_tr(tr);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005236 if (!d_tracer)
5237 return NULL;
5238
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005239 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005240
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005241 WARN_ONCE(!tr->percpu_dir,
5242 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005243
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005244 return tr->percpu_dir;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005245}
5246
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005247static void
5248tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005249{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005250 struct trace_array_cpu *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005251 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005252 struct dentry *d_cpu;
Steven Rostedtdd49a382010-10-20 21:51:26 -04005253 char cpu_dir[30]; /* 30 characters should be more than enough */
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005254
Namhyung Kim0a3d7ce2012-04-23 10:11:57 +09005255 if (!d_percpu)
5256 return;
5257
Steven Rostedtdd49a382010-10-20 21:51:26 -04005258 snprintf(cpu_dir, 30, "cpu%ld", cpu);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01005259 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5260 if (!d_cpu) {
5261 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5262 return;
5263 }
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005264
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01005265 /* per cpu trace_pipe */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005266 trace_create_file("trace_pipe", 0444, d_cpu,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005267 (void *)&data->trace_cpu, &tracing_pipe_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005268
5269 /* per cpu trace */
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005270 trace_create_file("trace", 0644, d_cpu,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005271 (void *)&data->trace_cpu, &tracing_fops);
Steven Rostedt7f96f932009-03-13 00:37:42 -04005272
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005273 trace_create_file("trace_pipe_raw", 0444, d_cpu,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005274 (void *)&data->trace_cpu, &tracing_buffers_fops);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005275
5276 trace_create_file("stats", 0444, d_cpu,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005277 (void *)&data->trace_cpu, &tracing_stats_fops);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08005278
5279 trace_create_file("buffer_size_kb", 0444, d_cpu,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005280 (void *)&data->trace_cpu, &tracing_entries_fops);
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005281
5282#ifdef CONFIG_TRACER_SNAPSHOT
5283 trace_create_file("snapshot", 0644, d_cpu,
5284 (void *)&data->trace_cpu, &snapshot_fops);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005285
5286 trace_create_file("snapshot_raw", 0444, d_cpu,
5287 (void *)&data->trace_cpu, &snapshot_raw_fops);
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005288#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005289}
5290
Steven Rostedt60a11772008-05-12 21:20:44 +02005291#ifdef CONFIG_FTRACE_SELFTEST
5292/* Let selftest have access to static functions in this file */
5293#include "trace_selftest.c"
5294#endif
5295
Steven Rostedt577b7852009-02-26 23:43:05 -05005296struct trace_option_dentry {
5297 struct tracer_opt *opt;
5298 struct tracer_flags *flags;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005299 struct trace_array *tr;
Steven Rostedt577b7852009-02-26 23:43:05 -05005300 struct dentry *entry;
5301};
5302
5303static ssize_t
5304trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5305 loff_t *ppos)
5306{
5307 struct trace_option_dentry *topt = filp->private_data;
5308 char *buf;
5309
5310 if (topt->flags->val & topt->opt->bit)
5311 buf = "1\n";
5312 else
5313 buf = "0\n";
5314
5315 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5316}
5317
5318static ssize_t
5319trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5320 loff_t *ppos)
5321{
5322 struct trace_option_dentry *topt = filp->private_data;
5323 unsigned long val;
Steven Rostedt577b7852009-02-26 23:43:05 -05005324 int ret;
5325
Peter Huewe22fe9b52011-06-07 21:58:27 +02005326 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5327 if (ret)
Steven Rostedt577b7852009-02-26 23:43:05 -05005328 return ret;
5329
Li Zefan8d18eaa2009-12-08 11:17:06 +08005330 if (val != 0 && val != 1)
Steven Rostedt577b7852009-02-26 23:43:05 -05005331 return -EINVAL;
Li Zefan8d18eaa2009-12-08 11:17:06 +08005332
5333 if (!!(topt->flags->val & topt->opt->bit) != val) {
5334 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005335 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
Steven Rostedtc757bea2009-12-21 22:35:16 -05005336 topt->opt, !val);
Li Zefan8d18eaa2009-12-08 11:17:06 +08005337 mutex_unlock(&trace_types_lock);
5338 if (ret)
5339 return ret;
Steven Rostedt577b7852009-02-26 23:43:05 -05005340 }
5341
5342 *ppos += cnt;
5343
5344 return cnt;
5345}
5346
5347
5348static const struct file_operations trace_options_fops = {
5349 .open = tracing_open_generic,
5350 .read = trace_options_read,
5351 .write = trace_options_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005352 .llseek = generic_file_llseek,
Steven Rostedt577b7852009-02-26 23:43:05 -05005353};
5354
Steven Rostedta8259072009-02-26 22:19:12 -05005355static ssize_t
5356trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5357 loff_t *ppos)
5358{
5359 long index = (long)filp->private_data;
5360 char *buf;
5361
5362 if (trace_flags & (1 << index))
5363 buf = "1\n";
5364 else
5365 buf = "0\n";
5366
5367 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5368}
5369
5370static ssize_t
5371trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5372 loff_t *ppos)
5373{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005374 struct trace_array *tr = &global_trace;
Steven Rostedta8259072009-02-26 22:19:12 -05005375 long index = (long)filp->private_data;
Steven Rostedta8259072009-02-26 22:19:12 -05005376 unsigned long val;
5377 int ret;
5378
Peter Huewe22fe9b52011-06-07 21:58:27 +02005379 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5380 if (ret)
Steven Rostedta8259072009-02-26 22:19:12 -05005381 return ret;
5382
Zhaoleif2d84b62009-08-07 18:55:48 +08005383 if (val != 0 && val != 1)
Steven Rostedta8259072009-02-26 22:19:12 -05005384 return -EINVAL;
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04005385
5386 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005387 ret = set_tracer_flag(tr, 1 << index, val);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04005388 mutex_unlock(&trace_types_lock);
Steven Rostedta8259072009-02-26 22:19:12 -05005389
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04005390 if (ret < 0)
5391 return ret;
5392
Steven Rostedta8259072009-02-26 22:19:12 -05005393 *ppos += cnt;
5394
5395 return cnt;
5396}
5397
Steven Rostedta8259072009-02-26 22:19:12 -05005398static const struct file_operations trace_options_core_fops = {
5399 .open = tracing_open_generic,
5400 .read = trace_options_core_read,
5401 .write = trace_options_core_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005402 .llseek = generic_file_llseek,
Steven Rostedta8259072009-02-26 22:19:12 -05005403};
5404
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005405struct dentry *trace_create_file(const char *name,
Al Virof4ae40a2011-07-24 04:33:43 -04005406 umode_t mode,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005407 struct dentry *parent,
5408 void *data,
5409 const struct file_operations *fops)
5410{
5411 struct dentry *ret;
5412
5413 ret = debugfs_create_file(name, mode, parent, data, fops);
5414 if (!ret)
5415 pr_warning("Could not create debugfs '%s' entry\n", name);
5416
5417 return ret;
5418}
5419
5420
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005421static struct dentry *trace_options_init_dentry(struct trace_array *tr)
Steven Rostedta8259072009-02-26 22:19:12 -05005422{
5423 struct dentry *d_tracer;
Steven Rostedta8259072009-02-26 22:19:12 -05005424
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005425 if (tr->options)
5426 return tr->options;
Steven Rostedta8259072009-02-26 22:19:12 -05005427
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005428 d_tracer = tracing_init_dentry_tr(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05005429 if (!d_tracer)
5430 return NULL;
5431
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005432 tr->options = debugfs_create_dir("options", d_tracer);
5433 if (!tr->options) {
Steven Rostedta8259072009-02-26 22:19:12 -05005434 pr_warning("Could not create debugfs directory 'options'\n");
5435 return NULL;
5436 }
5437
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005438 return tr->options;
Steven Rostedta8259072009-02-26 22:19:12 -05005439}
5440
Steven Rostedt577b7852009-02-26 23:43:05 -05005441static void
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005442create_trace_option_file(struct trace_array *tr,
5443 struct trace_option_dentry *topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05005444 struct tracer_flags *flags,
5445 struct tracer_opt *opt)
5446{
5447 struct dentry *t_options;
Steven Rostedt577b7852009-02-26 23:43:05 -05005448
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005449 t_options = trace_options_init_dentry(tr);
Steven Rostedt577b7852009-02-26 23:43:05 -05005450 if (!t_options)
5451 return;
5452
5453 topt->flags = flags;
5454 topt->opt = opt;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005455 topt->tr = tr;
Steven Rostedt577b7852009-02-26 23:43:05 -05005456
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005457 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05005458 &trace_options_fops);
5459
Steven Rostedt577b7852009-02-26 23:43:05 -05005460}
5461
5462static struct trace_option_dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005463create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
Steven Rostedt577b7852009-02-26 23:43:05 -05005464{
5465 struct trace_option_dentry *topts;
5466 struct tracer_flags *flags;
5467 struct tracer_opt *opts;
5468 int cnt;
5469
5470 if (!tracer)
5471 return NULL;
5472
5473 flags = tracer->flags;
5474
5475 if (!flags || !flags->opts)
5476 return NULL;
5477
5478 opts = flags->opts;
5479
5480 for (cnt = 0; opts[cnt].name; cnt++)
5481 ;
5482
Steven Rostedt0cfe8242009-02-27 10:51:10 -05005483 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
Steven Rostedt577b7852009-02-26 23:43:05 -05005484 if (!topts)
5485 return NULL;
5486
5487 for (cnt = 0; opts[cnt].name; cnt++)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005488 create_trace_option_file(tr, &topts[cnt], flags,
Steven Rostedt577b7852009-02-26 23:43:05 -05005489 &opts[cnt]);
5490
5491 return topts;
5492}
5493
5494static void
5495destroy_trace_option_files(struct trace_option_dentry *topts)
5496{
5497 int cnt;
5498
5499 if (!topts)
5500 return;
5501
5502 for (cnt = 0; topts[cnt].opt; cnt++) {
5503 if (topts[cnt].entry)
5504 debugfs_remove(topts[cnt].entry);
5505 }
5506
5507 kfree(topts);
5508}
5509
Steven Rostedta8259072009-02-26 22:19:12 -05005510static struct dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005511create_trace_option_core_file(struct trace_array *tr,
5512 const char *option, long index)
Steven Rostedta8259072009-02-26 22:19:12 -05005513{
5514 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05005515
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005516 t_options = trace_options_init_dentry(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05005517 if (!t_options)
5518 return NULL;
5519
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005520 return trace_create_file(option, 0644, t_options, (void *)index,
Steven Rostedta8259072009-02-26 22:19:12 -05005521 &trace_options_core_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05005522}
5523
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005524static __init void create_trace_options_dir(struct trace_array *tr)
Steven Rostedta8259072009-02-26 22:19:12 -05005525{
5526 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05005527 int i;
5528
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005529 t_options = trace_options_init_dentry(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05005530 if (!t_options)
5531 return;
5532
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005533 for (i = 0; trace_options[i]; i++)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005534 create_trace_option_core_file(tr, trace_options[i], i);
Steven Rostedta8259072009-02-26 22:19:12 -05005535}
5536
Steven Rostedt499e5472012-02-22 15:50:28 -05005537static ssize_t
5538rb_simple_read(struct file *filp, char __user *ubuf,
5539 size_t cnt, loff_t *ppos)
5540{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04005541 struct trace_array *tr = filp->private_data;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005542 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt499e5472012-02-22 15:50:28 -05005543 char buf[64];
5544 int r;
5545
5546 if (buffer)
5547 r = ring_buffer_record_is_on(buffer);
5548 else
5549 r = 0;
5550
5551 r = sprintf(buf, "%d\n", r);
5552
5553 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5554}
5555
5556static ssize_t
5557rb_simple_write(struct file *filp, const char __user *ubuf,
5558 size_t cnt, loff_t *ppos)
5559{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04005560 struct trace_array *tr = filp->private_data;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005561 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt499e5472012-02-22 15:50:28 -05005562 unsigned long val;
5563 int ret;
5564
5565 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5566 if (ret)
5567 return ret;
5568
5569 if (buffer) {
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05005570 mutex_lock(&trace_types_lock);
5571 if (val) {
Steven Rostedt499e5472012-02-22 15:50:28 -05005572 ring_buffer_record_on(buffer);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005573 if (tr->current_trace->start)
5574 tr->current_trace->start(tr);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05005575 } else {
Steven Rostedt499e5472012-02-22 15:50:28 -05005576 ring_buffer_record_off(buffer);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005577 if (tr->current_trace->stop)
5578 tr->current_trace->stop(tr);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05005579 }
5580 mutex_unlock(&trace_types_lock);
Steven Rostedt499e5472012-02-22 15:50:28 -05005581 }
5582
5583 (*ppos)++;
5584
5585 return cnt;
5586}
5587
5588static const struct file_operations rb_simple_fops = {
5589 .open = tracing_open_generic,
5590 .read = rb_simple_read,
5591 .write = rb_simple_write,
5592 .llseek = default_llseek,
5593};
5594
Steven Rostedt277ba042012-08-03 16:10:49 -04005595struct dentry *trace_instance_dir;
5596
5597static void
5598init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5599
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005600static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
5601{
5602 int cpu;
5603
5604 for_each_tracing_cpu(cpu) {
5605 memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
5606 per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
5607 per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
5608 }
5609}
5610
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005611static int
5612allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
Steven Rostedt277ba042012-08-03 16:10:49 -04005613{
5614 enum ring_buffer_flags rb_flags;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005615
5616 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5617
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005618 buf->buffer = ring_buffer_alloc(size, rb_flags);
5619 if (!buf->buffer)
5620 return -ENOMEM;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005621
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005622 buf->data = alloc_percpu(struct trace_array_cpu);
5623 if (!buf->data) {
5624 ring_buffer_free(buf->buffer);
5625 return -ENOMEM;
5626 }
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005627
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005628 init_trace_buffers(tr, buf);
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005629
5630 /* Allocate the first page for all buffers */
5631 set_buffer_entries(&tr->trace_buffer,
5632 ring_buffer_size(tr->trace_buffer.buffer, 0));
5633
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005634 return 0;
5635}
5636
5637static int allocate_trace_buffers(struct trace_array *tr, int size)
5638{
5639 int ret;
5640
5641 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5642 if (ret)
5643 return ret;
5644
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005645#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005646 ret = allocate_trace_buffer(tr, &tr->max_buffer,
5647 allocate_snapshot ? size : 1);
5648 if (WARN_ON(ret)) {
5649 ring_buffer_free(tr->trace_buffer.buffer);
5650 free_percpu(tr->trace_buffer.data);
5651 return -ENOMEM;
5652 }
5653 tr->allocated_snapshot = allocate_snapshot;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005654
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05005655 /*
5656 * Only the top level trace array gets its snapshot allocated
5657 * from the kernel command line.
5658 */
5659 allocate_snapshot = false;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005660#endif
5661 return 0;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005662}
5663
5664static int new_instance_create(const char *name)
5665{
Steven Rostedt277ba042012-08-03 16:10:49 -04005666 struct trace_array *tr;
5667 int ret;
Steven Rostedt277ba042012-08-03 16:10:49 -04005668
5669 mutex_lock(&trace_types_lock);
5670
5671 ret = -EEXIST;
5672 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5673 if (tr->name && strcmp(tr->name, name) == 0)
5674 goto out_unlock;
5675 }
5676
5677 ret = -ENOMEM;
5678 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5679 if (!tr)
5680 goto out_unlock;
5681
5682 tr->name = kstrdup(name, GFP_KERNEL);
5683 if (!tr->name)
5684 goto out_free_tr;
5685
5686 raw_spin_lock_init(&tr->start_lock);
5687
5688 tr->current_trace = &nop_trace;
5689
5690 INIT_LIST_HEAD(&tr->systems);
5691 INIT_LIST_HEAD(&tr->events);
5692
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05005693 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
Steven Rostedt277ba042012-08-03 16:10:49 -04005694 goto out_free_tr;
5695
Steven Rostedt277ba042012-08-03 16:10:49 -04005696 /* Holder for file callbacks */
5697 tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5698 tr->trace_cpu.tr = tr;
5699
5700 tr->dir = debugfs_create_dir(name, trace_instance_dir);
5701 if (!tr->dir)
5702 goto out_free_tr;
5703
5704 ret = event_trace_add_tracer(tr->dir, tr);
5705 if (ret)
5706 goto out_free_tr;
5707
5708 init_tracer_debugfs(tr, tr->dir);
5709
5710 list_add(&tr->list, &ftrace_trace_arrays);
5711
5712 mutex_unlock(&trace_types_lock);
5713
5714 return 0;
5715
5716 out_free_tr:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005717 if (tr->trace_buffer.buffer)
5718 ring_buffer_free(tr->trace_buffer.buffer);
Steven Rostedt277ba042012-08-03 16:10:49 -04005719 kfree(tr->name);
5720 kfree(tr);
5721
5722 out_unlock:
5723 mutex_unlock(&trace_types_lock);
5724
5725 return ret;
5726
5727}
5728
Steven Rostedt0c8916c2012-08-07 16:14:16 -04005729static int instance_delete(const char *name)
5730{
5731 struct trace_array *tr;
5732 int found = 0;
5733 int ret;
5734
5735 mutex_lock(&trace_types_lock);
5736
5737 ret = -ENODEV;
5738 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5739 if (tr->name && strcmp(tr->name, name) == 0) {
5740 found = 1;
5741 break;
5742 }
5743 }
5744 if (!found)
5745 goto out_unlock;
5746
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005747 ret = -EBUSY;
5748 if (tr->ref)
5749 goto out_unlock;
5750
Steven Rostedt0c8916c2012-08-07 16:14:16 -04005751 list_del(&tr->list);
5752
5753 event_trace_del_tracer(tr);
5754 debugfs_remove_recursive(tr->dir);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005755 free_percpu(tr->trace_buffer.data);
5756 ring_buffer_free(tr->trace_buffer.buffer);
Steven Rostedt0c8916c2012-08-07 16:14:16 -04005757
5758 kfree(tr->name);
5759 kfree(tr);
5760
5761 ret = 0;
5762
5763 out_unlock:
5764 mutex_unlock(&trace_types_lock);
5765
5766 return ret;
5767}
5768
Steven Rostedt277ba042012-08-03 16:10:49 -04005769static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
5770{
5771 struct dentry *parent;
5772 int ret;
5773
5774 /* Paranoid: Make sure the parent is the "instances" directory */
5775 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5776 if (WARN_ON_ONCE(parent != trace_instance_dir))
5777 return -ENOENT;
5778
5779 /*
5780 * The inode mutex is locked, but debugfs_create_dir() will also
5781 * take the mutex. As the instances directory can not be destroyed
5782 * or changed in any other way, it is safe to unlock it, and
5783 * let the dentry try. If two users try to make the same dir at
5784 * the same time, then the new_instance_create() will determine the
5785 * winner.
5786 */
5787 mutex_unlock(&inode->i_mutex);
5788
5789 ret = new_instance_create(dentry->d_iname);
5790
5791 mutex_lock(&inode->i_mutex);
5792
5793 return ret;
5794}
5795
Steven Rostedt0c8916c2012-08-07 16:14:16 -04005796static int instance_rmdir(struct inode *inode, struct dentry *dentry)
5797{
5798 struct dentry *parent;
5799 int ret;
5800
5801 /* Paranoid: Make sure the parent is the "instances" directory */
5802 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5803 if (WARN_ON_ONCE(parent != trace_instance_dir))
5804 return -ENOENT;
5805
5806 /* The caller did a dget() on dentry */
5807 mutex_unlock(&dentry->d_inode->i_mutex);
5808
5809 /*
5810 * The inode mutex is locked, but debugfs_create_dir() will also
5811 * take the mutex. As the instances directory can not be destroyed
5812 * or changed in any other way, it is safe to unlock it, and
5813 * let the dentry try. If two users try to make the same dir at
5814 * the same time, then the instance_delete() will determine the
5815 * winner.
5816 */
5817 mutex_unlock(&inode->i_mutex);
5818
5819 ret = instance_delete(dentry->d_iname);
5820
5821 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
5822 mutex_lock(&dentry->d_inode->i_mutex);
5823
5824 return ret;
5825}
5826
Steven Rostedt277ba042012-08-03 16:10:49 -04005827static const struct inode_operations instance_dir_inode_operations = {
5828 .lookup = simple_lookup,
5829 .mkdir = instance_mkdir,
Steven Rostedt0c8916c2012-08-07 16:14:16 -04005830 .rmdir = instance_rmdir,
Steven Rostedt277ba042012-08-03 16:10:49 -04005831};
5832
5833static __init void create_trace_instances(struct dentry *d_tracer)
5834{
5835 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
5836 if (WARN_ON(!trace_instance_dir))
5837 return;
5838
5839 /* Hijack the dir inode operations, to allow mkdir */
5840 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
5841}
5842
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005843static void
5844init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
5845{
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05005846 int cpu;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005847
5848 trace_create_file("trace_options", 0644, d_tracer,
5849 tr, &tracing_iter_fops);
5850
5851 trace_create_file("trace", 0644, d_tracer,
5852 (void *)&tr->trace_cpu, &tracing_fops);
5853
5854 trace_create_file("trace_pipe", 0444, d_tracer,
5855 (void *)&tr->trace_cpu, &tracing_pipe_fops);
5856
5857 trace_create_file("buffer_size_kb", 0644, d_tracer,
5858 (void *)&tr->trace_cpu, &tracing_entries_fops);
5859
5860 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
5861 tr, &tracing_total_entries_fops);
5862
5863 trace_create_file("free_buffer", 0644, d_tracer,
5864 tr, &tracing_free_buffer_fops);
5865
5866 trace_create_file("trace_marker", 0220, d_tracer,
5867 tr, &tracing_mark_fops);
5868
5869 trace_create_file("trace_clock", 0644, d_tracer, tr,
5870 &trace_clock_fops);
5871
5872 trace_create_file("tracing_on", 0644, d_tracer,
5873 tr, &rb_simple_fops);
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05005874
5875#ifdef CONFIG_TRACER_SNAPSHOT
5876 trace_create_file("snapshot", 0644, d_tracer,
5877 (void *)&tr->trace_cpu, &snapshot_fops);
5878#endif
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05005879
5880 for_each_tracing_cpu(cpu)
5881 tracing_init_debugfs_percpu(tr, cpu);
5882
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005883}
5884
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01005885static __init int tracer_init_debugfs(void)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005886{
5887 struct dentry *d_tracer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005888
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08005889 trace_access_lock_init();
5890
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005891 d_tracer = tracing_init_dentry();
5892
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005893 init_tracer_debugfs(&global_trace, d_tracer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005894
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005895 trace_create_file("tracing_cpumask", 0644, d_tracer,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005896 &global_trace, &tracing_cpumask_fops);
Ingo Molnarc7078de2008-05-12 21:20:52 +02005897
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005898 trace_create_file("available_tracers", 0444, d_tracer,
5899 &global_trace, &show_traces_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005900
Li Zefan339ae5d2009-04-17 10:34:30 +08005901 trace_create_file("current_tracer", 0644, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005902 &global_trace, &set_tracer_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005903
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04005904#ifdef CONFIG_TRACER_MAX_TRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005905 trace_create_file("tracing_max_latency", 0644, d_tracer,
5906 &tracing_max_latency, &tracing_max_lat_fops);
Tim Bird0e950172010-02-25 15:36:43 -08005907#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005908
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005909 trace_create_file("tracing_thresh", 0644, d_tracer,
5910 &tracing_thresh, &tracing_max_lat_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005911
Li Zefan339ae5d2009-04-17 10:34:30 +08005912 trace_create_file("README", 0444, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005913 NULL, &tracing_readme_fops);
Ingo Molnar7bd2f242008-05-12 21:20:45 +02005914
Avadh Patel69abe6a2009-04-10 16:04:48 -04005915 trace_create_file("saved_cmdlines", 0444, d_tracer,
5916 NULL, &tracing_saved_cmdlines_fops);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005917
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005918#ifdef CONFIG_DYNAMIC_FTRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005919 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
5920 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005921#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01005922
Steven Rostedt277ba042012-08-03 16:10:49 -04005923 create_trace_instances(d_tracer);
5924
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005925 create_trace_options_dir(&global_trace);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01005926
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01005927 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005928}
5929
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005930static int trace_panic_handler(struct notifier_block *this,
5931 unsigned long event, void *unused)
5932{
Steven Rostedt944ac422008-10-23 19:26:08 -04005933 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005934 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005935 return NOTIFY_OK;
5936}
5937
5938static struct notifier_block trace_panic_notifier = {
5939 .notifier_call = trace_panic_handler,
5940 .next = NULL,
5941 .priority = 150 /* priority: INT_MAX >= x >= 0 */
5942};
5943
5944static int trace_die_handler(struct notifier_block *self,
5945 unsigned long val,
5946 void *data)
5947{
5948 switch (val) {
5949 case DIE_OOPS:
Steven Rostedt944ac422008-10-23 19:26:08 -04005950 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02005951 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005952 break;
5953 default:
5954 break;
5955 }
5956 return NOTIFY_OK;
5957}
5958
5959static struct notifier_block trace_die_notifier = {
5960 .notifier_call = trace_die_handler,
5961 .priority = 200
5962};
5963
5964/*
5965 * printk is set to max of 1024, we really don't need it that big.
5966 * Nothing should be printing 1000 characters anyway.
5967 */
5968#define TRACE_MAX_PRINT 1000
5969
5970/*
5971 * Define here KERN_TRACE so that we have one place to modify
5972 * it if we decide to change what log level the ftrace dump
5973 * should be at.
5974 */
Steven Rostedt428aee12009-01-14 12:24:42 -05005975#define KERN_TRACE KERN_EMERG
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005976
Jason Wessel955b61e2010-08-05 09:22:23 -05005977void
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005978trace_printk_seq(struct trace_seq *s)
5979{
5980 /* Probably should print a warning here. */
5981 if (s->len >= 1000)
5982 s->len = 1000;
5983
5984 /* should be zero ended, but we are paranoid. */
5985 s->buffer[s->len] = 0;
5986
5987 printk(KERN_TRACE "%s", s->buffer);
5988
Steven Rostedtf9520752009-03-02 14:04:40 -05005989 trace_seq_init(s);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04005990}
5991
Jason Wessel955b61e2010-08-05 09:22:23 -05005992void trace_init_global_iter(struct trace_iterator *iter)
5993{
5994 iter->tr = &global_trace;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005995 iter->trace = iter->tr->current_trace;
Steven Rostedtae3b5092013-01-23 15:22:59 -05005996 iter->cpu_file = RING_BUFFER_ALL_CPUS;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005997 iter->trace_buffer = &global_trace.trace_buffer;
Jason Wessel955b61e2010-08-05 09:22:23 -05005998}
5999
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006000static void
6001__ftrace_dump(bool disable_tracing, enum ftrace_dump_mode oops_dump_mode)
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006002{
Thomas Gleixner445c8952009-12-02 19:49:50 +01006003 static arch_spinlock_t ftrace_dump_lock =
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01006004 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006005 /* use static because iter can be a bit big for the stack */
6006 static struct trace_iterator iter;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006007 unsigned int old_userobj;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006008 static int dump_ran;
Steven Rostedtd7690412008-10-01 00:29:53 -04006009 unsigned long flags;
6010 int cnt = 0, cpu;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006011
6012 /* only one dump */
Steven Rostedtcd891ae2009-04-28 11:39:34 -04006013 local_irq_save(flags);
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01006014 arch_spin_lock(&ftrace_dump_lock);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006015 if (dump_ran)
6016 goto out;
6017
6018 dump_ran = 1;
6019
Steven Rostedt0ee6b6c2009-01-14 14:50:19 -05006020 tracing_off();
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006021
Steven Rostedte0a413f2011-09-29 21:26:16 -04006022 /* Did function tracer already get disabled? */
6023 if (ftrace_is_dead()) {
6024 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6025 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6026 }
6027
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006028 if (disable_tracing)
6029 ftrace_kill();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006030
Jovi Zhang38dbe0b2013-01-25 18:03:07 +08006031 /* Simulate the iterator */
Jason Wessel955b61e2010-08-05 09:22:23 -05006032 trace_init_global_iter(&iter);
6033
Steven Rostedtd7690412008-10-01 00:29:53 -04006034 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006035 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -04006036 }
6037
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006038 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6039
Török Edwinb54d3de2008-11-22 13:28:48 +02006040 /* don't look at user memory in panic mode */
6041 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6042
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006043 switch (oops_dump_mode) {
6044 case DUMP_ALL:
Steven Rostedtae3b5092013-01-23 15:22:59 -05006045 iter.cpu_file = RING_BUFFER_ALL_CPUS;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006046 break;
6047 case DUMP_ORIG:
6048 iter.cpu_file = raw_smp_processor_id();
6049 break;
6050 case DUMP_NONE:
6051 goto out_enable;
6052 default:
6053 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
Steven Rostedtae3b5092013-01-23 15:22:59 -05006054 iter.cpu_file = RING_BUFFER_ALL_CPUS;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006055 }
6056
6057 printk(KERN_TRACE "Dumping ftrace buffer:\n");
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006058
6059 /*
6060 * We need to stop all tracing on all CPUS to read the
6061 * the next buffer. This is a bit expensive, but is
6062 * not done often. We fill all what we can read,
6063 * and then release the locks again.
6064 */
6065
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006066 while (!trace_empty(&iter)) {
6067
6068 if (!cnt)
6069 printk(KERN_TRACE "---------------------------------\n");
6070
6071 cnt++;
6072
6073 /* reset all but tr, trace, and overruns */
6074 memset(&iter.seq, 0,
6075 sizeof(struct trace_iterator) -
6076 offsetof(struct trace_iterator, seq));
6077 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6078 iter.pos = -1;
6079
Jason Wessel955b61e2010-08-05 09:22:23 -05006080 if (trace_find_next_entry_inc(&iter) != NULL) {
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08006081 int ret;
6082
6083 ret = print_trace_line(&iter);
6084 if (ret != TRACE_TYPE_NO_CONSUME)
6085 trace_consume(&iter);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006086 }
Steven Rostedtb892e5c2012-03-01 22:06:48 -05006087 touch_nmi_watchdog();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006088
6089 trace_printk_seq(&iter.seq);
6090 }
6091
6092 if (!cnt)
6093 printk(KERN_TRACE " (ftrace buffer empty)\n");
6094 else
6095 printk(KERN_TRACE "---------------------------------\n");
6096
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006097 out_enable:
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006098 /* Re-enable tracing if requested */
6099 if (!disable_tracing) {
6100 trace_flags |= old_userobj;
6101
6102 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006103 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006104 }
6105 tracing_on();
6106 }
6107
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006108 out:
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01006109 arch_spin_unlock(&ftrace_dump_lock);
Steven Rostedtcd891ae2009-04-28 11:39:34 -04006110 local_irq_restore(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006111}
6112
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006113/* By default: disable tracing after the dump */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006114void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006115{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006116 __ftrace_dump(true, oops_dump_mode);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006117}
Paul E. McKenneya8eecf22011-10-02 11:01:15 -07006118EXPORT_SYMBOL_GPL(ftrace_dump);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01006119
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006120__init static int tracer_alloc_buffers(void)
6121{
Steven Rostedt73c51622009-03-11 13:42:01 -04006122 int ring_buf_size;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306123 int ret = -ENOMEM;
6124
David Sharp750912f2010-12-08 13:46:47 -08006125
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306126 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6127 goto out;
6128
6129 if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
6130 goto out_free_buffer_mask;
6131
Steven Rostedt07d777f2011-09-22 14:01:55 -04006132 /* Only allocate trace_printk buffers if a trace_printk exists */
6133 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
Steven Rostedt81698832012-10-11 10:15:05 -04006134 /* Must be called before global_trace.buffer is allocated */
Steven Rostedt07d777f2011-09-22 14:01:55 -04006135 trace_printk_init_buffers();
6136
Steven Rostedt73c51622009-03-11 13:42:01 -04006137 /* To save memory, keep the ring buffer size to its minimum */
6138 if (ring_buffer_expanded)
6139 ring_buf_size = trace_buf_size;
6140 else
6141 ring_buf_size = 1;
6142
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306143 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6144 cpumask_copy(tracing_cpumask, cpu_all_mask);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006145
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006146 raw_spin_lock_init(&global_trace.start_lock);
6147
Steven Rostedtab464282008-05-12 21:21:00 +02006148 /* TODO: make the number of buffers hot pluggable with CPUS */
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006149 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04006150 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6151 WARN_ON(1);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306152 goto out_free_cpumask;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04006153 }
Steven Rostedta7603ff2012-08-06 16:24:11 -04006154
Steven Rostedt499e5472012-02-22 15:50:28 -05006155 if (global_trace.buffer_disabled)
6156 tracing_off();
Steven Rostedt3928a8a2008-09-29 23:02:41 -04006157
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006158 trace_init_cmdlines();
6159
Frédéric Weisbecker43a15382008-09-21 20:16:30 +02006160 register_tracer(&nop_trace);
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05006161
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006162 global_trace.current_trace = &nop_trace;
6163
Steven Rostedt60a11772008-05-12 21:20:44 +02006164 /* All seems OK, enable tracing */
6165 tracing_disabled = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04006166
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006167 atomic_notifier_chain_register(&panic_notifier_list,
6168 &trace_panic_notifier);
6169
6170 register_die_notifier(&trace_die_notifier);
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01006171
Steven Rostedtae63b312012-05-03 23:09:03 -04006172 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6173
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006174 /* Holder for file callbacks */
6175 global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6176 global_trace.trace_cpu.tr = &global_trace;
6177
Steven Rostedtae63b312012-05-03 23:09:03 -04006178 INIT_LIST_HEAD(&global_trace.systems);
6179 INIT_LIST_HEAD(&global_trace.events);
6180 list_add(&global_trace.list, &ftrace_trace_arrays);
6181
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04006182 while (trace_boot_options) {
6183 char *option;
6184
6185 option = strsep(&trace_boot_options, ",");
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006186 trace_set_options(&global_trace, option);
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04006187 }
6188
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04006189 register_snapshot_cmd();
6190
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01006191 return 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006192
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306193out_free_cpumask:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006194 free_percpu(global_trace.trace_buffer.data);
6195#ifdef CONFIG_TRACER_MAX_TRACE
6196 free_percpu(global_trace.max_buffer.data);
6197#endif
Rusty Russell9e01c1b2009-01-01 10:12:22 +10306198 free_cpumask_var(tracing_cpumask);
6199out_free_buffer_mask:
6200 free_cpumask_var(tracing_buffer_mask);
6201out:
6202 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006203}
Steven Rostedtb2821ae2009-02-02 21:38:32 -05006204
6205__init static int clear_boot_tracer(void)
6206{
6207 /*
6208 * The default tracer at boot buffer is an init section.
6209 * This function is called in lateinit. If we did not
6210 * find the boot tracer, then clear it out, to prevent
6211 * later registration from accessing the buffer that is
6212 * about to be freed.
6213 */
6214 if (!default_bootup_tracer)
6215 return 0;
6216
6217 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6218 default_bootup_tracer);
6219 default_bootup_tracer = NULL;
6220
6221 return 0;
6222}
6223
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01006224early_initcall(tracer_alloc_buffers);
6225fs_initcall(tracer_init_debugfs);
Steven Rostedtb2821ae2009-02-02 21:38:32 -05006226late_initcall(clear_boot_tracer);