blob: 2198a630ef5826a30e51a2974d472b078273f80c [file] [log] [blame]
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001/*
2 * ring buffer based function tracer
3 *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +010012 * Copyright (C) 2004 Nadia Yvette Chambers
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020013 */
Steven Rostedt2cadf912008-12-01 22:20:19 -050014#include <linux/ring_buffer.h>
Sam Ravnborg273b2812009-10-18 00:52:28 +020015#include <generated/utsrelease.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050016#include <linux/stacktrace.h>
17#include <linux/writeback.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020018#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040020#include <linux/notifier.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050021#include <linux/irqflags.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020022#include <linux/debugfs.h>
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -050023#include <linux/tracefs.h>
Steven Rostedt4c11d7a2008-05-12 21:20:43 +020024#include <linux/pagemap.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020025#include <linux/hardirq.h>
26#include <linux/linkage.h>
27#include <linux/uaccess.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050028#include <linux/kprobes.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020029#include <linux/ftrace.h>
30#include <linux/module.h>
31#include <linux/percpu.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050032#include <linux/splice.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040033#include <linux/kdebug.h>
Frederic Weisbecker5f0c6c02009-03-27 14:22:10 +010034#include <linux/string.h>
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -050035#include <linux/mount.h>
Lai Jiangshan7e53bd42010-01-06 20:08:50 +080036#include <linux/rwsem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090037#include <linux/slab.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020038#include <linux/ctype.h>
39#include <linux/init.h>
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +020040#include <linux/poll.h>
Steven Rostedtb892e5c2012-03-01 22:06:48 -050041#include <linux/nmi.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020042#include <linux/fs.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060043#include <linux/sched/rt.h>
Ingo Molnar86387f72008-05-12 21:20:51 +020044
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020045#include "trace.h"
Steven Rostedtf0868d12008-12-23 23:24:12 -050046#include "trace_output.h"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020047
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010048/*
Steven Rostedt73c51622009-03-11 13:42:01 -040049 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
51 */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -050052bool ring_buffer_expanded;
Steven Rostedt73c51622009-03-11 13:42:01 -040053
54/*
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010055 * We need to change this state when a selftest is running.
Frederic Weisbeckerff325042008-12-04 23:47:35 +010056 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
Ingo Molnar5e1607a2009-03-05 10:24:48 +010058 * insertions into the ring-buffer such as trace_printk could occurred
Frederic Weisbeckerff325042008-12-04 23:47:35 +010059 * at the same time, giving false positive or negative results.
60 */
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010061static bool __read_mostly tracing_selftest_running;
Frederic Weisbeckerff325042008-12-04 23:47:35 +010062
Steven Rostedtb2821ae2009-02-02 21:38:32 -050063/*
64 * If a tracer is running, we do not want to run SELFTEST.
65 */
Li Zefan020e5f82009-07-01 10:47:05 +080066bool __read_mostly tracing_selftest_disabled;
Steven Rostedtb2821ae2009-02-02 21:38:32 -050067
Steven Rostedt (Red Hat)0daa23022014-12-12 22:27:10 -050068/* Pipe tracepoints to printk */
69struct trace_iterator *tracepoint_print_iter;
70int tracepoint_printk;
71
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010072/* For tracers that don't implement custom flags */
73static struct tracer_opt dummy_tracer_opt[] = {
74 { }
75};
76
77static struct tracer_flags dummy_tracer_flags = {
78 .val = 0,
79 .opts = dummy_tracer_opt
80};
81
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -050082static int
83dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010084{
85 return 0;
86}
Steven Rostedt0f048702008-11-05 16:05:44 -050087
88/*
Steven Rostedt7ffbd482012-10-11 12:14:25 -040089 * To prevent the comm cache from being overwritten when no
90 * tracing is active, only save the comm when a trace event
91 * occurred.
92 */
93static DEFINE_PER_CPU(bool, trace_cmdline_save);
94
95/*
Steven Rostedt0f048702008-11-05 16:05:44 -050096 * Kill all tracing for good (never come back).
97 * It is initialized to 1 but will turn to zero if the initialization
98 * of the tracer is successful. But that is the only place that sets
99 * this back to zero.
100 */
Hannes Eder4fd27352009-02-10 19:44:12 +0100101static int tracing_disabled = 1;
Steven Rostedt0f048702008-11-05 16:05:44 -0500102
Christoph Lameter9288f992009-10-07 19:17:45 -0400103DEFINE_PER_CPU(int, ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -0400104
Jason Wessel955b61e2010-08-05 09:22:23 -0500105cpumask_var_t __read_mostly tracing_buffer_mask;
Steven Rostedtab464282008-05-12 21:21:00 +0200106
Steven Rostedt944ac422008-10-23 19:26:08 -0400107/*
108 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
109 *
110 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
111 * is set, then ftrace_dump is called. This will output the contents
112 * of the ftrace buffers to the console. This is very useful for
113 * capturing traces that lead to crashes and outputing it to a
114 * serial console.
115 *
116 * It is default off, but you can enable it with either specifying
117 * "ftrace_dump_on_oops" in the kernel command line, or setting
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200118 * /proc/sys/kernel/ftrace_dump_on_oops
119 * Set 1 if you want to dump buffers of all CPUs
120 * Set 2 if you want to dump the buffer of the CPU that triggered oops
Steven Rostedt944ac422008-10-23 19:26:08 -0400121 */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200122
123enum ftrace_dump_mode ftrace_dump_on_oops;
Steven Rostedt944ac422008-10-23 19:26:08 -0400124
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400125/* When set, tracing will stop when a WARN*() is hit */
126int __disable_trace_on_warning;
127
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -0400128#ifdef CONFIG_TRACE_ENUM_MAP_FILE
129/* Map of enums to their values, for "enum_map" file */
130struct trace_enum_map_head {
131 struct module *mod;
132 unsigned long length;
133};
134
135union trace_enum_map_item;
136
137struct trace_enum_map_tail {
138 /*
139 * "end" is first and points to NULL as it must be different
140 * than "mod" or "enum_string"
141 */
142 union trace_enum_map_item *next;
143 const char *end; /* points to NULL */
144};
145
146static DEFINE_MUTEX(trace_enum_mutex);
147
148/*
149 * The trace_enum_maps are saved in an array with two extra elements,
150 * one at the beginning, and one at the end. The beginning item contains
151 * the count of the saved maps (head.length), and the module they
152 * belong to if not built in (head.mod). The ending item contains a
153 * pointer to the next array of saved enum_map items.
154 */
155union trace_enum_map_item {
156 struct trace_enum_map map;
157 struct trace_enum_map_head head;
158 struct trace_enum_map_tail tail;
159};
160
161static union trace_enum_map_item *trace_enum_maps;
162#endif /* CONFIG_TRACE_ENUM_MAP_FILE */
163
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -0500164static int tracing_set_tracer(struct trace_array *tr, const char *buf);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500165
Li Zefanee6c2c12009-09-18 14:06:47 +0800166#define MAX_TRACER_SIZE 100
167static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500168static char *default_bootup_tracer;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100169
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500170static bool allocate_snapshot;
171
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200172static int __init set_cmdline_ftrace(char *str)
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100173{
Chen Gang67012ab2013-04-08 12:06:44 +0800174 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500175 default_bootup_tracer = bootup_tracer_buf;
Steven Rostedt73c51622009-03-11 13:42:01 -0400176 /* We are using ftrace early, expand it */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500177 ring_buffer_expanded = true;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100178 return 1;
179}
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200180__setup("ftrace=", set_cmdline_ftrace);
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100181
Steven Rostedt944ac422008-10-23 19:26:08 -0400182static int __init set_ftrace_dump_on_oops(char *str)
183{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200184 if (*str++ != '=' || !*str) {
185 ftrace_dump_on_oops = DUMP_ALL;
186 return 1;
187 }
188
189 if (!strcmp("orig_cpu", str)) {
190 ftrace_dump_on_oops = DUMP_ORIG;
191 return 1;
192 }
193
194 return 0;
Steven Rostedt944ac422008-10-23 19:26:08 -0400195}
196__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
Steven Rostedt60a11772008-05-12 21:20:44 +0200197
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400198static int __init stop_trace_on_warning(char *str)
199{
Luis Claudio R. Goncalves933ff9f2014-11-12 21:14:00 -0200200 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
201 __disable_trace_on_warning = 1;
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400202 return 1;
203}
Luis Claudio R. Goncalves933ff9f2014-11-12 21:14:00 -0200204__setup("traceoff_on_warning", stop_trace_on_warning);
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400205
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400206static int __init boot_alloc_snapshot(char *str)
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500207{
208 allocate_snapshot = true;
209 /* We also need the main ring buffer expanded */
210 ring_buffer_expanded = true;
211 return 1;
212}
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400213__setup("alloc_snapshot", boot_alloc_snapshot);
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500214
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400215
216static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400217
218static int __init set_trace_boot_options(char *str)
219{
Chen Gang67012ab2013-04-08 12:06:44 +0800220 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400221 return 0;
222}
223__setup("trace_options=", set_trace_boot_options);
224
Steven Rostedte1e232c2014-02-10 23:38:46 -0500225static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
226static char *trace_boot_clock __initdata;
227
228static int __init set_trace_boot_clock(char *str)
229{
230 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
231 trace_boot_clock = trace_boot_clock_buf;
232 return 0;
233}
234__setup("trace_clock=", set_trace_boot_clock);
235
Steven Rostedt (Red Hat)0daa23022014-12-12 22:27:10 -0500236static int __init set_tracepoint_printk(char *str)
237{
238 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
239 tracepoint_printk = 1;
240 return 1;
241}
242__setup("tp_printk", set_tracepoint_printk);
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400243
Lai Jiangshancf8e3472009-03-30 13:48:00 +0800244unsigned long long ns2usecs(cycle_t nsec)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200245{
246 nsec += 500;
247 do_div(nsec, 1000);
248 return nsec;
249}
250
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400251/* trace_flags holds trace_options default values */
252#define TRACE_DEFAULT_FLAGS \
253 (FUNCTION_DEFAULT_FLAGS | \
254 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
255 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
256 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
257 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
258
Steven Rostedt (Red Hat)16270142015-09-30 12:30:06 -0400259/* trace_options that are only supported by global_trace */
260#define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
261 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
262
263
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200264/*
265 * The global_trace is the descriptor that holds the tracing
266 * buffers for the live tracing. For each CPU, it contains
267 * a link list of pages that will store trace entries. The
268 * page descriptor of the pages in the memory is used to hold
269 * the link list by linking the lru item in the page descriptor
270 * to each of the pages in the buffer per CPU.
271 *
272 * For each active CPU there is a data field that holds the
273 * pages for the buffer for that CPU. Each CPU has the same number
274 * of pages allocated for its buffer.
275 */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400276static struct trace_array global_trace = {
277 .trace_flags = TRACE_DEFAULT_FLAGS,
278};
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200279
Steven Rostedtae63b312012-05-03 23:09:03 -0400280LIST_HEAD(ftrace_trace_arrays);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200281
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -0400282int trace_array_get(struct trace_array *this_tr)
283{
284 struct trace_array *tr;
285 int ret = -ENODEV;
286
287 mutex_lock(&trace_types_lock);
288 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
289 if (tr == this_tr) {
290 tr->ref++;
291 ret = 0;
292 break;
293 }
294 }
295 mutex_unlock(&trace_types_lock);
296
297 return ret;
298}
299
300static void __trace_array_put(struct trace_array *this_tr)
301{
302 WARN_ON(!this_tr->ref);
303 this_tr->ref--;
304}
305
306void trace_array_put(struct trace_array *this_tr)
307{
308 mutex_lock(&trace_types_lock);
309 __trace_array_put(this_tr);
310 mutex_unlock(&trace_types_lock);
311}
312
Steven Rostedt (Red Hat)7f1d2f82015-05-05 10:09:53 -0400313int filter_check_discard(struct trace_event_file *file, void *rec,
Tom Zanussif306cc82013-10-24 08:34:17 -0500314 struct ring_buffer *buffer,
315 struct ring_buffer_event *event)
Tom Zanussieb02ce02009-04-08 03:15:54 -0500316{
Steven Rostedt (Red Hat)5d6ad962015-05-13 15:12:33 -0400317 if (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
Tom Zanussif306cc82013-10-24 08:34:17 -0500318 !filter_match_preds(file->filter, rec)) {
319 ring_buffer_discard_commit(buffer, event);
320 return 1;
321 }
322
323 return 0;
Tom Zanussieb02ce02009-04-08 03:15:54 -0500324}
Tom Zanussif306cc82013-10-24 08:34:17 -0500325EXPORT_SYMBOL_GPL(filter_check_discard);
326
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -0400327int call_filter_check_discard(struct trace_event_call *call, void *rec,
Tom Zanussif306cc82013-10-24 08:34:17 -0500328 struct ring_buffer *buffer,
329 struct ring_buffer_event *event)
330{
331 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
332 !filter_match_preds(call->filter, rec)) {
333 ring_buffer_discard_commit(buffer, event);
334 return 1;
335 }
336
337 return 0;
338}
339EXPORT_SYMBOL_GPL(call_filter_check_discard);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500340
Fabian Frederickad1438a2014-04-17 21:44:42 +0200341static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
Steven Rostedt37886f62009-03-17 17:22:06 -0400342{
343 u64 ts;
344
345 /* Early boot up does not have a buffer yet */
Alexander Z Lam94571582013-08-02 18:36:16 -0700346 if (!buf->buffer)
Steven Rostedt37886f62009-03-17 17:22:06 -0400347 return trace_clock_local();
348
Alexander Z Lam94571582013-08-02 18:36:16 -0700349 ts = ring_buffer_time_stamp(buf->buffer, cpu);
350 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
Steven Rostedt37886f62009-03-17 17:22:06 -0400351
352 return ts;
353}
354
Alexander Z Lam94571582013-08-02 18:36:16 -0700355cycle_t ftrace_now(int cpu)
356{
357 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
358}
359
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400360/**
361 * tracing_is_enabled - Show if global_trace has been disabled
362 *
363 * Shows if the global trace has been enabled or not. It uses the
364 * mirror flag "buffer_disabled" to be used in fast paths such as for
365 * the irqsoff tracer. But it may be inaccurate due to races. If you
366 * need to know the accurate state, use tracing_is_on() which is a little
367 * slower, but accurate.
368 */
Steven Rostedt90369902008-11-05 16:05:44 -0500369int tracing_is_enabled(void)
370{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400371 /*
372 * For quick access (irqsoff uses this in fast path), just
373 * return the mirror variable of the state of the ring buffer.
374 * It's a little racy, but we don't really care.
375 */
376 smp_rmb();
377 return !global_trace.buffer_disabled;
Steven Rostedt90369902008-11-05 16:05:44 -0500378}
379
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200380/*
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400381 * trace_buf_size is the size in bytes that is allocated
382 * for a buffer. Note, the number of bytes is always rounded
383 * to page size.
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400384 *
385 * This number is purposely set to a low number of 16384.
386 * If the dump on oops happens, it will be much appreciated
387 * to not have to wait for all that output. Anyway this can be
388 * boot time and run time configurable.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200389 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400390#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400391
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400392static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200393
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200394/* trace_types holds a link list of available tracers. */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200395static struct tracer *trace_types __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200396
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200397/*
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200398 * trace_types_lock is used to protect the trace_types list.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200399 */
Alexander Z Lama8227412013-07-01 19:37:54 -0700400DEFINE_MUTEX(trace_types_lock);
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200401
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800402/*
403 * serialize the access of the ring buffer
404 *
405 * ring buffer serializes readers, but it is low level protection.
406 * The validity of the events (which returns by ring_buffer_peek() ..etc)
407 * are not protected by ring buffer.
408 *
409 * The content of events may become garbage if we allow other process consumes
410 * these events concurrently:
411 * A) the page of the consumed events may become a normal page
412 * (not reader page) in ring buffer, and this page will be rewrited
413 * by events producer.
414 * B) The page of the consumed events may become a page for splice_read,
415 * and this page will be returned to system.
416 *
417 * These primitives allow multi process access to different cpu ring buffer
418 * concurrently.
419 *
420 * These primitives don't distinguish read-only and read-consume access.
421 * Multi read-only access are also serialized.
422 */
423
424#ifdef CONFIG_SMP
425static DECLARE_RWSEM(all_cpu_access_lock);
426static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
427
428static inline void trace_access_lock(int cpu)
429{
Steven Rostedtae3b5092013-01-23 15:22:59 -0500430 if (cpu == RING_BUFFER_ALL_CPUS) {
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800431 /* gain it for accessing the whole ring buffer. */
432 down_write(&all_cpu_access_lock);
433 } else {
434 /* gain it for accessing a cpu ring buffer. */
435
Steven Rostedtae3b5092013-01-23 15:22:59 -0500436 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800437 down_read(&all_cpu_access_lock);
438
439 /* Secondly block other access to this @cpu ring buffer. */
440 mutex_lock(&per_cpu(cpu_access_lock, cpu));
441 }
442}
443
444static inline void trace_access_unlock(int cpu)
445{
Steven Rostedtae3b5092013-01-23 15:22:59 -0500446 if (cpu == RING_BUFFER_ALL_CPUS) {
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800447 up_write(&all_cpu_access_lock);
448 } else {
449 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
450 up_read(&all_cpu_access_lock);
451 }
452}
453
454static inline void trace_access_lock_init(void)
455{
456 int cpu;
457
458 for_each_possible_cpu(cpu)
459 mutex_init(&per_cpu(cpu_access_lock, cpu));
460}
461
462#else
463
464static DEFINE_MUTEX(access_lock);
465
466static inline void trace_access_lock(int cpu)
467{
468 (void)cpu;
469 mutex_lock(&access_lock);
470}
471
472static inline void trace_access_unlock(int cpu)
473{
474 (void)cpu;
475 mutex_unlock(&access_lock);
476}
477
478static inline void trace_access_lock_init(void)
479{
480}
481
482#endif
483
Steven Rostedt (Red Hat)d78a4612015-09-25 13:30:47 -0400484#ifdef CONFIG_STACKTRACE
485static void __ftrace_trace_stack(struct ring_buffer *buffer,
486 unsigned long flags,
487 int skip, int pc, struct pt_regs *regs);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400488static inline void ftrace_trace_stack(struct trace_array *tr,
489 struct ring_buffer *buffer,
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -0400490 unsigned long flags,
491 int skip, int pc, struct pt_regs *regs);
Steven Rostedt (Red Hat)ca475e82015-09-28 09:41:11 -0400492
Steven Rostedt (Red Hat)d78a4612015-09-25 13:30:47 -0400493#else
494static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
495 unsigned long flags,
496 int skip, int pc, struct pt_regs *regs)
497{
498}
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400499static inline void ftrace_trace_stack(struct trace_array *tr,
500 struct ring_buffer *buffer,
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -0400501 unsigned long flags,
502 int skip, int pc, struct pt_regs *regs)
Steven Rostedt (Red Hat)ca475e82015-09-28 09:41:11 -0400503{
504}
505
Steven Rostedt (Red Hat)d78a4612015-09-25 13:30:47 -0400506#endif
507
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400508static void tracer_tracing_on(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400509{
510 if (tr->trace_buffer.buffer)
511 ring_buffer_record_on(tr->trace_buffer.buffer);
512 /*
513 * This flag is looked at when buffers haven't been allocated
514 * yet, or by some tracers (like irqsoff), that just want to
515 * know if the ring buffer has been disabled, but it can handle
516 * races of where it gets disabled but we still do a record.
517 * As the check is in the fast path of the tracers, it is more
518 * important to be fast than accurate.
519 */
520 tr->buffer_disabled = 0;
521 /* Make the flag seen by readers */
522 smp_wmb();
523}
524
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200525/**
Steven Rostedt499e5472012-02-22 15:50:28 -0500526 * tracing_on - enable tracing buffers
527 *
528 * This function enables tracing buffers that may have been
529 * disabled with tracing_off.
530 */
531void tracing_on(void)
532{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400533 tracer_tracing_on(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500534}
535EXPORT_SYMBOL_GPL(tracing_on);
536
537/**
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500538 * __trace_puts - write a constant string into the trace buffer.
539 * @ip: The address of the caller
540 * @str: The constant string to write
541 * @size: The size of the string.
542 */
543int __trace_puts(unsigned long ip, const char *str, int size)
544{
545 struct ring_buffer_event *event;
546 struct ring_buffer *buffer;
547 struct print_entry *entry;
548 unsigned long irq_flags;
549 int alloc;
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800550 int pc;
551
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400552 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
zhangwei(Jovi)f0160a52013-07-18 16:31:18 +0800553 return 0;
554
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800555 pc = preempt_count();
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500556
Steven Rostedt (Red Hat)3132e102014-01-23 12:27:59 -0500557 if (unlikely(tracing_selftest_running || tracing_disabled))
558 return 0;
559
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500560 alloc = sizeof(*entry) + size + 2; /* possible \n added */
561
562 local_save_flags(irq_flags);
563 buffer = global_trace.trace_buffer.buffer;
564 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800565 irq_flags, pc);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500566 if (!event)
567 return 0;
568
569 entry = ring_buffer_event_data(event);
570 entry->ip = ip;
571
572 memcpy(&entry->buf, str, size);
573
574 /* Add a newline if necessary */
575 if (entry->buf[size - 1] != '\n') {
576 entry->buf[size] = '\n';
577 entry->buf[size + 1] = '\0';
578 } else
579 entry->buf[size] = '\0';
580
581 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400582 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500583
584 return size;
585}
586EXPORT_SYMBOL_GPL(__trace_puts);
587
588/**
589 * __trace_bputs - write the pointer to a constant string into trace buffer
590 * @ip: The address of the caller
591 * @str: The constant string to write to the buffer to
592 */
593int __trace_bputs(unsigned long ip, const char *str)
594{
595 struct ring_buffer_event *event;
596 struct ring_buffer *buffer;
597 struct bputs_entry *entry;
598 unsigned long irq_flags;
599 int size = sizeof(struct bputs_entry);
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800600 int pc;
601
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400602 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
zhangwei(Jovi)f0160a52013-07-18 16:31:18 +0800603 return 0;
604
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800605 pc = preempt_count();
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500606
Steven Rostedt (Red Hat)3132e102014-01-23 12:27:59 -0500607 if (unlikely(tracing_selftest_running || tracing_disabled))
608 return 0;
609
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500610 local_save_flags(irq_flags);
611 buffer = global_trace.trace_buffer.buffer;
612 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800613 irq_flags, pc);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500614 if (!event)
615 return 0;
616
617 entry = ring_buffer_event_data(event);
618 entry->ip = ip;
619 entry->str = str;
620
621 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400622 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500623
624 return 1;
625}
626EXPORT_SYMBOL_GPL(__trace_bputs);
627
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500628#ifdef CONFIG_TRACER_SNAPSHOT
629/**
630 * trace_snapshot - take a snapshot of the current buffer.
631 *
632 * This causes a swap between the snapshot buffer and the current live
633 * tracing buffer. You can use this to take snapshots of the live
634 * trace when some condition is triggered, but continue to trace.
635 *
636 * Note, make sure to allocate the snapshot with either
637 * a tracing_snapshot_alloc(), or by doing it manually
638 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
639 *
640 * If the snapshot buffer is not allocated, it will stop tracing.
641 * Basically making a permanent snapshot.
642 */
643void tracing_snapshot(void)
644{
645 struct trace_array *tr = &global_trace;
646 struct tracer *tracer = tr->current_trace;
647 unsigned long flags;
648
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500649 if (in_nmi()) {
650 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
651 internal_trace_puts("*** snapshot is being ignored ***\n");
652 return;
653 }
654
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500655 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)ca268da2013-03-09 00:40:58 -0500656 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
657 internal_trace_puts("*** stopping trace here! ***\n");
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500658 tracing_off();
659 return;
660 }
661
662 /* Note, snapshot can not be used when the tracer uses it */
663 if (tracer->use_max_tr) {
Steven Rostedt (Red Hat)ca268da2013-03-09 00:40:58 -0500664 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
665 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500666 return;
667 }
668
669 local_irq_save(flags);
670 update_max_tr(tr, current, smp_processor_id());
671 local_irq_restore(flags);
672}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500673EXPORT_SYMBOL_GPL(tracing_snapshot);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500674
675static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
676 struct trace_buffer *size_buf, int cpu_id);
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400677static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
678
679static int alloc_snapshot(struct trace_array *tr)
680{
681 int ret;
682
683 if (!tr->allocated_snapshot) {
684
685 /* allocate spare buffer */
686 ret = resize_buffer_duplicate_size(&tr->max_buffer,
687 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
688 if (ret < 0)
689 return ret;
690
691 tr->allocated_snapshot = true;
692 }
693
694 return 0;
695}
696
Fabian Frederickad1438a2014-04-17 21:44:42 +0200697static void free_snapshot(struct trace_array *tr)
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400698{
699 /*
700 * We don't free the ring buffer. instead, resize it because
701 * The max_tr ring buffer has some state (e.g. ring->clock) and
702 * we want preserve it.
703 */
704 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
705 set_buffer_entries(&tr->max_buffer, 1);
706 tracing_reset_online_cpus(&tr->max_buffer);
707 tr->allocated_snapshot = false;
708}
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500709
710/**
Tom Zanussi93e31ff2013-10-24 08:59:26 -0500711 * tracing_alloc_snapshot - allocate snapshot buffer.
712 *
713 * This only allocates the snapshot buffer if it isn't already
714 * allocated - it doesn't also take a snapshot.
715 *
716 * This is meant to be used in cases where the snapshot buffer needs
717 * to be set up for events that can't sleep but need to be able to
718 * trigger a snapshot.
719 */
720int tracing_alloc_snapshot(void)
721{
722 struct trace_array *tr = &global_trace;
723 int ret;
724
725 ret = alloc_snapshot(tr);
726 WARN_ON(ret < 0);
727
728 return ret;
729}
730EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
731
732/**
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500733 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
734 *
735 * This is similar to trace_snapshot(), but it will allocate the
736 * snapshot buffer if it isn't already allocated. Use this only
737 * where it is safe to sleep, as the allocation may sleep.
738 *
739 * This causes a swap between the snapshot buffer and the current live
740 * tracing buffer. You can use this to take snapshots of the live
741 * trace when some condition is triggered, but continue to trace.
742 */
743void tracing_snapshot_alloc(void)
744{
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500745 int ret;
746
Tom Zanussi93e31ff2013-10-24 08:59:26 -0500747 ret = tracing_alloc_snapshot();
748 if (ret < 0)
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400749 return;
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500750
751 tracing_snapshot();
752}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500753EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500754#else
755void tracing_snapshot(void)
756{
757 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
758}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500759EXPORT_SYMBOL_GPL(tracing_snapshot);
Tom Zanussi93e31ff2013-10-24 08:59:26 -0500760int tracing_alloc_snapshot(void)
761{
762 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
763 return -ENODEV;
764}
765EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500766void tracing_snapshot_alloc(void)
767{
768 /* Give warning */
769 tracing_snapshot();
770}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500771EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500772#endif /* CONFIG_TRACER_SNAPSHOT */
773
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400774static void tracer_tracing_off(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400775{
776 if (tr->trace_buffer.buffer)
777 ring_buffer_record_off(tr->trace_buffer.buffer);
778 /*
779 * This flag is looked at when buffers haven't been allocated
780 * yet, or by some tracers (like irqsoff), that just want to
781 * know if the ring buffer has been disabled, but it can handle
782 * races of where it gets disabled but we still do a record.
783 * As the check is in the fast path of the tracers, it is more
784 * important to be fast than accurate.
785 */
786 tr->buffer_disabled = 1;
787 /* Make the flag seen by readers */
788 smp_wmb();
789}
790
Steven Rostedt499e5472012-02-22 15:50:28 -0500791/**
792 * tracing_off - turn off tracing buffers
793 *
794 * This function stops the tracing buffers from recording data.
795 * It does not disable any overhead the tracers themselves may
796 * be causing. This function simply causes all recording to
797 * the ring buffers to fail.
798 */
799void tracing_off(void)
800{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400801 tracer_tracing_off(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500802}
803EXPORT_SYMBOL_GPL(tracing_off);
804
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400805void disable_trace_on_warning(void)
806{
807 if (__disable_trace_on_warning)
808 tracing_off();
809}
810
Steven Rostedt499e5472012-02-22 15:50:28 -0500811/**
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400812 * tracer_tracing_is_on - show real state of ring buffer enabled
813 * @tr : the trace array to know if ring buffer is enabled
814 *
815 * Shows real state of the ring buffer if it is enabled or not.
816 */
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400817static int tracer_tracing_is_on(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400818{
819 if (tr->trace_buffer.buffer)
820 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
821 return !tr->buffer_disabled;
822}
823
Steven Rostedt499e5472012-02-22 15:50:28 -0500824/**
825 * tracing_is_on - show state of ring buffers enabled
826 */
827int tracing_is_on(void)
828{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400829 return tracer_tracing_is_on(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500830}
831EXPORT_SYMBOL_GPL(tracing_is_on);
832
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400833static int __init set_buf_size(char *str)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200834{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400835 unsigned long buf_size;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200836
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200837 if (!str)
838 return 0;
Li Zefan9d612be2009-06-24 17:33:15 +0800839 buf_size = memparse(str, &str);
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200840 /* nr_entries can not be zero */
Li Zefan9d612be2009-06-24 17:33:15 +0800841 if (buf_size == 0)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200842 return 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400843 trace_buf_size = buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200844 return 1;
845}
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400846__setup("trace_buf_size=", set_buf_size);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200847
Tim Bird0e950172010-02-25 15:36:43 -0800848static int __init set_tracing_thresh(char *str)
849{
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800850 unsigned long threshold;
Tim Bird0e950172010-02-25 15:36:43 -0800851 int ret;
852
853 if (!str)
854 return 0;
Daniel Walterbcd83ea2012-09-26 22:08:38 +0200855 ret = kstrtoul(str, 0, &threshold);
Tim Bird0e950172010-02-25 15:36:43 -0800856 if (ret < 0)
857 return 0;
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800858 tracing_thresh = threshold * 1000;
Tim Bird0e950172010-02-25 15:36:43 -0800859 return 1;
860}
861__setup("tracing_thresh=", set_tracing_thresh);
862
Steven Rostedt57f50be2008-05-12 21:20:44 +0200863unsigned long nsecs_to_usecs(unsigned long nsecs)
864{
865 return nsecs / 1000;
866}
867
Steven Rostedt (Red Hat)a3418a32015-09-29 09:43:30 -0400868/*
869 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
870 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
871 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
872 * of strings in the order that the enums were defined.
873 */
874#undef C
875#define C(a, b) b
876
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200877/* These must match the bit postions in trace_iterator_flags */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200878static const char *trace_options[] = {
Steven Rostedt (Red Hat)a3418a32015-09-29 09:43:30 -0400879 TRACE_FLAGS
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200880 NULL
881};
882
Zhaolei5079f322009-08-25 16:12:56 +0800883static struct {
884 u64 (*func)(void);
885 const char *name;
David Sharp8be07092012-11-13 12:18:22 -0800886 int in_ns; /* is this clock in nanoseconds? */
Zhaolei5079f322009-08-25 16:12:56 +0800887} trace_clocks[] = {
Thomas Gleixner1b3e5c02014-07-16 21:05:25 +0000888 { trace_clock_local, "local", 1 },
889 { trace_clock_global, "global", 1 },
890 { trace_clock_counter, "counter", 0 },
Linus Torvaldse7fda6c2014-08-05 17:46:42 -0700891 { trace_clock_jiffies, "uptime", 0 },
Thomas Gleixner1b3e5c02014-07-16 21:05:25 +0000892 { trace_clock, "perf", 1 },
893 { ktime_get_mono_fast_ns, "mono", 1 },
Drew Richardsonaabfa5f2015-05-08 07:30:39 -0700894 { ktime_get_raw_fast_ns, "mono_raw", 1 },
David Sharp8cbd9cc2012-11-13 12:18:21 -0800895 ARCH_TRACE_CLOCKS
Zhaolei5079f322009-08-25 16:12:56 +0800896};
897
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200898/*
899 * trace_parser_get_init - gets the buffer for trace parser
900 */
901int trace_parser_get_init(struct trace_parser *parser, int size)
902{
903 memset(parser, 0, sizeof(*parser));
904
905 parser->buffer = kmalloc(size, GFP_KERNEL);
906 if (!parser->buffer)
907 return 1;
908
909 parser->size = size;
910 return 0;
911}
912
913/*
914 * trace_parser_put - frees the buffer for trace parser
915 */
916void trace_parser_put(struct trace_parser *parser)
917{
918 kfree(parser->buffer);
919}
920
921/*
922 * trace_get_user - reads the user input string separated by space
923 * (matched by isspace(ch))
924 *
925 * For each string found the 'struct trace_parser' is updated,
926 * and the function returns.
927 *
928 * Returns number of bytes read.
929 *
930 * See kernel/trace/trace.h for 'struct trace_parser' details.
931 */
932int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
933 size_t cnt, loff_t *ppos)
934{
935 char ch;
936 size_t read = 0;
937 ssize_t ret;
938
939 if (!*ppos)
940 trace_parser_clear(parser);
941
942 ret = get_user(ch, ubuf++);
943 if (ret)
944 goto out;
945
946 read++;
947 cnt--;
948
949 /*
950 * The parser is not finished with the last write,
951 * continue reading the user input without skipping spaces.
952 */
953 if (!parser->cont) {
954 /* skip white space */
955 while (cnt && isspace(ch)) {
956 ret = get_user(ch, ubuf++);
957 if (ret)
958 goto out;
959 read++;
960 cnt--;
961 }
962
963 /* only spaces were written */
964 if (isspace(ch)) {
965 *ppos += read;
966 ret = read;
967 goto out;
968 }
969
970 parser->idx = 0;
971 }
972
973 /* read the non-space input */
974 while (cnt && !isspace(ch)) {
Li Zefan3c235a32009-09-22 13:51:54 +0800975 if (parser->idx < parser->size - 1)
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200976 parser->buffer[parser->idx++] = ch;
977 else {
978 ret = -EINVAL;
979 goto out;
980 }
981 ret = get_user(ch, ubuf++);
982 if (ret)
983 goto out;
984 read++;
985 cnt--;
986 }
987
988 /* We either got finished input or we have to wait for another call. */
989 if (isspace(ch)) {
990 parser->buffer[parser->idx] = 0;
991 parser->cont = false;
Steven Rostedt057db842013-10-09 22:23:23 -0400992 } else if (parser->idx < parser->size - 1) {
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200993 parser->cont = true;
994 parser->buffer[parser->idx++] = ch;
Steven Rostedt057db842013-10-09 22:23:23 -0400995 } else {
996 ret = -EINVAL;
997 goto out;
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200998 }
999
1000 *ppos += read;
1001 ret = read;
1002
1003out:
1004 return ret;
1005}
1006
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04001007/* TODO add a seq_buf_to_buffer() */
Dmitri Vorobievb8b94262009-03-22 19:11:11 +02001008static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001009{
1010 int len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001011
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05001012 if (trace_seq_used(s) <= s->seq.readpos)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001013 return -EBUSY;
1014
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05001015 len = trace_seq_used(s) - s->seq.readpos;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001016 if (cnt > len)
1017 cnt = len;
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04001018 memcpy(buf, s->buffer + s->seq.readpos, cnt);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001019
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04001020 s->seq.readpos += cnt;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001021 return cnt;
1022}
1023
Tim Bird0e950172010-02-25 15:36:43 -08001024unsigned long __read_mostly tracing_thresh;
1025
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001026#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001027/*
1028 * Copy the new maximum trace into the separate maximum-trace
1029 * structure. (this way the maximum trace is permanently saved,
1030 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1031 */
1032static void
1033__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1034{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001035 struct trace_buffer *trace_buf = &tr->trace_buffer;
1036 struct trace_buffer *max_buf = &tr->max_buffer;
1037 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1038 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001039
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001040 max_buf->cpu = cpu;
1041 max_buf->time_start = data->preempt_timestamp;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001042
Steven Rostedt (Red Hat)6d9b3fa2014-01-14 11:28:38 -05001043 max_data->saved_latency = tr->max_latency;
Steven Rostedt8248ac02009-09-02 12:27:41 -04001044 max_data->critical_start = data->critical_start;
1045 max_data->critical_end = data->critical_end;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001046
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -03001047 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
Steven Rostedt8248ac02009-09-02 12:27:41 -04001048 max_data->pid = tsk->pid;
Steven Rostedt (Red Hat)f17a5192013-05-30 21:10:37 -04001049 /*
1050 * If tsk == current, then use current_uid(), as that does not use
1051 * RCU. The irq tracer can be called out of RCU scope.
1052 */
1053 if (tsk == current)
1054 max_data->uid = current_uid();
1055 else
1056 max_data->uid = task_uid(tsk);
1057
Steven Rostedt8248ac02009-09-02 12:27:41 -04001058 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1059 max_data->policy = tsk->policy;
1060 max_data->rt_priority = tsk->rt_priority;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001061
1062 /* record this tasks comm */
1063 tracing_record_cmdline(tsk);
1064}
1065
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001066/**
1067 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1068 * @tr: tracer
1069 * @tsk: the task with the latency
1070 * @cpu: The cpu that initiated the trace.
1071 *
1072 * Flip the buffers between the @tr and the max_tr and record information
1073 * about which task was the cause of this latency.
1074 */
Ingo Molnare309b412008-05-12 21:20:51 +02001075void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001076update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1077{
Steven Rostedt (Red Hat)2721e722013-03-12 11:32:32 -04001078 struct ring_buffer *buf;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001079
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001080 if (tr->stop_count)
Steven Rostedtb8de7bd12009-08-31 22:32:27 -04001081 return;
1082
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02001083 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt34600f02013-01-22 13:35:11 -05001084
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05001085 if (!tr->allocated_snapshot) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09001086 /* Only the nop tracer should hit this when disabling */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001087 WARN_ON_ONCE(tr->current_trace != &nop_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001088 return;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09001089 }
Steven Rostedt34600f02013-01-22 13:35:11 -05001090
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001091 arch_spin_lock(&tr->max_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001092
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001093 buf = tr->trace_buffer.buffer;
1094 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1095 tr->max_buffer.buffer = buf;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001096
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001097 __update_max_tr(tr, tsk, cpu);
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001098 arch_spin_unlock(&tr->max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001099}
1100
1101/**
1102 * update_max_tr_single - only copy one trace over, and reset the rest
1103 * @tr - tracer
1104 * @tsk - task with the latency
1105 * @cpu - the cpu of the buffer to copy.
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001106 *
1107 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001108 */
Ingo Molnare309b412008-05-12 21:20:51 +02001109void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001110update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1111{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001112 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001113
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001114 if (tr->stop_count)
Steven Rostedtb8de7bd12009-08-31 22:32:27 -04001115 return;
1116
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02001117 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt6c244992013-04-29 20:08:14 -04001118 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)2930e042013-03-26 17:33:00 -04001119 /* Only the nop tracer should hit this when disabling */
Linus Torvalds9e8529a2013-04-29 13:55:38 -07001120 WARN_ON_ONCE(tr->current_trace != &nop_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001121 return;
Steven Rostedt (Red Hat)2930e042013-03-26 17:33:00 -04001122 }
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001123
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001124 arch_spin_lock(&tr->max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001125
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001126 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001127
Steven Rostedte8165db2009-09-03 19:13:05 -04001128 if (ret == -EBUSY) {
1129 /*
1130 * We failed to swap the buffer due to a commit taking
1131 * place on this CPU. We fail to record, but we reset
1132 * the max trace buffer (no one writes directly to it)
1133 * and flag that it failed.
1134 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001135 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
Steven Rostedte8165db2009-09-03 19:13:05 -04001136 "Failed to swap buffers due to commit in progress\n");
1137 }
1138
Steven Rostedte8165db2009-09-03 19:13:05 -04001139 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001140
1141 __update_max_tr(tr, tsk, cpu);
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001142 arch_spin_unlock(&tr->max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001143}
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001144#endif /* CONFIG_TRACER_MAX_TRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001145
Rabin Vincente30f53a2014-11-10 19:46:34 +01001146static int wait_on_pipe(struct trace_iterator *iter, bool full)
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001147{
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05001148 /* Iterators are static, they should be filled or empty */
1149 if (trace_buffer_iter(iter, iter->cpu_file))
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04001150 return 0;
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001151
Rabin Vincente30f53a2014-11-10 19:46:34 +01001152 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1153 full);
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001154}
1155
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001156#ifdef CONFIG_FTRACE_STARTUP_TEST
1157static int run_tracer_selftest(struct tracer *type)
1158{
1159 struct trace_array *tr = &global_trace;
1160 struct tracer *saved_tracer = tr->current_trace;
1161 int ret;
1162
1163 if (!type->selftest || tracing_selftest_disabled)
1164 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001165
1166 /*
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001167 * Run a selftest on this tracer.
1168 * Here we reset the trace buffer, and set the current
1169 * tracer to be this tracer. The tracer can then run some
1170 * internal tracing to verify that everything is in order.
1171 * If we fail, we do not register this tracer.
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001172 */
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001173 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001174
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001175 tr->current_trace = type;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001176
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001177#ifdef CONFIG_TRACER_MAX_TRACE
1178 if (type->use_max_tr) {
1179 /* If we expanded the buffers, make sure the max is expanded too */
1180 if (ring_buffer_expanded)
1181 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1182 RING_BUFFER_ALL_CPUS);
1183 tr->allocated_snapshot = true;
1184 }
1185#endif
1186
1187 /* the test is responsible for initializing and enabling */
1188 pr_info("Testing tracer %s: ", type->name);
1189 ret = type->selftest(type, tr);
1190 /* the test is responsible for resetting too */
1191 tr->current_trace = saved_tracer;
1192 if (ret) {
1193 printk(KERN_CONT "FAILED!\n");
1194 /* Add the warning after printing 'FAILED' */
1195 WARN_ON(1);
1196 return -1;
1197 }
1198 /* Only reset on passing, to avoid touching corrupted buffers */
1199 tracing_reset_online_cpus(&tr->trace_buffer);
1200
1201#ifdef CONFIG_TRACER_MAX_TRACE
1202 if (type->use_max_tr) {
1203 tr->allocated_snapshot = false;
1204
1205 /* Shrink the max buffer again */
1206 if (ring_buffer_expanded)
1207 ring_buffer_resize(tr->max_buffer.buffer, 1,
1208 RING_BUFFER_ALL_CPUS);
1209 }
1210#endif
1211
1212 printk(KERN_CONT "PASSED\n");
1213 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001214}
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001215#else
1216static inline int run_tracer_selftest(struct tracer *type)
1217{
1218 return 0;
1219}
1220#endif /* CONFIG_FTRACE_STARTUP_TEST */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001221
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04001222static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1223
Jiaxing Wanga4d1e682015-11-04 09:14:29 +08001224static void __init apply_trace_boot_options(void);
1225
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001226/**
1227 * register_tracer - register a tracer with the ftrace system.
1228 * @type - the plugin for the tracer
1229 *
1230 * Register a new plugin tracer.
1231 */
Jiaxing Wanga4d1e682015-11-04 09:14:29 +08001232int __init register_tracer(struct tracer *type)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001233{
1234 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001235 int ret = 0;
1236
1237 if (!type->name) {
1238 pr_info("Tracer must have a name\n");
1239 return -1;
1240 }
1241
Dan Carpenter24a461d2010-07-10 12:06:44 +02001242 if (strlen(type->name) >= MAX_TRACER_SIZE) {
Li Zefanee6c2c12009-09-18 14:06:47 +08001243 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1244 return -1;
1245 }
1246
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001247 mutex_lock(&trace_types_lock);
Ingo Molnar86fa2f62008-11-19 10:00:15 +01001248
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +01001249 tracing_selftest_running = true;
1250
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001251 for (t = trace_types; t; t = t->next) {
1252 if (strcmp(type->name, t->name) == 0) {
1253 /* already found */
Li Zefanee6c2c12009-09-18 14:06:47 +08001254 pr_info("Tracer %s already registered\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001255 type->name);
1256 ret = -1;
1257 goto out;
1258 }
1259 }
1260
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01001261 if (!type->set_flag)
1262 type->set_flag = &dummy_set_flag;
1263 if (!type->flags)
1264 type->flags = &dummy_tracer_flags;
1265 else
1266 if (!type->flags->opts)
1267 type->flags->opts = dummy_tracer_opt;
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01001268
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001269 ret = run_tracer_selftest(type);
1270 if (ret < 0)
1271 goto out;
Steven Rostedt60a11772008-05-12 21:20:44 +02001272
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001273 type->next = trace_types;
1274 trace_types = type;
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04001275 add_tracer_options(&global_trace, type);
Steven Rostedt60a11772008-05-12 21:20:44 +02001276
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001277 out:
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +01001278 tracing_selftest_running = false;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001279 mutex_unlock(&trace_types_lock);
1280
Steven Rostedtdac74942009-02-05 01:13:38 -05001281 if (ret || !default_bootup_tracer)
1282 goto out_unlock;
Steven Rostedtb2821ae2009-02-02 21:38:32 -05001283
Li Zefanee6c2c12009-09-18 14:06:47 +08001284 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
Steven Rostedtdac74942009-02-05 01:13:38 -05001285 goto out_unlock;
1286
1287 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1288 /* Do we want this tracer to start on bootup? */
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05001289 tracing_set_tracer(&global_trace, type->name);
Steven Rostedtdac74942009-02-05 01:13:38 -05001290 default_bootup_tracer = NULL;
Jiaxing Wanga4d1e682015-11-04 09:14:29 +08001291
1292 apply_trace_boot_options();
1293
Steven Rostedtdac74942009-02-05 01:13:38 -05001294 /* disable other selftests, since this will break it. */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05001295 tracing_selftest_disabled = true;
Steven Rostedtdac74942009-02-05 01:13:38 -05001296#ifdef CONFIG_FTRACE_STARTUP_TEST
1297 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1298 type->name);
1299#endif
1300
1301 out_unlock:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001302 return ret;
1303}
1304
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001305void tracing_reset(struct trace_buffer *buf, int cpu)
Steven Rostedtf6339032009-09-04 12:35:16 -04001306{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001307 struct ring_buffer *buffer = buf->buffer;
Steven Rostedtf6339032009-09-04 12:35:16 -04001308
Hiraku Toyookaa5416412012-12-19 16:02:34 +09001309 if (!buffer)
1310 return;
1311
Steven Rostedtf6339032009-09-04 12:35:16 -04001312 ring_buffer_record_disable(buffer);
1313
1314 /* Make sure all commits have finished */
1315 synchronize_sched();
Steven Rostedt68179682012-05-08 20:57:53 -04001316 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedtf6339032009-09-04 12:35:16 -04001317
1318 ring_buffer_record_enable(buffer);
1319}
1320
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001321void tracing_reset_online_cpus(struct trace_buffer *buf)
Pekka J Enberg213cc062008-12-19 12:08:39 +02001322{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001323 struct ring_buffer *buffer = buf->buffer;
Pekka J Enberg213cc062008-12-19 12:08:39 +02001324 int cpu;
1325
Hiraku Toyookaa5416412012-12-19 16:02:34 +09001326 if (!buffer)
1327 return;
1328
Steven Rostedt621968c2009-09-04 12:02:35 -04001329 ring_buffer_record_disable(buffer);
1330
1331 /* Make sure all commits have finished */
1332 synchronize_sched();
1333
Alexander Z Lam94571582013-08-02 18:36:16 -07001334 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
Pekka J Enberg213cc062008-12-19 12:08:39 +02001335
1336 for_each_online_cpu(cpu)
Steven Rostedt68179682012-05-08 20:57:53 -04001337 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedt621968c2009-09-04 12:02:35 -04001338
1339 ring_buffer_record_enable(buffer);
Pekka J Enberg213cc062008-12-19 12:08:39 +02001340}
1341
Steven Rostedt (Red Hat)09d80912013-07-23 22:21:59 -04001342/* Must have trace_types_lock held */
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001343void tracing_reset_all_online_cpus(void)
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001344{
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001345 struct trace_array *tr;
1346
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001347 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001348 tracing_reset_online_cpus(&tr->trace_buffer);
1349#ifdef CONFIG_TRACER_MAX_TRACE
1350 tracing_reset_online_cpus(&tr->max_buffer);
1351#endif
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001352 }
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001353}
1354
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001355#define SAVED_CMDLINES_DEFAULT 128
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001356#define NO_CMDLINE_MAP UINT_MAX
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001357static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001358struct saved_cmdlines_buffer {
1359 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1360 unsigned *map_cmdline_to_pid;
1361 unsigned cmdline_num;
1362 int cmdline_idx;
1363 char *saved_cmdlines;
1364};
1365static struct saved_cmdlines_buffer *savedcmd;
Steven Rostedt25b0b442008-05-12 21:21:00 +02001366
Steven Rostedt25b0b442008-05-12 21:21:00 +02001367/* temporary disable recording */
Hannes Eder4fd27352009-02-10 19:44:12 +01001368static atomic_t trace_record_cmdline_disabled __read_mostly;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001369
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001370static inline char *get_saved_cmdlines(int idx)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001371{
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001372 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1373}
1374
1375static inline void set_cmdline(int idx, const char *cmdline)
1376{
1377 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1378}
1379
1380static int allocate_cmdlines_buffer(unsigned int val,
1381 struct saved_cmdlines_buffer *s)
1382{
1383 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1384 GFP_KERNEL);
1385 if (!s->map_cmdline_to_pid)
1386 return -ENOMEM;
1387
1388 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1389 if (!s->saved_cmdlines) {
1390 kfree(s->map_cmdline_to_pid);
1391 return -ENOMEM;
1392 }
1393
1394 s->cmdline_idx = 0;
1395 s->cmdline_num = val;
1396 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1397 sizeof(s->map_pid_to_cmdline));
1398 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1399 val * sizeof(*s->map_cmdline_to_pid));
1400
1401 return 0;
1402}
1403
1404static int trace_create_savedcmd(void)
1405{
1406 int ret;
1407
Namhyung Kima6af8fb2014-06-10 16:11:35 +09001408 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001409 if (!savedcmd)
1410 return -ENOMEM;
1411
1412 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1413 if (ret < 0) {
1414 kfree(savedcmd);
1415 savedcmd = NULL;
1416 return -ENOMEM;
1417 }
1418
1419 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001420}
1421
Carsten Emdeb5130b12009-09-13 01:43:07 +02001422int is_tracing_stopped(void)
1423{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001424 return global_trace.stop_count;
Carsten Emdeb5130b12009-09-13 01:43:07 +02001425}
1426
Steven Rostedt0f048702008-11-05 16:05:44 -05001427/**
1428 * tracing_start - quick start of the tracer
1429 *
1430 * If tracing is enabled but was stopped by tracing_stop,
1431 * this will start the tracer back up.
1432 */
1433void tracing_start(void)
1434{
1435 struct ring_buffer *buffer;
1436 unsigned long flags;
1437
1438 if (tracing_disabled)
1439 return;
1440
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001441 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1442 if (--global_trace.stop_count) {
1443 if (global_trace.stop_count < 0) {
Steven Rostedtb06a8302009-01-22 14:26:15 -05001444 /* Someone screwed up their debugging */
1445 WARN_ON_ONCE(1);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001446 global_trace.stop_count = 0;
Steven Rostedtb06a8302009-01-22 14:26:15 -05001447 }
Steven Rostedt0f048702008-11-05 16:05:44 -05001448 goto out;
1449 }
1450
Steven Rostedta2f80712010-03-12 19:56:00 -05001451 /* Prevent the buffers from switching */
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001452 arch_spin_lock(&global_trace.max_lock);
Steven Rostedt0f048702008-11-05 16:05:44 -05001453
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001454 buffer = global_trace.trace_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001455 if (buffer)
1456 ring_buffer_record_enable(buffer);
1457
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001458#ifdef CONFIG_TRACER_MAX_TRACE
1459 buffer = global_trace.max_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001460 if (buffer)
1461 ring_buffer_record_enable(buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001462#endif
Steven Rostedt0f048702008-11-05 16:05:44 -05001463
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001464 arch_spin_unlock(&global_trace.max_lock);
Steven Rostedta2f80712010-03-12 19:56:00 -05001465
Steven Rostedt0f048702008-11-05 16:05:44 -05001466 out:
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001467 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1468}
1469
1470static void tracing_start_tr(struct trace_array *tr)
1471{
1472 struct ring_buffer *buffer;
1473 unsigned long flags;
1474
1475 if (tracing_disabled)
1476 return;
1477
1478 /* If global, we need to also start the max tracer */
1479 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1480 return tracing_start();
1481
1482 raw_spin_lock_irqsave(&tr->start_lock, flags);
1483
1484 if (--tr->stop_count) {
1485 if (tr->stop_count < 0) {
1486 /* Someone screwed up their debugging */
1487 WARN_ON_ONCE(1);
1488 tr->stop_count = 0;
1489 }
1490 goto out;
1491 }
1492
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001493 buffer = tr->trace_buffer.buffer;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001494 if (buffer)
1495 ring_buffer_record_enable(buffer);
1496
1497 out:
1498 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001499}
1500
1501/**
1502 * tracing_stop - quick stop of the tracer
1503 *
1504 * Light weight way to stop tracing. Use in conjunction with
1505 * tracing_start.
1506 */
1507void tracing_stop(void)
1508{
1509 struct ring_buffer *buffer;
1510 unsigned long flags;
1511
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001512 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1513 if (global_trace.stop_count++)
Steven Rostedt0f048702008-11-05 16:05:44 -05001514 goto out;
1515
Steven Rostedta2f80712010-03-12 19:56:00 -05001516 /* Prevent the buffers from switching */
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001517 arch_spin_lock(&global_trace.max_lock);
Steven Rostedta2f80712010-03-12 19:56:00 -05001518
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001519 buffer = global_trace.trace_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001520 if (buffer)
1521 ring_buffer_record_disable(buffer);
1522
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001523#ifdef CONFIG_TRACER_MAX_TRACE
1524 buffer = global_trace.max_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001525 if (buffer)
1526 ring_buffer_record_disable(buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001527#endif
Steven Rostedt0f048702008-11-05 16:05:44 -05001528
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001529 arch_spin_unlock(&global_trace.max_lock);
Steven Rostedta2f80712010-03-12 19:56:00 -05001530
Steven Rostedt0f048702008-11-05 16:05:44 -05001531 out:
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001532 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1533}
1534
1535static void tracing_stop_tr(struct trace_array *tr)
1536{
1537 struct ring_buffer *buffer;
1538 unsigned long flags;
1539
1540 /* If global, we need to also stop the max tracer */
1541 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1542 return tracing_stop();
1543
1544 raw_spin_lock_irqsave(&tr->start_lock, flags);
1545 if (tr->stop_count++)
1546 goto out;
1547
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001548 buffer = tr->trace_buffer.buffer;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001549 if (buffer)
1550 ring_buffer_record_disable(buffer);
1551
1552 out:
1553 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001554}
1555
Ingo Molnare309b412008-05-12 21:20:51 +02001556void trace_stop_cmdline_recording(void);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001557
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001558static int trace_save_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001559{
Carsten Emdea635cf02009-03-18 09:00:41 +01001560 unsigned pid, idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001561
1562 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001563 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001564
1565 /*
1566 * It's not the end of the world if we don't get
1567 * the lock, but we also don't want to spin
1568 * nor do we want to disable interrupts,
1569 * so if we miss here, then better luck next time.
1570 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001571 if (!arch_spin_trylock(&trace_cmdline_lock))
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001572 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001573
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001574 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001575 if (idx == NO_CMDLINE_MAP) {
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001576 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001577
Carsten Emdea635cf02009-03-18 09:00:41 +01001578 /*
1579 * Check whether the cmdline buffer at idx has a pid
1580 * mapped. We are going to overwrite that entry so we
1581 * need to clear the map_pid_to_cmdline. Otherwise we
1582 * would read the new comm for the old pid.
1583 */
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001584 pid = savedcmd->map_cmdline_to_pid[idx];
Carsten Emdea635cf02009-03-18 09:00:41 +01001585 if (pid != NO_CMDLINE_MAP)
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001586 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001587
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001588 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1589 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001590
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001591 savedcmd->cmdline_idx = idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001592 }
1593
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001594 set_cmdline(idx, tsk->comm);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001595
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001596 arch_spin_unlock(&trace_cmdline_lock);
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001597
1598 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001599}
1600
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04001601static void __trace_find_cmdline(int pid, char comm[])
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001602{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001603 unsigned map;
1604
Steven Rostedt4ca53082009-03-16 19:20:15 -04001605 if (!pid) {
1606 strcpy(comm, "<idle>");
1607 return;
1608 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001609
Steven Rostedt74bf4072010-01-25 15:11:53 -05001610 if (WARN_ON_ONCE(pid < 0)) {
1611 strcpy(comm, "<XXX>");
1612 return;
1613 }
1614
Steven Rostedt4ca53082009-03-16 19:20:15 -04001615 if (pid > PID_MAX_DEFAULT) {
1616 strcpy(comm, "<...>");
1617 return;
1618 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001619
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001620 map = savedcmd->map_pid_to_cmdline[pid];
Thomas Gleixner50d88752009-03-18 08:58:44 +01001621 if (map != NO_CMDLINE_MAP)
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001622 strcpy(comm, get_saved_cmdlines(map));
Thomas Gleixner50d88752009-03-18 08:58:44 +01001623 else
1624 strcpy(comm, "<...>");
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04001625}
1626
1627void trace_find_cmdline(int pid, char comm[])
1628{
1629 preempt_disable();
1630 arch_spin_lock(&trace_cmdline_lock);
1631
1632 __trace_find_cmdline(pid, comm);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001633
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001634 arch_spin_unlock(&trace_cmdline_lock);
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001635 preempt_enable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001636}
1637
Ingo Molnare309b412008-05-12 21:20:51 +02001638void tracing_record_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001639{
Steven Rostedt0fb96562012-05-11 14:25:30 -04001640 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001641 return;
1642
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001643 if (!__this_cpu_read(trace_cmdline_save))
1644 return;
1645
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001646 if (trace_save_cmdline(tsk))
1647 __this_cpu_write(trace_cmdline_save, false);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001648}
1649
Pekka Paalanen45dcd8b2008-09-16 21:56:41 +03001650void
Steven Rostedt38697052008-10-01 13:14:09 -04001651tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1652 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001653{
1654 struct task_struct *tsk = current;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001655
Steven Rostedt777e2082008-09-29 23:02:42 -04001656 entry->preempt_count = pc & 0xff;
1657 entry->pid = (tsk) ? tsk->pid : 0;
1658 entry->flags =
Steven Rostedt92444892008-10-24 09:42:59 -04001659#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
Steven Rostedt2e2ca152008-08-01 12:26:40 -04001660 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
Steven Rostedt92444892008-10-24 09:42:59 -04001661#else
1662 TRACE_FLAG_IRQS_NOSUPPORT |
1663#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001664 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1665 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
Peter Zijlstrae5137b52013-10-04 17:28:26 +02001666 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1667 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001668}
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +02001669EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001670
Steven Rostedte77405a2009-09-02 14:17:06 -04001671struct ring_buffer_event *
1672trace_buffer_lock_reserve(struct ring_buffer *buffer,
1673 int type,
1674 unsigned long len,
1675 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001676{
1677 struct ring_buffer_event *event;
1678
Steven Rostedte77405a2009-09-02 14:17:06 -04001679 event = ring_buffer_lock_reserve(buffer, len);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001680 if (event != NULL) {
1681 struct trace_entry *ent = ring_buffer_event_data(event);
1682
1683 tracing_generic_entry_update(ent, flags, pc);
1684 ent->type = type;
1685 }
1686
1687 return event;
1688}
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001689
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001690void
1691__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1692{
1693 __this_cpu_write(trace_cmdline_save, true);
1694 ring_buffer_unlock_commit(buffer, event);
1695}
1696
Steven Rostedt (Red Hat)b7f0c952015-09-25 17:38:44 -04001697void trace_buffer_unlock_commit(struct trace_array *tr,
1698 struct ring_buffer *buffer,
1699 struct ring_buffer_event *event,
1700 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001701{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001702 __buffer_unlock_commit(buffer, event);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001703
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001704 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
Steven Rostedte77405a2009-09-02 14:17:06 -04001705 ftrace_trace_userstack(buffer, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001706}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001707EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001708
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001709static struct ring_buffer *temp_buffer;
1710
Steven Rostedtef5580d2009-02-27 19:38:04 -05001711struct ring_buffer_event *
Steven Rostedtccb469a2012-08-02 10:32:10 -04001712trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
Steven Rostedt (Red Hat)7f1d2f82015-05-05 10:09:53 -04001713 struct trace_event_file *trace_file,
Steven Rostedtccb469a2012-08-02 10:32:10 -04001714 int type, unsigned long len,
1715 unsigned long flags, int pc)
1716{
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001717 struct ring_buffer_event *entry;
1718
Steven Rostedt (Red Hat)7f1d2f82015-05-05 10:09:53 -04001719 *current_rb = trace_file->tr->trace_buffer.buffer;
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001720 entry = trace_buffer_lock_reserve(*current_rb,
Steven Rostedtccb469a2012-08-02 10:32:10 -04001721 type, len, flags, pc);
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001722 /*
1723 * If tracing is off, but we have triggers enabled
1724 * we still need to look at the event data. Use the temp_buffer
1725 * to store the trace event for the tigger to use. It's recusive
1726 * safe and will not be recorded anywhere.
1727 */
Steven Rostedt (Red Hat)5d6ad962015-05-13 15:12:33 -04001728 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001729 *current_rb = temp_buffer;
1730 entry = trace_buffer_lock_reserve(*current_rb,
1731 type, len, flags, pc);
1732 }
1733 return entry;
Steven Rostedtccb469a2012-08-02 10:32:10 -04001734}
1735EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1736
1737struct ring_buffer_event *
Steven Rostedte77405a2009-09-02 14:17:06 -04001738trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1739 int type, unsigned long len,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001740 unsigned long flags, int pc)
1741{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001742 *current_rb = global_trace.trace_buffer.buffer;
Steven Rostedte77405a2009-09-02 14:17:06 -04001743 return trace_buffer_lock_reserve(*current_rb,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001744 type, len, flags, pc);
1745}
Steven Rostedt94487d62009-05-05 19:22:53 -04001746EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001747
Steven Rostedt (Red Hat)b7f0c952015-09-25 17:38:44 -04001748void trace_buffer_unlock_commit_regs(struct trace_array *tr,
1749 struct ring_buffer *buffer,
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001750 struct ring_buffer_event *event,
1751 unsigned long flags, int pc,
1752 struct pt_regs *regs)
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001753{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001754 __buffer_unlock_commit(buffer, event);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001755
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001756 ftrace_trace_stack(tr, buffer, flags, 6, pc, regs);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001757 ftrace_trace_userstack(buffer, flags, pc);
1758}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001759EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001760
Steven Rostedte77405a2009-09-02 14:17:06 -04001761void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1762 struct ring_buffer_event *event)
Steven Rostedt77d9f462009-04-02 01:16:59 -04001763{
Steven Rostedte77405a2009-09-02 14:17:06 -04001764 ring_buffer_discard_commit(buffer, event);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001765}
Steven Rostedt12acd472009-04-17 16:01:56 -04001766EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001767
Ingo Molnare309b412008-05-12 21:20:51 +02001768void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001769trace_function(struct trace_array *tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001770 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1771 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001772{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04001773 struct trace_event_call *call = &event_function;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001774 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001775 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001776 struct ftrace_entry *entry;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001777
Steven Rostedtd7690412008-10-01 00:29:53 -04001778 /* If we are reading the ring buffer, don't trace */
Rusty Russelldd17c8f2009-10-29 22:34:15 +09001779 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
Steven Rostedtd7690412008-10-01 00:29:53 -04001780 return;
1781
Steven Rostedte77405a2009-09-02 14:17:06 -04001782 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001783 flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001784 if (!event)
1785 return;
1786 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001787 entry->ip = ip;
1788 entry->parent_ip = parent_ip;
Tom Zanussie1112b42009-03-31 00:48:49 -05001789
Tom Zanussif306cc82013-10-24 08:34:17 -05001790 if (!call_filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001791 __buffer_unlock_commit(buffer, event);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001792}
1793
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001794#ifdef CONFIG_STACKTRACE
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001795
1796#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1797struct ftrace_stack {
1798 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1799};
1800
1801static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1802static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1803
Steven Rostedte77405a2009-09-02 14:17:06 -04001804static void __ftrace_trace_stack(struct ring_buffer *buffer,
Steven Rostedt53614992009-01-15 19:12:40 -05001805 unsigned long flags,
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001806 int skip, int pc, struct pt_regs *regs)
Ingo Molnar86387f72008-05-12 21:20:51 +02001807{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04001808 struct trace_event_call *call = &event_kernel_stack;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001809 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001810 struct stack_entry *entry;
Ingo Molnar86387f72008-05-12 21:20:51 +02001811 struct stack_trace trace;
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001812 int use_stack;
1813 int size = FTRACE_STACK_ENTRIES;
Ingo Molnar86387f72008-05-12 21:20:51 +02001814
1815 trace.nr_entries = 0;
Ingo Molnar86387f72008-05-12 21:20:51 +02001816 trace.skip = skip;
Ingo Molnar86387f72008-05-12 21:20:51 +02001817
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001818 /*
1819 * Since events can happen in NMIs there's no safe way to
1820 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1821 * or NMI comes in, it will just have to use the default
1822 * FTRACE_STACK_SIZE.
1823 */
1824 preempt_disable_notrace();
1825
Shan Wei82146522012-11-19 13:21:01 +08001826 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001827 /*
1828 * We don't need any atomic variables, just a barrier.
1829 * If an interrupt comes in, we don't care, because it would
1830 * have exited and put the counter back to what we want.
1831 * We just need a barrier to keep gcc from moving things
1832 * around.
1833 */
1834 barrier();
1835 if (use_stack == 1) {
Christoph Lameterbdffd892014-04-29 14:17:40 -05001836 trace.entries = this_cpu_ptr(ftrace_stack.calls);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001837 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1838
1839 if (regs)
1840 save_stack_trace_regs(regs, &trace);
1841 else
1842 save_stack_trace(&trace);
1843
1844 if (trace.nr_entries > size)
1845 size = trace.nr_entries;
1846 } else
1847 /* From now on, use_stack is a boolean */
1848 use_stack = 0;
1849
1850 size *= sizeof(unsigned long);
1851
1852 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1853 sizeof(*entry) + size, flags, pc);
1854 if (!event)
1855 goto out;
1856 entry = ring_buffer_event_data(event);
1857
1858 memset(&entry->caller, 0, size);
1859
1860 if (use_stack)
1861 memcpy(&entry->caller, trace.entries,
1862 trace.nr_entries * sizeof(unsigned long));
1863 else {
1864 trace.max_entries = FTRACE_STACK_ENTRIES;
1865 trace.entries = entry->caller;
1866 if (regs)
1867 save_stack_trace_regs(regs, &trace);
1868 else
1869 save_stack_trace(&trace);
1870 }
1871
1872 entry->size = trace.nr_entries;
1873
Tom Zanussif306cc82013-10-24 08:34:17 -05001874 if (!call_filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001875 __buffer_unlock_commit(buffer, event);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001876
1877 out:
1878 /* Again, don't let gcc optimize things here */
1879 barrier();
Shan Wei82146522012-11-19 13:21:01 +08001880 __this_cpu_dec(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001881 preempt_enable_notrace();
1882
Ingo Molnarf0a920d2008-05-12 21:20:47 +02001883}
1884
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001885static inline void ftrace_trace_stack(struct trace_array *tr,
1886 struct ring_buffer *buffer,
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -04001887 unsigned long flags,
1888 int skip, int pc, struct pt_regs *regs)
Steven Rostedt53614992009-01-15 19:12:40 -05001889{
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001890 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
Steven Rostedt53614992009-01-15 19:12:40 -05001891 return;
1892
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -04001893 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
Steven Rostedt53614992009-01-15 19:12:40 -05001894}
1895
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001896void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1897 int pc)
Steven Rostedt38697052008-10-01 13:14:09 -04001898{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001899 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
Steven Rostedt38697052008-10-01 13:14:09 -04001900}
1901
Steven Rostedt03889382009-12-11 09:48:22 -05001902/**
1903 * trace_dump_stack - record a stack back trace in the trace buffer
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001904 * @skip: Number of functions to skip (helper handlers)
Steven Rostedt03889382009-12-11 09:48:22 -05001905 */
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001906void trace_dump_stack(int skip)
Steven Rostedt03889382009-12-11 09:48:22 -05001907{
1908 unsigned long flags;
1909
1910 if (tracing_disabled || tracing_selftest_running)
Steven Rostedte36c5452009-12-14 15:58:33 -05001911 return;
Steven Rostedt03889382009-12-11 09:48:22 -05001912
1913 local_save_flags(flags);
1914
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001915 /*
1916 * Skip 3 more, seems to get us at the caller of
1917 * this function.
1918 */
1919 skip += 3;
1920 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1921 flags, skip, preempt_count(), NULL);
Steven Rostedt03889382009-12-11 09:48:22 -05001922}
1923
Steven Rostedt91e86e52010-11-10 12:56:12 +01001924static DEFINE_PER_CPU(int, user_stack_count);
1925
Steven Rostedte77405a2009-09-02 14:17:06 -04001926void
1927ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
Török Edwin02b67512008-11-22 13:28:47 +02001928{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04001929 struct trace_event_call *call = &event_user_stack;
Török Edwin8d7c6a92008-11-23 12:39:06 +02001930 struct ring_buffer_event *event;
Török Edwin02b67512008-11-22 13:28:47 +02001931 struct userstack_entry *entry;
1932 struct stack_trace trace;
Török Edwin02b67512008-11-22 13:28:47 +02001933
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04001934 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
Török Edwin02b67512008-11-22 13:28:47 +02001935 return;
1936
Steven Rostedtb6345872010-03-12 20:03:30 -05001937 /*
1938 * NMIs can not handle page faults, even with fix ups.
1939 * The save user stack can (and often does) fault.
1940 */
1941 if (unlikely(in_nmi()))
1942 return;
1943
Steven Rostedt91e86e52010-11-10 12:56:12 +01001944 /*
1945 * prevent recursion, since the user stack tracing may
1946 * trigger other kernel events.
1947 */
1948 preempt_disable();
1949 if (__this_cpu_read(user_stack_count))
1950 goto out;
1951
1952 __this_cpu_inc(user_stack_count);
1953
Steven Rostedte77405a2009-09-02 14:17:06 -04001954 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001955 sizeof(*entry), flags, pc);
Török Edwin02b67512008-11-22 13:28:47 +02001956 if (!event)
Li Zefan1dbd1952010-12-09 15:47:56 +08001957 goto out_drop_count;
Török Edwin02b67512008-11-22 13:28:47 +02001958 entry = ring_buffer_event_data(event);
Török Edwin02b67512008-11-22 13:28:47 +02001959
Steven Rostedt48659d32009-09-11 11:36:23 -04001960 entry->tgid = current->tgid;
Török Edwin02b67512008-11-22 13:28:47 +02001961 memset(&entry->caller, 0, sizeof(entry->caller));
1962
1963 trace.nr_entries = 0;
1964 trace.max_entries = FTRACE_STACK_ENTRIES;
1965 trace.skip = 0;
1966 trace.entries = entry->caller;
1967
1968 save_stack_trace_user(&trace);
Tom Zanussif306cc82013-10-24 08:34:17 -05001969 if (!call_filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001970 __buffer_unlock_commit(buffer, event);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001971
Li Zefan1dbd1952010-12-09 15:47:56 +08001972 out_drop_count:
Steven Rostedt91e86e52010-11-10 12:56:12 +01001973 __this_cpu_dec(user_stack_count);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001974 out:
1975 preempt_enable();
Török Edwin02b67512008-11-22 13:28:47 +02001976}
1977
Hannes Eder4fd27352009-02-10 19:44:12 +01001978#ifdef UNUSED
1979static void __trace_userstack(struct trace_array *tr, unsigned long flags)
Török Edwin02b67512008-11-22 13:28:47 +02001980{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001981 ftrace_trace_userstack(tr, flags, preempt_count());
Török Edwin02b67512008-11-22 13:28:47 +02001982}
Hannes Eder4fd27352009-02-10 19:44:12 +01001983#endif /* UNUSED */
Török Edwin02b67512008-11-22 13:28:47 +02001984
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001985#endif /* CONFIG_STACKTRACE */
1986
Steven Rostedt07d777f2011-09-22 14:01:55 -04001987/* created for use with alloc_percpu */
1988struct trace_buffer_struct {
1989 char buffer[TRACE_BUF_SIZE];
1990};
1991
1992static struct trace_buffer_struct *trace_percpu_buffer;
1993static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1994static struct trace_buffer_struct *trace_percpu_irq_buffer;
1995static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1996
1997/*
1998 * The buffer used is dependent on the context. There is a per cpu
1999 * buffer for normal context, softirq contex, hard irq context and
2000 * for NMI context. Thise allows for lockless recording.
2001 *
2002 * Note, if the buffers failed to be allocated, then this returns NULL
2003 */
2004static char *get_trace_buf(void)
2005{
2006 struct trace_buffer_struct *percpu_buffer;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002007
2008 /*
2009 * If we have allocated per cpu buffers, then we do not
2010 * need to do any locking.
2011 */
2012 if (in_nmi())
2013 percpu_buffer = trace_percpu_nmi_buffer;
2014 else if (in_irq())
2015 percpu_buffer = trace_percpu_irq_buffer;
2016 else if (in_softirq())
2017 percpu_buffer = trace_percpu_sirq_buffer;
2018 else
2019 percpu_buffer = trace_percpu_buffer;
2020
2021 if (!percpu_buffer)
2022 return NULL;
2023
Shan Weid8a03492012-11-13 09:53:04 +08002024 return this_cpu_ptr(&percpu_buffer->buffer[0]);
Steven Rostedt07d777f2011-09-22 14:01:55 -04002025}
2026
2027static int alloc_percpu_trace_buffer(void)
2028{
2029 struct trace_buffer_struct *buffers;
2030 struct trace_buffer_struct *sirq_buffers;
2031 struct trace_buffer_struct *irq_buffers;
2032 struct trace_buffer_struct *nmi_buffers;
2033
2034 buffers = alloc_percpu(struct trace_buffer_struct);
2035 if (!buffers)
2036 goto err_warn;
2037
2038 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2039 if (!sirq_buffers)
2040 goto err_sirq;
2041
2042 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2043 if (!irq_buffers)
2044 goto err_irq;
2045
2046 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2047 if (!nmi_buffers)
2048 goto err_nmi;
2049
2050 trace_percpu_buffer = buffers;
2051 trace_percpu_sirq_buffer = sirq_buffers;
2052 trace_percpu_irq_buffer = irq_buffers;
2053 trace_percpu_nmi_buffer = nmi_buffers;
2054
2055 return 0;
2056
2057 err_nmi:
2058 free_percpu(irq_buffers);
2059 err_irq:
2060 free_percpu(sirq_buffers);
2061 err_sirq:
2062 free_percpu(buffers);
2063 err_warn:
2064 WARN(1, "Could not allocate percpu trace_printk buffer");
2065 return -ENOMEM;
2066}
2067
Steven Rostedt81698832012-10-11 10:15:05 -04002068static int buffers_allocated;
2069
Steven Rostedt07d777f2011-09-22 14:01:55 -04002070void trace_printk_init_buffers(void)
2071{
Steven Rostedt07d777f2011-09-22 14:01:55 -04002072 if (buffers_allocated)
2073 return;
2074
2075 if (alloc_percpu_trace_buffer())
2076 return;
2077
Steven Rostedt2184db42014-05-28 13:14:40 -04002078 /* trace_printk() is for debug use only. Don't use it in production. */
2079
Borislav Petkov69a1c992015-01-27 17:17:20 +01002080 pr_warning("\n");
2081 pr_warning("**********************************************************\n");
Steven Rostedt2184db42014-05-28 13:14:40 -04002082 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2083 pr_warning("** **\n");
2084 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2085 pr_warning("** **\n");
2086 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
Frans Klavereff264e2014-11-07 15:53:44 +01002087 pr_warning("** unsafe for production use. **\n");
Steven Rostedt2184db42014-05-28 13:14:40 -04002088 pr_warning("** **\n");
2089 pr_warning("** If you see this message and you are not debugging **\n");
2090 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2091 pr_warning("** **\n");
2092 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2093 pr_warning("**********************************************************\n");
Steven Rostedt07d777f2011-09-22 14:01:55 -04002094
Steven Rostedtb382ede62012-10-10 21:44:34 -04002095 /* Expand the buffers to set size */
2096 tracing_update_buffers();
2097
Steven Rostedt07d777f2011-09-22 14:01:55 -04002098 buffers_allocated = 1;
Steven Rostedt81698832012-10-11 10:15:05 -04002099
2100 /*
2101 * trace_printk_init_buffers() can be called by modules.
2102 * If that happens, then we need to start cmdline recording
2103 * directly here. If the global_trace.buffer is already
2104 * allocated here, then this was called by module code.
2105 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002106 if (global_trace.trace_buffer.buffer)
Steven Rostedt81698832012-10-11 10:15:05 -04002107 tracing_start_cmdline_record();
2108}
2109
2110void trace_printk_start_comm(void)
2111{
2112 /* Start tracing comms if trace printk is set */
2113 if (!buffers_allocated)
2114 return;
2115 tracing_start_cmdline_record();
2116}
2117
2118static void trace_printk_start_stop_comm(int enabled)
2119{
2120 if (!buffers_allocated)
2121 return;
2122
2123 if (enabled)
2124 tracing_start_cmdline_record();
2125 else
2126 tracing_stop_cmdline_record();
Steven Rostedt07d777f2011-09-22 14:01:55 -04002127}
2128
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002129/**
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002130 * trace_vbprintk - write binary msg to tracing buffer
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002131 *
2132 */
Steven Rostedt40ce74f2009-03-19 14:03:53 -04002133int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002134{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04002135 struct trace_event_call *call = &event_bprint;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002136 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04002137 struct ring_buffer *buffer;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002138 struct trace_array *tr = &global_trace;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002139 struct bprint_entry *entry;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002140 unsigned long flags;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002141 char *tbuffer;
2142 int len = 0, size, pc;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002143
2144 if (unlikely(tracing_selftest_running || tracing_disabled))
2145 return 0;
2146
2147 /* Don't pollute graph traces with trace_vprintk internals */
2148 pause_graph_tracing();
2149
2150 pc = preempt_count();
Steven Rostedt5168ae52010-06-03 09:36:50 -04002151 preempt_disable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002152
Steven Rostedt07d777f2011-09-22 14:01:55 -04002153 tbuffer = get_trace_buf();
2154 if (!tbuffer) {
2155 len = 0;
2156 goto out;
2157 }
2158
2159 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2160
2161 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002162 goto out;
2163
Steven Rostedt07d777f2011-09-22 14:01:55 -04002164 local_save_flags(flags);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002165 size = sizeof(*entry) + sizeof(u32) * len;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002166 buffer = tr->trace_buffer.buffer;
Steven Rostedte77405a2009-09-02 14:17:06 -04002167 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2168 flags, pc);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002169 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04002170 goto out;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002171 entry = ring_buffer_event_data(event);
2172 entry->ip = ip;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002173 entry->fmt = fmt;
2174
Steven Rostedt07d777f2011-09-22 14:01:55 -04002175 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
Tom Zanussif306cc82013-10-24 08:34:17 -05002176 if (!call_filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04002177 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04002178 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
Steven Rostedtd9313692010-01-06 17:27:11 -05002179 }
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002180
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002181out:
Steven Rostedt5168ae52010-06-03 09:36:50 -04002182 preempt_enable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002183 unpause_graph_tracing();
2184
2185 return len;
2186}
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002187EXPORT_SYMBOL_GPL(trace_vbprintk);
2188
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002189static int
2190__trace_array_vprintk(struct ring_buffer *buffer,
2191 unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002192{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04002193 struct trace_event_call *call = &event_print;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002194 struct ring_buffer_event *event;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002195 int len = 0, size, pc;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002196 struct print_entry *entry;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002197 unsigned long flags;
2198 char *tbuffer;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002199
2200 if (tracing_disabled || tracing_selftest_running)
2201 return 0;
2202
Steven Rostedt07d777f2011-09-22 14:01:55 -04002203 /* Don't pollute graph traces with trace_vprintk internals */
2204 pause_graph_tracing();
2205
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002206 pc = preempt_count();
2207 preempt_disable_notrace();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002208
Steven Rostedt07d777f2011-09-22 14:01:55 -04002209
2210 tbuffer = get_trace_buf();
2211 if (!tbuffer) {
2212 len = 0;
2213 goto out;
2214 }
2215
Dan Carpenter3558a5a2014-11-27 18:57:52 +03002216 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002217
Steven Rostedt07d777f2011-09-22 14:01:55 -04002218 local_save_flags(flags);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002219 size = sizeof(*entry) + len + 1;
Steven Rostedte77405a2009-09-02 14:17:06 -04002220 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
Steven Rostedt07d777f2011-09-22 14:01:55 -04002221 flags, pc);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002222 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04002223 goto out;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002224 entry = ring_buffer_event_data(event);
Carsten Emdec13d2f72009-11-16 20:56:13 +01002225 entry->ip = ip;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002226
Dan Carpenter3558a5a2014-11-27 18:57:52 +03002227 memcpy(&entry->buf, tbuffer, len + 1);
Tom Zanussif306cc82013-10-24 08:34:17 -05002228 if (!call_filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04002229 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04002230 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
Steven Rostedtd9313692010-01-06 17:27:11 -05002231 }
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002232 out:
2233 preempt_enable_notrace();
Steven Rostedt07d777f2011-09-22 14:01:55 -04002234 unpause_graph_tracing();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002235
2236 return len;
2237}
Steven Rostedt659372d2009-09-03 19:11:07 -04002238
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002239int trace_array_vprintk(struct trace_array *tr,
2240 unsigned long ip, const char *fmt, va_list args)
2241{
2242 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2243}
2244
2245int trace_array_printk(struct trace_array *tr,
2246 unsigned long ip, const char *fmt, ...)
2247{
2248 int ret;
2249 va_list ap;
2250
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002251 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002252 return 0;
2253
2254 va_start(ap, fmt);
2255 ret = trace_array_vprintk(tr, ip, fmt, ap);
2256 va_end(ap);
2257 return ret;
2258}
2259
2260int trace_array_printk_buf(struct ring_buffer *buffer,
2261 unsigned long ip, const char *fmt, ...)
2262{
2263 int ret;
2264 va_list ap;
2265
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002266 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002267 return 0;
2268
2269 va_start(ap, fmt);
2270 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2271 va_end(ap);
2272 return ret;
2273}
2274
Steven Rostedt659372d2009-09-03 19:11:07 -04002275int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2276{
Steven Rostedta813a152009-10-09 01:41:35 -04002277 return trace_array_vprintk(&global_trace, ip, fmt, args);
Steven Rostedt659372d2009-09-03 19:11:07 -04002278}
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002279EXPORT_SYMBOL_GPL(trace_vprintk);
2280
Robert Richtere2ac8ef2008-11-12 12:59:32 +01002281static void trace_iterator_increment(struct trace_iterator *iter)
Steven Rostedt5a90f572008-09-03 17:42:51 -04002282{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002283 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2284
Steven Rostedt5a90f572008-09-03 17:42:51 -04002285 iter->idx++;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002286 if (buf_iter)
2287 ring_buffer_read(buf_iter, NULL);
Steven Rostedt5a90f572008-09-03 17:42:51 -04002288}
2289
Ingo Molnare309b412008-05-12 21:20:51 +02002290static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04002291peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2292 unsigned long *lost_events)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002293{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002294 struct ring_buffer_event *event;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002295 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002296
Steven Rostedtd7690412008-10-01 00:29:53 -04002297 if (buf_iter)
2298 event = ring_buffer_iter_peek(buf_iter, ts);
2299 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002300 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
Steven Rostedtbc21b472010-03-31 19:49:26 -04002301 lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04002302
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04002303 if (event) {
2304 iter->ent_size = ring_buffer_event_length(event);
2305 return ring_buffer_event_data(event);
2306 }
2307 iter->ent_size = 0;
2308 return NULL;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002309}
Steven Rostedtd7690412008-10-01 00:29:53 -04002310
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002311static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04002312__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2313 unsigned long *missing_events, u64 *ent_ts)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002314{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002315 struct ring_buffer *buffer = iter->trace_buffer->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002316 struct trace_entry *ent, *next = NULL;
Lai Jiangshanaa274972010-04-05 17:11:05 +08002317 unsigned long lost_events = 0, next_lost = 0;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002318 int cpu_file = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002319 u64 next_ts = 0, ts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002320 int next_cpu = -1;
Steven Rostedt12b5da32012-03-27 10:43:28 -04002321 int next_size = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002322 int cpu;
2323
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002324 /*
2325 * If we are in a per_cpu trace file, don't bother by iterating over
2326 * all cpu and peek directly.
2327 */
Steven Rostedtae3b5092013-01-23 15:22:59 -05002328 if (cpu_file > RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002329 if (ring_buffer_empty_cpu(buffer, cpu_file))
2330 return NULL;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002331 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002332 if (ent_cpu)
2333 *ent_cpu = cpu_file;
2334
2335 return ent;
2336 }
2337
Steven Rostedtab464282008-05-12 21:21:00 +02002338 for_each_tracing_cpu(cpu) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002339
2340 if (ring_buffer_empty_cpu(buffer, cpu))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002341 continue;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002342
Steven Rostedtbc21b472010-03-31 19:49:26 -04002343 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002344
Ingo Molnarcdd31cd2008-05-12 21:20:46 +02002345 /*
2346 * Pick the entry with the smallest timestamp:
2347 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002348 if (ent && (!next || ts < next_ts)) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002349 next = ent;
2350 next_cpu = cpu;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002351 next_ts = ts;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002352 next_lost = lost_events;
Steven Rostedt12b5da32012-03-27 10:43:28 -04002353 next_size = iter->ent_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002354 }
2355 }
2356
Steven Rostedt12b5da32012-03-27 10:43:28 -04002357 iter->ent_size = next_size;
2358
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002359 if (ent_cpu)
2360 *ent_cpu = next_cpu;
2361
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002362 if (ent_ts)
2363 *ent_ts = next_ts;
2364
Steven Rostedtbc21b472010-03-31 19:49:26 -04002365 if (missing_events)
2366 *missing_events = next_lost;
2367
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002368 return next;
2369}
2370
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002371/* Find the next real entry, without updating the iterator itself */
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002372struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2373 int *ent_cpu, u64 *ent_ts)
Steven Rostedtb3806b42008-05-12 21:20:46 +02002374{
Steven Rostedtbc21b472010-03-31 19:49:26 -04002375 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002376}
Ingo Molnar8c523a92008-05-12 21:20:46 +02002377
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002378/* Find the next real entry, and increment the iterator to the next entry */
Jason Wessel955b61e2010-08-05 09:22:23 -05002379void *trace_find_next_entry_inc(struct trace_iterator *iter)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002380{
Steven Rostedtbc21b472010-03-31 19:49:26 -04002381 iter->ent = __find_next_entry(iter, &iter->cpu,
2382 &iter->lost_events, &iter->ts);
Steven Rostedtb3806b42008-05-12 21:20:46 +02002383
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002384 if (iter->ent)
Robert Richtere2ac8ef2008-11-12 12:59:32 +01002385 trace_iterator_increment(iter);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002386
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002387 return iter->ent ? iter : NULL;
Steven Rostedtb3806b42008-05-12 21:20:46 +02002388}
2389
Ingo Molnare309b412008-05-12 21:20:51 +02002390static void trace_consume(struct trace_iterator *iter)
Steven Rostedtb3806b42008-05-12 21:20:46 +02002391{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002392 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
Steven Rostedtbc21b472010-03-31 19:49:26 -04002393 &iter->lost_events);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002394}
2395
Ingo Molnare309b412008-05-12 21:20:51 +02002396static void *s_next(struct seq_file *m, void *v, loff_t *pos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002397{
2398 struct trace_iterator *iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002399 int i = (int)*pos;
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002400 void *ent;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002401
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002402 WARN_ON_ONCE(iter->leftover);
2403
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002404 (*pos)++;
2405
2406 /* can't go backwards */
2407 if (iter->idx > i)
2408 return NULL;
2409
2410 if (iter->idx < 0)
Jason Wessel955b61e2010-08-05 09:22:23 -05002411 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002412 else
2413 ent = iter;
2414
2415 while (ent && iter->idx < i)
Jason Wessel955b61e2010-08-05 09:22:23 -05002416 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002417
2418 iter->pos = *pos;
2419
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002420 return ent;
2421}
2422
Jason Wessel955b61e2010-08-05 09:22:23 -05002423void tracing_iter_reset(struct trace_iterator *iter, int cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002424{
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002425 struct ring_buffer_event *event;
2426 struct ring_buffer_iter *buf_iter;
2427 unsigned long entries = 0;
2428 u64 ts;
2429
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002430 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002431
Steven Rostedt6d158a82012-06-27 20:46:14 -04002432 buf_iter = trace_buffer_iter(iter, cpu);
2433 if (!buf_iter)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002434 return;
2435
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002436 ring_buffer_iter_reset(buf_iter);
2437
2438 /*
2439 * We could have the case with the max latency tracers
2440 * that a reset never took place on a cpu. This is evident
2441 * by the timestamp being before the start of the buffer.
2442 */
2443 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002444 if (ts >= iter->trace_buffer->time_start)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002445 break;
2446 entries++;
2447 ring_buffer_read(buf_iter, NULL);
2448 }
2449
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002450 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002451}
2452
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002453/*
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002454 * The current tracer is copied to avoid a global locking
2455 * all around.
2456 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002457static void *s_start(struct seq_file *m, loff_t *pos)
2458{
2459 struct trace_iterator *iter = m->private;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002460 struct trace_array *tr = iter->tr;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002461 int cpu_file = iter->cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002462 void *p = NULL;
2463 loff_t l = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002464 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002465
Hiraku Toyooka2fd196e2012-12-26 11:52:52 +09002466 /*
2467 * copy the tracer to avoid using a global lock all around.
2468 * iter->trace is a copy of current_trace, the pointer to the
2469 * name may be used instead of a strcmp(), as iter->trace->name
2470 * will point to the same string as current_trace->name.
2471 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002472 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002473 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2474 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002475 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002476
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002477#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002478 if (iter->snapshot && iter->trace->use_max_tr)
2479 return ERR_PTR(-EBUSY);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002480#endif
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002481
2482 if (!iter->snapshot)
2483 atomic_inc(&trace_record_cmdline_disabled);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002484
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002485 if (*pos != iter->pos) {
2486 iter->ent = NULL;
2487 iter->cpu = 0;
2488 iter->idx = -1;
2489
Steven Rostedtae3b5092013-01-23 15:22:59 -05002490 if (cpu_file == RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002491 for_each_tracing_cpu(cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002492 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002493 } else
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002494 tracing_iter_reset(iter, cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002495
Lai Jiangshanac91d852010-03-02 17:54:50 +08002496 iter->leftover = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002497 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2498 ;
2499
2500 } else {
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002501 /*
2502 * If we overflowed the seq_file before, then we want
2503 * to just reuse the trace_seq buffer again.
2504 */
2505 if (iter->leftover)
2506 p = iter;
2507 else {
2508 l = *pos - 1;
2509 p = s_next(m, p, &l);
2510 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002511 }
2512
Lai Jiangshan4f535962009-05-18 19:35:34 +08002513 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002514 trace_access_lock(cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002515 return p;
2516}
2517
2518static void s_stop(struct seq_file *m, void *p)
2519{
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002520 struct trace_iterator *iter = m->private;
2521
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002522#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002523 if (iter->snapshot && iter->trace->use_max_tr)
2524 return;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002525#endif
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002526
2527 if (!iter->snapshot)
2528 atomic_dec(&trace_record_cmdline_disabled);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002529
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002530 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08002531 trace_event_read_unlock();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002532}
2533
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002534static void
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002535get_total_entries(struct trace_buffer *buf,
2536 unsigned long *total, unsigned long *entries)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002537{
2538 unsigned long count;
2539 int cpu;
2540
2541 *total = 0;
2542 *entries = 0;
2543
2544 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002545 count = ring_buffer_entries_cpu(buf->buffer, cpu);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002546 /*
2547 * If this buffer has skipped entries, then we hold all
2548 * entries for the trace and we need to ignore the
2549 * ones before the time stamp.
2550 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002551 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2552 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002553 /* total is the same as the entries */
2554 *total += count;
2555 } else
2556 *total += count +
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002557 ring_buffer_overrun_cpu(buf->buffer, cpu);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002558 *entries += count;
2559 }
2560}
2561
Ingo Molnare309b412008-05-12 21:20:51 +02002562static void print_lat_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002563{
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002564 seq_puts(m, "# _------=> CPU# \n"
2565 "# / _-----=> irqs-off \n"
2566 "# | / _----=> need-resched \n"
2567 "# || / _---=> hardirq/softirq \n"
2568 "# ||| / _--=> preempt-depth \n"
2569 "# |||| / delay \n"
2570 "# cmd pid ||||| time | caller \n"
2571 "# \\ / ||||| \\ | / \n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002572}
2573
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002574static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002575{
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002576 unsigned long total;
2577 unsigned long entries;
2578
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002579 get_total_entries(buf, &total, &entries);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002580 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2581 entries, total, num_online_cpus());
2582 seq_puts(m, "#\n");
2583}
2584
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002585static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002586{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002587 print_event_info(buf, m);
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002588 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2589 "# | | | | |\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002590}
2591
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002592static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedt77271ce2011-11-17 09:34:33 -05002593{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002594 print_event_info(buf, m);
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002595 seq_puts(m, "# _-----=> irqs-off\n"
2596 "# / _----=> need-resched\n"
2597 "# | / _---=> hardirq/softirq\n"
2598 "# || / _--=> preempt-depth\n"
2599 "# ||| / delay\n"
2600 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2601 "# | | | |||| | |\n");
Steven Rostedt77271ce2011-11-17 09:34:33 -05002602}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002603
Jiri Olsa62b915f2010-04-02 19:01:22 +02002604void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002605print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2606{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002607 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002608 struct trace_buffer *buf = iter->trace_buffer;
2609 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002610 struct tracer *type = iter->trace;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002611 unsigned long entries;
2612 unsigned long total;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002613 const char *name = "preemption";
2614
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05002615 name = type->name;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002616
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002617 get_total_entries(buf, &total, &entries);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002618
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002619 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002620 name, UTS_RELEASE);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002621 seq_puts(m, "# -----------------------------------"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002622 "---------------------------------\n");
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002623 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002624 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
Steven Rostedt57f50be2008-05-12 21:20:44 +02002625 nsecs_to_usecs(data->saved_latency),
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002626 entries,
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02002627 total,
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002628 buf->cpu,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002629#if defined(CONFIG_PREEMPT_NONE)
2630 "server",
2631#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2632 "desktop",
Steven Rostedtb5c21b42008-07-10 20:58:12 -04002633#elif defined(CONFIG_PREEMPT)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002634 "preempt",
2635#else
2636 "unknown",
2637#endif
2638 /* These are reserved for later use */
2639 0, 0, 0, 0);
2640#ifdef CONFIG_SMP
2641 seq_printf(m, " #P:%d)\n", num_online_cpus());
2642#else
2643 seq_puts(m, ")\n");
2644#endif
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002645 seq_puts(m, "# -----------------\n");
2646 seq_printf(m, "# | task: %.16s-%d "
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002647 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
Eric W. Biedermand20b92a2012-03-13 16:02:19 -07002648 data->comm, data->pid,
2649 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002650 data->policy, data->rt_priority);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002651 seq_puts(m, "# -----------------\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002652
2653 if (data->critical_start) {
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002654 seq_puts(m, "# => started at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002655 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2656 trace_print_seq(m, &iter->seq);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002657 seq_puts(m, "\n# => ended at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002658 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2659 trace_print_seq(m, &iter->seq);
Steven Rostedt8248ac02009-09-02 12:27:41 -04002660 seq_puts(m, "\n#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002661 }
2662
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002663 seq_puts(m, "#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002664}
2665
Steven Rostedta3097202008-11-07 22:36:02 -05002666static void test_cpu_buff_start(struct trace_iterator *iter)
2667{
2668 struct trace_seq *s = &iter->seq;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002669 struct trace_array *tr = iter->tr;
Steven Rostedta3097202008-11-07 22:36:02 -05002670
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002671 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002672 return;
2673
2674 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2675 return;
2676
Sasha Levin919cd972015-09-04 12:45:56 -04002677 if (iter->started && cpumask_test_cpu(iter->cpu, iter->started))
Steven Rostedta3097202008-11-07 22:36:02 -05002678 return;
2679
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002680 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002681 return;
2682
Sasha Levin919cd972015-09-04 12:45:56 -04002683 if (iter->started)
2684 cpumask_set_cpu(iter->cpu, iter->started);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002685
2686 /* Don't print started cpu buffer for the first entry of the trace */
2687 if (iter->idx > 1)
2688 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2689 iter->cpu);
Steven Rostedta3097202008-11-07 22:36:02 -05002690}
2691
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002692static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002693{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002694 struct trace_array *tr = iter->tr;
Steven Rostedt214023c2008-05-12 21:20:46 +02002695 struct trace_seq *s = &iter->seq;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002696 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002697 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002698 struct trace_event *event;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002699
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002700 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002701
Steven Rostedta3097202008-11-07 22:36:02 -05002702 test_cpu_buff_start(iter);
2703
Steven Rostedtf633cef2008-12-23 23:24:13 -05002704 event = ftrace_find_event(entry->type);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002705
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002706 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002707 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2708 trace_print_lat_context(iter);
2709 else
2710 trace_print_context(iter);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002711 }
2712
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002713 if (trace_seq_has_overflowed(s))
2714 return TRACE_TYPE_PARTIAL_LINE;
2715
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002716 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002717 return event->funcs->trace(iter, sym_flags, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002718
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002719 trace_seq_printf(s, "Unknown type %d\n", entry->type);
Steven Rostedt7104f302008-10-01 10:52:51 -04002720
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002721 return trace_handle_return(s);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002722}
2723
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002724static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002725{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002726 struct trace_array *tr = iter->tr;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002727 struct trace_seq *s = &iter->seq;
2728 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002729 struct trace_event *event;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002730
2731 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002732
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002733 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002734 trace_seq_printf(s, "%d %d %llu ",
2735 entry->pid, iter->cpu, iter->ts);
2736
2737 if (trace_seq_has_overflowed(s))
2738 return TRACE_TYPE_PARTIAL_LINE;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002739
Steven Rostedtf633cef2008-12-23 23:24:13 -05002740 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002741 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002742 return event->funcs->raw(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002743
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002744 trace_seq_printf(s, "%d ?\n", entry->type);
Steven Rostedt7104f302008-10-01 10:52:51 -04002745
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002746 return trace_handle_return(s);
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002747}
2748
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002749static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002750{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002751 struct trace_array *tr = iter->tr;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002752 struct trace_seq *s = &iter->seq;
2753 unsigned char newline = '\n';
2754 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002755 struct trace_event *event;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002756
2757 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002758
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002759 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002760 SEQ_PUT_HEX_FIELD(s, entry->pid);
2761 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2762 SEQ_PUT_HEX_FIELD(s, iter->ts);
2763 if (trace_seq_has_overflowed(s))
2764 return TRACE_TYPE_PARTIAL_LINE;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002765 }
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002766
Steven Rostedtf633cef2008-12-23 23:24:13 -05002767 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002768 if (event) {
Steven Rostedta9a57762010-04-22 18:46:14 -04002769 enum print_line_t ret = event->funcs->hex(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002770 if (ret != TRACE_TYPE_HANDLED)
2771 return ret;
2772 }
Steven Rostedt7104f302008-10-01 10:52:51 -04002773
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002774 SEQ_PUT_FIELD(s, newline);
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002775
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002776 return trace_handle_return(s);
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002777}
2778
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002779static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002780{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002781 struct trace_array *tr = iter->tr;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002782 struct trace_seq *s = &iter->seq;
2783 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002784 struct trace_event *event;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002785
2786 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002787
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002788 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002789 SEQ_PUT_FIELD(s, entry->pid);
2790 SEQ_PUT_FIELD(s, iter->cpu);
2791 SEQ_PUT_FIELD(s, iter->ts);
2792 if (trace_seq_has_overflowed(s))
2793 return TRACE_TYPE_PARTIAL_LINE;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002794 }
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002795
Steven Rostedtf633cef2008-12-23 23:24:13 -05002796 event = ftrace_find_event(entry->type);
Steven Rostedta9a57762010-04-22 18:46:14 -04002797 return event ? event->funcs->binary(iter, 0, event) :
2798 TRACE_TYPE_HANDLED;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002799}
2800
Jiri Olsa62b915f2010-04-02 19:01:22 +02002801int trace_empty(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002802{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002803 struct ring_buffer_iter *buf_iter;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002804 int cpu;
2805
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002806 /* If we are looking at one CPU buffer, only check that one */
Steven Rostedtae3b5092013-01-23 15:22:59 -05002807 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002808 cpu = iter->cpu_file;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002809 buf_iter = trace_buffer_iter(iter, cpu);
2810 if (buf_iter) {
2811 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002812 return 0;
2813 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002814 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002815 return 0;
2816 }
2817 return 1;
2818 }
2819
Steven Rostedtab464282008-05-12 21:21:00 +02002820 for_each_tracing_cpu(cpu) {
Steven Rostedt6d158a82012-06-27 20:46:14 -04002821 buf_iter = trace_buffer_iter(iter, cpu);
2822 if (buf_iter) {
2823 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedtd7690412008-10-01 00:29:53 -04002824 return 0;
2825 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002826 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
Steven Rostedtd7690412008-10-01 00:29:53 -04002827 return 0;
2828 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002829 }
Steven Rostedtd7690412008-10-01 00:29:53 -04002830
Frederic Weisbecker797d3712008-09-30 18:13:45 +02002831 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002832}
2833
Lai Jiangshan4f535962009-05-18 19:35:34 +08002834/* Called with trace_event_read_lock() held. */
Jason Wessel955b61e2010-08-05 09:22:23 -05002835enum print_line_t print_trace_line(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002836{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002837 struct trace_array *tr = iter->tr;
2838 unsigned long trace_flags = tr->trace_flags;
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002839 enum print_line_t ret;
2840
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002841 if (iter->lost_events) {
2842 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2843 iter->cpu, iter->lost_events);
2844 if (trace_seq_has_overflowed(&iter->seq))
2845 return TRACE_TYPE_PARTIAL_LINE;
2846 }
Steven Rostedtbc21b472010-03-31 19:49:26 -04002847
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002848 if (iter->trace && iter->trace->print_line) {
2849 ret = iter->trace->print_line(iter);
2850 if (ret != TRACE_TYPE_UNHANDLED)
2851 return ret;
2852 }
Thomas Gleixner72829bc2008-05-23 21:37:28 +02002853
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -05002854 if (iter->ent->type == TRACE_BPUTS &&
2855 trace_flags & TRACE_ITER_PRINTK &&
2856 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2857 return trace_print_bputs_msg_only(iter);
2858
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002859 if (iter->ent->type == TRACE_BPRINT &&
2860 trace_flags & TRACE_ITER_PRINTK &&
2861 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002862 return trace_print_bprintk_msg_only(iter);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002863
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002864 if (iter->ent->type == TRACE_PRINT &&
2865 trace_flags & TRACE_ITER_PRINTK &&
2866 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002867 return trace_print_printk_msg_only(iter);
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002868
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002869 if (trace_flags & TRACE_ITER_BIN)
2870 return print_bin_fmt(iter);
2871
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002872 if (trace_flags & TRACE_ITER_HEX)
2873 return print_hex_fmt(iter);
2874
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002875 if (trace_flags & TRACE_ITER_RAW)
2876 return print_raw_fmt(iter);
2877
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002878 return print_trace_fmt(iter);
2879}
2880
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002881void trace_latency_header(struct seq_file *m)
2882{
2883 struct trace_iterator *iter = m->private;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002884 struct trace_array *tr = iter->tr;
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002885
2886 /* print nothing if the buffers are empty */
2887 if (trace_empty(iter))
2888 return;
2889
2890 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2891 print_trace_header(m, iter);
2892
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002893 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002894 print_lat_help_header(m);
2895}
2896
Jiri Olsa62b915f2010-04-02 19:01:22 +02002897void trace_default_header(struct seq_file *m)
2898{
2899 struct trace_iterator *iter = m->private;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002900 struct trace_array *tr = iter->tr;
2901 unsigned long trace_flags = tr->trace_flags;
Jiri Olsa62b915f2010-04-02 19:01:22 +02002902
Jiri Olsaf56e7f82011-06-03 16:58:49 +02002903 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2904 return;
2905
Jiri Olsa62b915f2010-04-02 19:01:22 +02002906 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2907 /* print nothing if the buffers are empty */
2908 if (trace_empty(iter))
2909 return;
2910 print_trace_header(m, iter);
2911 if (!(trace_flags & TRACE_ITER_VERBOSE))
2912 print_lat_help_header(m);
2913 } else {
Steven Rostedt77271ce2011-11-17 09:34:33 -05002914 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2915 if (trace_flags & TRACE_ITER_IRQ_INFO)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002916 print_func_help_header_irq(iter->trace_buffer, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002917 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002918 print_func_help_header(iter->trace_buffer, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002919 }
Jiri Olsa62b915f2010-04-02 19:01:22 +02002920 }
2921}
2922
Steven Rostedte0a413f2011-09-29 21:26:16 -04002923static void test_ftrace_alive(struct seq_file *m)
2924{
2925 if (!ftrace_is_dead())
2926 return;
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002927 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2928 "# MAY BE MISSING FUNCTION EVENTS\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002929}
2930
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002931#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002932static void show_snapshot_main_help(struct seq_file *m)
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002933{
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002934 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2935 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2936 "# Takes a snapshot of the main buffer.\n"
2937 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2938 "# (Doesn't have to be '2' works with any number that\n"
2939 "# is not a '0' or '1')\n");
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002940}
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002941
2942static void show_snapshot_percpu_help(struct seq_file *m)
2943{
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002944 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002945#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002946 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2947 "# Takes a snapshot of the main buffer for this cpu.\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002948#else
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002949 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2950 "# Must use main snapshot file to allocate.\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002951#endif
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002952 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2953 "# (Doesn't have to be '2' works with any number that\n"
2954 "# is not a '0' or '1')\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002955}
2956
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002957static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2958{
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05002959 if (iter->tr->allocated_snapshot)
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002960 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002961 else
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002962 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002963
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002964 seq_puts(m, "# Snapshot commands:\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002965 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2966 show_snapshot_main_help(m);
2967 else
2968 show_snapshot_percpu_help(m);
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002969}
2970#else
2971/* Should never be called */
2972static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2973#endif
2974
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002975static int s_show(struct seq_file *m, void *v)
2976{
2977 struct trace_iterator *iter = v;
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002978 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002979
2980 if (iter->ent == NULL) {
2981 if (iter->tr) {
2982 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2983 seq_puts(m, "#\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002984 test_ftrace_alive(m);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002985 }
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002986 if (iter->snapshot && trace_empty(iter))
2987 print_snapshot_help(m, iter);
2988 else if (iter->trace && iter->trace->print_header)
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002989 iter->trace->print_header(m);
Jiri Olsa62b915f2010-04-02 19:01:22 +02002990 else
2991 trace_default_header(m);
2992
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002993 } else if (iter->leftover) {
2994 /*
2995 * If we filled the seq_file buffer earlier, we
2996 * want to just show it now.
2997 */
2998 ret = trace_print_seq(m, &iter->seq);
2999
3000 /* ret should this time be zero, but you never know */
3001 iter->leftover = ret;
3002
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003003 } else {
Ingo Molnarf9896bf2008-05-12 21:20:47 +02003004 print_trace_line(iter);
Steven Rostedta63ce5b2009-12-07 09:11:39 -05003005 ret = trace_print_seq(m, &iter->seq);
3006 /*
3007 * If we overflow the seq_file buffer, then it will
3008 * ask us for this data again at start up.
3009 * Use that instead.
3010 * ret is 0 if seq_file write succeeded.
3011 * -1 otherwise.
3012 */
3013 iter->leftover = ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003014 }
3015
3016 return 0;
3017}
3018
Oleg Nesterov649e9c72013-07-23 17:25:54 +02003019/*
3020 * Should be used after trace_array_get(), trace_types_lock
3021 * ensures that i_cdev was already initialized.
3022 */
3023static inline int tracing_get_cpu(struct inode *inode)
3024{
3025 if (inode->i_cdev) /* See trace_create_cpu_file() */
3026 return (long)inode->i_cdev - 1;
3027 return RING_BUFFER_ALL_CPUS;
3028}
3029
James Morris88e9d342009-09-22 16:43:43 -07003030static const struct seq_operations tracer_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003031 .start = s_start,
3032 .next = s_next,
3033 .stop = s_stop,
3034 .show = s_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003035};
3036
Ingo Molnare309b412008-05-12 21:20:51 +02003037static struct trace_iterator *
Oleg Nesterov6484c712013-07-23 17:26:10 +02003038__tracing_open(struct inode *inode, struct file *file, bool snapshot)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003039{
Oleg Nesterov6484c712013-07-23 17:26:10 +02003040 struct trace_array *tr = inode->i_private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003041 struct trace_iterator *iter;
Jiri Olsa50e18b92012-04-25 10:23:39 +02003042 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003043
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003044 if (tracing_disabled)
3045 return ERR_PTR(-ENODEV);
Steven Rostedt60a11772008-05-12 21:20:44 +02003046
Jiri Olsa50e18b92012-04-25 10:23:39 +02003047 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003048 if (!iter)
3049 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003050
Gil Fruchter72917232015-06-09 10:32:35 +03003051 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
Steven Rostedt6d158a82012-06-27 20:46:14 -04003052 GFP_KERNEL);
Dan Carpenter93574fc2012-07-11 09:35:08 +03003053 if (!iter->buffer_iter)
3054 goto release;
3055
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003056 /*
3057 * We make a copy of the current tracer to avoid concurrent
3058 * changes on it while we are reading.
3059 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003060 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003061 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003062 if (!iter->trace)
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003063 goto fail;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003064
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003065 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003066
Li Zefan79f55992009-06-15 14:58:26 +08003067 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02003068 goto fail;
3069
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003070 iter->tr = tr;
3071
3072#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003073 /* Currently only the top directory has a snapshot */
3074 if (tr->current_trace->print_max || snapshot)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003075 iter->trace_buffer = &tr->max_buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003076 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003077#endif
3078 iter->trace_buffer = &tr->trace_buffer;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003079 iter->snapshot = snapshot;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003080 iter->pos = -1;
Oleg Nesterov6484c712013-07-23 17:26:10 +02003081 iter->cpu_file = tracing_get_cpu(inode);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003082 mutex_init(&iter->mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003083
Markus Metzger8bba1bf2008-11-25 09:12:31 +01003084 /* Notify the tracer early; before we stop tracing. */
3085 if (iter->trace && iter->trace->open)
Markus Metzgera93751c2008-12-11 13:53:26 +01003086 iter->trace->open(iter);
Markus Metzger8bba1bf2008-11-25 09:12:31 +01003087
Steven Rostedt12ef7d42008-11-12 17:52:38 -05003088 /* Annotate start of buffers if we had overruns */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003089 if (ring_buffer_overruns(iter->trace_buffer->buffer))
Steven Rostedt12ef7d42008-11-12 17:52:38 -05003090 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3091
David Sharp8be07092012-11-13 12:18:22 -08003092 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09003093 if (trace_clocks[tr->clock_id].in_ns)
David Sharp8be07092012-11-13 12:18:22 -08003094 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3095
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003096 /* stop the trace while dumping if we are not opening "snapshot" */
3097 if (!iter->snapshot)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003098 tracing_stop_tr(tr);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04003099
Steven Rostedtae3b5092013-01-23 15:22:59 -05003100 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003101 for_each_tracing_cpu(cpu) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003102 iter->buffer_iter[cpu] =
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003103 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
David Miller72c9ddf2010-04-20 15:47:11 -07003104 }
3105 ring_buffer_read_prepare_sync();
3106 for_each_tracing_cpu(cpu) {
3107 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04003108 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003109 }
3110 } else {
3111 cpu = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003112 iter->buffer_iter[cpu] =
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003113 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
David Miller72c9ddf2010-04-20 15:47:11 -07003114 ring_buffer_read_prepare_sync();
3115 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04003116 tracing_iter_reset(iter, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003117 }
3118
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003119 mutex_unlock(&trace_types_lock);
3120
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003121 return iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003122
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003123 fail:
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003124 mutex_unlock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003125 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04003126 kfree(iter->buffer_iter);
Dan Carpenter93574fc2012-07-11 09:35:08 +03003127release:
Jiri Olsa50e18b92012-04-25 10:23:39 +02003128 seq_release_private(inode, file);
3129 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003130}
3131
3132int tracing_open_generic(struct inode *inode, struct file *filp)
3133{
Steven Rostedt60a11772008-05-12 21:20:44 +02003134 if (tracing_disabled)
3135 return -ENODEV;
3136
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003137 filp->private_data = inode->i_private;
3138 return 0;
3139}
3140
Geyslan G. Bem2e864212013-10-18 21:15:54 -03003141bool tracing_is_disabled(void)
3142{
3143 return (tracing_disabled) ? true: false;
3144}
3145
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003146/*
3147 * Open and update trace_array ref count.
3148 * Must have the current trace_array passed to it.
3149 */
Steven Rostedt (Red Hat)dcc30222013-07-02 20:30:52 -04003150static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003151{
3152 struct trace_array *tr = inode->i_private;
3153
3154 if (tracing_disabled)
3155 return -ENODEV;
3156
3157 if (trace_array_get(tr) < 0)
3158 return -ENODEV;
3159
3160 filp->private_data = inode->i_private;
3161
3162 return 0;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003163}
3164
Hannes Eder4fd27352009-02-10 19:44:12 +01003165static int tracing_release(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003166{
Oleg Nesterov6484c712013-07-23 17:26:10 +02003167 struct trace_array *tr = inode->i_private;
matt mooney907f2782010-09-27 19:04:53 -07003168 struct seq_file *m = file->private_data;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003169 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003170 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003171
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003172 if (!(file->f_mode & FMODE_READ)) {
Oleg Nesterov6484c712013-07-23 17:26:10 +02003173 trace_array_put(tr);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003174 return 0;
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003175 }
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003176
Oleg Nesterov6484c712013-07-23 17:26:10 +02003177 /* Writes do not use seq_file */
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003178 iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003179 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05003180
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003181 for_each_tracing_cpu(cpu) {
3182 if (iter->buffer_iter[cpu])
3183 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3184 }
3185
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003186 if (iter->trace && iter->trace->close)
3187 iter->trace->close(iter);
3188
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003189 if (!iter->snapshot)
3190 /* reenable tracing if it was previously enabled */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003191 tracing_start_tr(tr);
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003192
3193 __trace_array_put(tr);
3194
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003195 mutex_unlock(&trace_types_lock);
3196
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003197 mutex_destroy(&iter->mutex);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02003198 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003199 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04003200 kfree(iter->buffer_iter);
Jiri Olsa50e18b92012-04-25 10:23:39 +02003201 seq_release_private(inode, file);
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003202
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003203 return 0;
3204}
3205
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003206static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3207{
3208 struct trace_array *tr = inode->i_private;
3209
3210 trace_array_put(tr);
3211 return 0;
3212}
3213
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003214static int tracing_single_release_tr(struct inode *inode, struct file *file)
3215{
3216 struct trace_array *tr = inode->i_private;
3217
3218 trace_array_put(tr);
3219
3220 return single_release(inode, file);
3221}
3222
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003223static int tracing_open(struct inode *inode, struct file *file)
3224{
Oleg Nesterov6484c712013-07-23 17:26:10 +02003225 struct trace_array *tr = inode->i_private;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003226 struct trace_iterator *iter;
3227 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003228
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003229 if (trace_array_get(tr) < 0)
3230 return -ENODEV;
3231
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003232 /* If this file was open for write, then erase contents */
Oleg Nesterov6484c712013-07-23 17:26:10 +02003233 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3234 int cpu = tracing_get_cpu(inode);
3235
3236 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003237 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003238 else
Oleg Nesterov6484c712013-07-23 17:26:10 +02003239 tracing_reset(&tr->trace_buffer, cpu);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003240 }
3241
3242 if (file->f_mode & FMODE_READ) {
Oleg Nesterov6484c712013-07-23 17:26:10 +02003243 iter = __tracing_open(inode, file, false);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003244 if (IS_ERR(iter))
3245 ret = PTR_ERR(iter);
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003246 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003247 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3248 }
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003249
3250 if (ret < 0)
3251 trace_array_put(tr);
3252
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003253 return ret;
3254}
3255
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003256/*
3257 * Some tracers are not suitable for instance buffers.
3258 * A tracer is always available for the global array (toplevel)
3259 * or if it explicitly states that it is.
3260 */
3261static bool
3262trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3263{
3264 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3265}
3266
3267/* Find the next tracer that this trace array may use */
3268static struct tracer *
3269get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3270{
3271 while (t && !trace_ok_for_array(t, tr))
3272 t = t->next;
3273
3274 return t;
3275}
3276
Ingo Molnare309b412008-05-12 21:20:51 +02003277static void *
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003278t_next(struct seq_file *m, void *v, loff_t *pos)
3279{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003280 struct trace_array *tr = m->private;
Li Zefanf129e962009-06-24 09:53:44 +08003281 struct tracer *t = v;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003282
3283 (*pos)++;
3284
3285 if (t)
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003286 t = get_tracer_for_array(tr, t->next);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003287
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003288 return t;
3289}
3290
3291static void *t_start(struct seq_file *m, loff_t *pos)
3292{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003293 struct trace_array *tr = m->private;
Li Zefanf129e962009-06-24 09:53:44 +08003294 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003295 loff_t l = 0;
3296
3297 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003298
3299 t = get_tracer_for_array(tr, trace_types);
3300 for (; t && l < *pos; t = t_next(m, t, &l))
3301 ;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003302
3303 return t;
3304}
3305
3306static void t_stop(struct seq_file *m, void *p)
3307{
3308 mutex_unlock(&trace_types_lock);
3309}
3310
3311static int t_show(struct seq_file *m, void *v)
3312{
3313 struct tracer *t = v;
3314
3315 if (!t)
3316 return 0;
3317
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01003318 seq_puts(m, t->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003319 if (t->next)
3320 seq_putc(m, ' ');
3321 else
3322 seq_putc(m, '\n');
3323
3324 return 0;
3325}
3326
James Morris88e9d342009-09-22 16:43:43 -07003327static const struct seq_operations show_traces_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003328 .start = t_start,
3329 .next = t_next,
3330 .stop = t_stop,
3331 .show = t_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003332};
3333
3334static int show_traces_open(struct inode *inode, struct file *file)
3335{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003336 struct trace_array *tr = inode->i_private;
3337 struct seq_file *m;
3338 int ret;
3339
Steven Rostedt60a11772008-05-12 21:20:44 +02003340 if (tracing_disabled)
3341 return -ENODEV;
3342
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003343 ret = seq_open(file, &show_traces_seq_ops);
3344 if (ret)
3345 return ret;
3346
3347 m = file->private_data;
3348 m->private = tr;
3349
3350 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003351}
3352
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003353static ssize_t
3354tracing_write_stub(struct file *filp, const char __user *ubuf,
3355 size_t count, loff_t *ppos)
3356{
3357 return count;
3358}
3359
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003360loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
Slava Pestov364829b2010-11-24 15:13:16 -08003361{
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003362 int ret;
3363
Slava Pestov364829b2010-11-24 15:13:16 -08003364 if (file->f_mode & FMODE_READ)
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003365 ret = seq_lseek(file, offset, whence);
Slava Pestov364829b2010-11-24 15:13:16 -08003366 else
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003367 file->f_pos = ret = 0;
3368
3369 return ret;
Slava Pestov364829b2010-11-24 15:13:16 -08003370}
3371
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003372static const struct file_operations tracing_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003373 .open = tracing_open,
3374 .read = seq_read,
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003375 .write = tracing_write_stub,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003376 .llseek = tracing_lseek,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003377 .release = tracing_release,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003378};
3379
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003380static const struct file_operations show_traces_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003381 .open = show_traces_open,
3382 .read = seq_read,
3383 .release = seq_release,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003384 .llseek = seq_lseek,
Ingo Molnarc7078de2008-05-12 21:20:52 +02003385};
3386
Ingo Molnar36dfe922008-05-12 21:20:52 +02003387/*
Ingo Molnar36dfe922008-05-12 21:20:52 +02003388 * The tracer itself will not take this lock, but still we want
3389 * to provide a consistent cpumask to user-space:
3390 */
3391static DEFINE_MUTEX(tracing_cpumask_update_lock);
3392
3393/*
3394 * Temporary storage for the character representation of the
3395 * CPU bitmask (and one more byte for the newline):
3396 */
3397static char mask_str[NR_CPUS + 1];
3398
Ingo Molnarc7078de2008-05-12 21:20:52 +02003399static ssize_t
3400tracing_cpumask_read(struct file *filp, char __user *ubuf,
3401 size_t count, loff_t *ppos)
3402{
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003403 struct trace_array *tr = file_inode(filp)->i_private;
Ingo Molnar36dfe922008-05-12 21:20:52 +02003404 int len;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003405
3406 mutex_lock(&tracing_cpumask_update_lock);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003407
Tejun Heo1a402432015-02-13 14:37:39 -08003408 len = snprintf(mask_str, count, "%*pb\n",
3409 cpumask_pr_args(tr->tracing_cpumask));
3410 if (len >= count) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02003411 count = -EINVAL;
3412 goto out_err;
3413 }
Ingo Molnar36dfe922008-05-12 21:20:52 +02003414 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3415
3416out_err:
Ingo Molnarc7078de2008-05-12 21:20:52 +02003417 mutex_unlock(&tracing_cpumask_update_lock);
3418
3419 return count;
3420}
3421
3422static ssize_t
3423tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3424 size_t count, loff_t *ppos)
3425{
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003426 struct trace_array *tr = file_inode(filp)->i_private;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303427 cpumask_var_t tracing_cpumask_new;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003428 int err, cpu;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303429
3430 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3431 return -ENOMEM;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003432
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303433 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003434 if (err)
3435 goto err_unlock;
3436
Li Zefan215368e2009-06-15 10:56:42 +08003437 mutex_lock(&tracing_cpumask_update_lock);
3438
Steven Rostedta5e25882008-12-02 15:34:05 -05003439 local_irq_disable();
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05003440 arch_spin_lock(&tr->max_lock);
Steven Rostedtab464282008-05-12 21:21:00 +02003441 for_each_tracing_cpu(cpu) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02003442 /*
3443 * Increase/decrease the disabled counter if we are
3444 * about to flip a bit in the cpumask:
3445 */
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003446 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303447 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003448 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3449 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003450 }
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003451 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303452 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003453 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3454 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003455 }
3456 }
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05003457 arch_spin_unlock(&tr->max_lock);
Steven Rostedta5e25882008-12-02 15:34:05 -05003458 local_irq_enable();
Ingo Molnar36dfe922008-05-12 21:20:52 +02003459
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003460 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003461
Ingo Molnarc7078de2008-05-12 21:20:52 +02003462 mutex_unlock(&tracing_cpumask_update_lock);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303463 free_cpumask_var(tracing_cpumask_new);
Ingo Molnarc7078de2008-05-12 21:20:52 +02003464
Ingo Molnarc7078de2008-05-12 21:20:52 +02003465 return count;
Ingo Molnar36dfe922008-05-12 21:20:52 +02003466
3467err_unlock:
Li Zefan215368e2009-06-15 10:56:42 +08003468 free_cpumask_var(tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003469
3470 return err;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003471}
3472
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003473static const struct file_operations tracing_cpumask_fops = {
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003474 .open = tracing_open_generic_tr,
Ingo Molnarc7078de2008-05-12 21:20:52 +02003475 .read = tracing_cpumask_read,
3476 .write = tracing_cpumask_write,
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003477 .release = tracing_release_generic_tr,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003478 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003479};
3480
Li Zefanfdb372e2009-12-08 11:15:59 +08003481static int tracing_trace_options_show(struct seq_file *m, void *v)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003482{
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003483 struct tracer_opt *trace_opts;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003484 struct trace_array *tr = m->private;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003485 u32 tracer_flags;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003486 int i;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003487
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003488 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003489 tracer_flags = tr->current_trace->flags->val;
3490 trace_opts = tr->current_trace->flags->opts;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003491
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003492 for (i = 0; trace_options[i]; i++) {
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003493 if (tr->trace_flags & (1 << i))
Li Zefanfdb372e2009-12-08 11:15:59 +08003494 seq_printf(m, "%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003495 else
Li Zefanfdb372e2009-12-08 11:15:59 +08003496 seq_printf(m, "no%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003497 }
3498
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003499 for (i = 0; trace_opts[i].name; i++) {
3500 if (tracer_flags & trace_opts[i].bit)
Li Zefanfdb372e2009-12-08 11:15:59 +08003501 seq_printf(m, "%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003502 else
Li Zefanfdb372e2009-12-08 11:15:59 +08003503 seq_printf(m, "no%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003504 }
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003505 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003506
Li Zefanfdb372e2009-12-08 11:15:59 +08003507 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003508}
3509
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003510static int __set_tracer_option(struct trace_array *tr,
Li Zefan8d18eaa2009-12-08 11:17:06 +08003511 struct tracer_flags *tracer_flags,
3512 struct tracer_opt *opts, int neg)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003513{
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003514 struct tracer *trace = tr->current_trace;
Li Zefan8d18eaa2009-12-08 11:17:06 +08003515 int ret;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003516
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003517 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003518 if (ret)
3519 return ret;
3520
3521 if (neg)
Zhaolei77708412009-08-07 18:53:21 +08003522 tracer_flags->val &= ~opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003523 else
Zhaolei77708412009-08-07 18:53:21 +08003524 tracer_flags->val |= opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003525 return 0;
3526}
3527
Li Zefan8d18eaa2009-12-08 11:17:06 +08003528/* Try to assign a tracer specific option */
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003529static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
Li Zefan8d18eaa2009-12-08 11:17:06 +08003530{
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003531 struct tracer *trace = tr->current_trace;
Li Zefan8d18eaa2009-12-08 11:17:06 +08003532 struct tracer_flags *tracer_flags = trace->flags;
3533 struct tracer_opt *opts = NULL;
3534 int i;
3535
3536 for (i = 0; tracer_flags->opts[i].name; i++) {
3537 opts = &tracer_flags->opts[i];
3538
3539 if (strcmp(cmp, opts->name) == 0)
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003540 return __set_tracer_option(tr, trace->flags, opts, neg);
Li Zefan8d18eaa2009-12-08 11:17:06 +08003541 }
3542
3543 return -EINVAL;
3544}
3545
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003546/* Some tracers require overwrite to stay enabled */
3547int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3548{
3549 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3550 return -1;
3551
3552 return 0;
3553}
3554
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003555int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003556{
3557 /* do nothing if flag is already set */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003558 if (!!(tr->trace_flags & mask) == !!enabled)
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003559 return 0;
3560
3561 /* Give the tracer a chance to approve the change */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003562 if (tr->current_trace->flag_changed)
Steven Rostedt (Red Hat)bf6065b2014-01-10 17:51:01 -05003563 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003564 return -EINVAL;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003565
3566 if (enabled)
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003567 tr->trace_flags |= mask;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003568 else
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003569 tr->trace_flags &= ~mask;
Li Zefane870e9a2010-07-02 11:07:32 +08003570
3571 if (mask == TRACE_ITER_RECORD_CMD)
3572 trace_event_enable_cmd_record(enabled);
David Sharp750912f2010-12-08 13:46:47 -08003573
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003574 if (mask == TRACE_ITER_OVERWRITE) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003575 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003576#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003577 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003578#endif
3579 }
Steven Rostedt81698832012-10-11 10:15:05 -04003580
Steven Rostedt (Red Hat)b9f91082015-09-29 18:21:35 -04003581 if (mask == TRACE_ITER_PRINTK) {
Steven Rostedt81698832012-10-11 10:15:05 -04003582 trace_printk_start_stop_comm(enabled);
Steven Rostedt (Red Hat)b9f91082015-09-29 18:21:35 -04003583 trace_printk_control(enabled);
3584 }
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003585
3586 return 0;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003587}
3588
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003589static int trace_set_options(struct trace_array *tr, char *option)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003590{
Li Zefan8d18eaa2009-12-08 11:17:06 +08003591 char *cmp;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003592 int neg = 0;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003593 int ret = -ENODEV;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003594 int i;
Jiaxing Wanga4d1e682015-11-04 09:14:29 +08003595 size_t orig_len = strlen(option);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003596
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003597 cmp = strstrip(option);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003598
Li Zefan8d18eaa2009-12-08 11:17:06 +08003599 if (strncmp(cmp, "no", 2) == 0) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003600 neg = 1;
3601 cmp += 2;
3602 }
3603
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003604 mutex_lock(&trace_types_lock);
3605
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003606 for (i = 0; trace_options[i]; i++) {
Li Zefan8d18eaa2009-12-08 11:17:06 +08003607 if (strcmp(cmp, trace_options[i]) == 0) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003608 ret = set_tracer_flag(tr, 1 << i, !neg);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003609 break;
3610 }
3611 }
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003612
3613 /* If no option could be set, test the specific tracer options */
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003614 if (!trace_options[i])
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003615 ret = set_tracer_option(tr, cmp, neg);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003616
3617 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003618
Jiaxing Wanga4d1e682015-11-04 09:14:29 +08003619 /*
3620 * If the first trailing whitespace is replaced with '\0' by strstrip,
3621 * turn it back into a space.
3622 */
3623 if (orig_len > strlen(option))
3624 option[strlen(option)] = ' ';
3625
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003626 return ret;
3627}
3628
Jiaxing Wanga4d1e682015-11-04 09:14:29 +08003629static void __init apply_trace_boot_options(void)
3630{
3631 char *buf = trace_boot_options_buf;
3632 char *option;
3633
3634 while (true) {
3635 option = strsep(&buf, ",");
3636
3637 if (!option)
3638 break;
Jiaxing Wanga4d1e682015-11-04 09:14:29 +08003639
Steven Rostedt (Red Hat)43ed3842015-11-03 22:15:14 -05003640 if (*option)
3641 trace_set_options(&global_trace, option);
Jiaxing Wanga4d1e682015-11-04 09:14:29 +08003642
3643 /* Put back the comma to allow this to be called again */
3644 if (buf)
3645 *(buf - 1) = ',';
3646 }
3647}
3648
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003649static ssize_t
3650tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3651 size_t cnt, loff_t *ppos)
3652{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003653 struct seq_file *m = filp->private_data;
3654 struct trace_array *tr = m->private;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003655 char buf[64];
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003656 int ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003657
3658 if (cnt >= sizeof(buf))
3659 return -EINVAL;
3660
3661 if (copy_from_user(&buf, ubuf, cnt))
3662 return -EFAULT;
3663
Steven Rostedta8dd2172013-01-09 20:54:17 -05003664 buf[cnt] = 0;
3665
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003666 ret = trace_set_options(tr, buf);
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003667 if (ret < 0)
3668 return ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003669
Jiri Olsacf8517c2009-10-23 19:36:16 -04003670 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003671
3672 return cnt;
3673}
3674
Li Zefanfdb372e2009-12-08 11:15:59 +08003675static int tracing_trace_options_open(struct inode *inode, struct file *file)
3676{
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003677 struct trace_array *tr = inode->i_private;
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003678 int ret;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003679
Li Zefanfdb372e2009-12-08 11:15:59 +08003680 if (tracing_disabled)
3681 return -ENODEV;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003682
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003683 if (trace_array_get(tr) < 0)
3684 return -ENODEV;
3685
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003686 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3687 if (ret < 0)
3688 trace_array_put(tr);
3689
3690 return ret;
Li Zefanfdb372e2009-12-08 11:15:59 +08003691}
3692
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003693static const struct file_operations tracing_iter_fops = {
Li Zefanfdb372e2009-12-08 11:15:59 +08003694 .open = tracing_trace_options_open,
3695 .read = seq_read,
3696 .llseek = seq_lseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003697 .release = tracing_single_release_tr,
Steven Rostedtee6bce52008-11-12 17:52:37 -05003698 .write = tracing_trace_options_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003699};
3700
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003701static const char readme_msg[] =
3702 "tracing mini-HOWTO:\n\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003703 "# echo 0 > tracing_on : quick way to disable tracing\n"
3704 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3705 " Important files:\n"
3706 " trace\t\t\t- The static contents of the buffer\n"
3707 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3708 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3709 " current_tracer\t- function and latency tracers\n"
3710 " available_tracers\t- list of configured tracers for current_tracer\n"
3711 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3712 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3713 " trace_clock\t\t-change the clock used to order events\n"
3714 " local: Per cpu clock but may not be synced across CPUs\n"
3715 " global: Synced across CPUs but slows tracing down.\n"
3716 " counter: Not a clock, but just an increment\n"
3717 " uptime: Jiffy counter from time of boot\n"
3718 " perf: Same clock that perf events use\n"
3719#ifdef CONFIG_X86_64
3720 " x86-tsc: TSC cycle counter\n"
3721#endif
3722 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3723 " tracing_cpumask\t- Limit which CPUs to trace\n"
3724 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3725 "\t\t\t Remove sub-buffer with rmdir\n"
3726 " trace_options\t\t- Set format or modify how tracing happens\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003727 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3728 "\t\t\t option name\n"
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003729 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003730#ifdef CONFIG_DYNAMIC_FTRACE
3731 "\n available_filter_functions - list of functions that can be filtered on\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003732 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3733 "\t\t\t functions\n"
3734 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3735 "\t modules: Can select a group via module\n"
3736 "\t Format: :mod:<module-name>\n"
3737 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3738 "\t triggers: a command to perform when function is hit\n"
3739 "\t Format: <function>:<trigger>[:count]\n"
3740 "\t trigger: traceon, traceoff\n"
3741 "\t\t enable_event:<system>:<event>\n"
3742 "\t\t disable_event:<system>:<event>\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003743#ifdef CONFIG_STACKTRACE
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003744 "\t\t stacktrace\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003745#endif
3746#ifdef CONFIG_TRACER_SNAPSHOT
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003747 "\t\t snapshot\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003748#endif
Steven Rostedt (Red Hat)17a280e2014-04-10 22:43:37 -04003749 "\t\t dump\n"
3750 "\t\t cpudump\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003751 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3752 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3753 "\t The first one will disable tracing every time do_fault is hit\n"
3754 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3755 "\t The first time do trap is hit and it disables tracing, the\n"
3756 "\t counter will decrement to 2. If tracing is already disabled,\n"
3757 "\t the counter will not decrement. It only decrements when the\n"
3758 "\t trigger did work\n"
3759 "\t To remove trigger without count:\n"
3760 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3761 "\t To remove trigger with a count:\n"
3762 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003763 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003764 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3765 "\t modules: Can select a group via module command :mod:\n"
3766 "\t Does not accept triggers\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003767#endif /* CONFIG_DYNAMIC_FTRACE */
3768#ifdef CONFIG_FUNCTION_TRACER
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003769 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3770 "\t\t (function)\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003771#endif
3772#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3773 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
Namhyung Kimd048a8c72014-06-13 01:23:53 +09003774 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003775 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3776#endif
3777#ifdef CONFIG_TRACER_SNAPSHOT
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003778 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3779 "\t\t\t snapshot buffer. Read the contents for more\n"
3780 "\t\t\t information\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003781#endif
zhangwei(Jovi)991821c2013-07-15 16:32:34 +08003782#ifdef CONFIG_STACK_TRACER
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003783 " stack_trace\t\t- Shows the max stack trace when active\n"
3784 " stack_max_size\t- Shows current max stack size that was traced\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003785 "\t\t\t Write into this file to reset the max size (trigger a\n"
3786 "\t\t\t new trace)\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003787#ifdef CONFIG_DYNAMIC_FTRACE
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003788 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3789 "\t\t\t traces\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003790#endif
zhangwei(Jovi)991821c2013-07-15 16:32:34 +08003791#endif /* CONFIG_STACK_TRACER */
Tom Zanussi26f25562014-01-17 15:11:44 -06003792 " events/\t\t- Directory containing all trace event subsystems:\n"
3793 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3794 " events/<system>/\t- Directory containing all trace events for <system>:\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003795 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3796 "\t\t\t events\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003797 " filter\t\t- If set, only events passing filter are traced\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003798 " events/<system>/<event>/\t- Directory containing control files for\n"
3799 "\t\t\t <event>:\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003800 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3801 " filter\t\t- If set, only events passing filter are traced\n"
3802 " trigger\t\t- If set, a command to perform when event is hit\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003803 "\t Format: <trigger>[:count][if <filter>]\n"
3804 "\t trigger: traceon, traceoff\n"
3805 "\t enable_event:<system>:<event>\n"
3806 "\t disable_event:<system>:<event>\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003807#ifdef CONFIG_STACKTRACE
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003808 "\t\t stacktrace\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003809#endif
3810#ifdef CONFIG_TRACER_SNAPSHOT
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003811 "\t\t snapshot\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003812#endif
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003813 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3814 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3815 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3816 "\t events/block/block_unplug/trigger\n"
3817 "\t The first disables tracing every time block_unplug is hit.\n"
3818 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3819 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3820 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3821 "\t Like function triggers, the counter is only decremented if it\n"
3822 "\t enabled or disabled tracing.\n"
3823 "\t To remove a trigger without a count:\n"
3824 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3825 "\t To remove a trigger with a count:\n"
3826 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3827 "\t Filters can be ignored when removing a trigger.\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003828;
3829
3830static ssize_t
3831tracing_readme_read(struct file *filp, char __user *ubuf,
3832 size_t cnt, loff_t *ppos)
3833{
3834 return simple_read_from_buffer(ubuf, cnt, ppos,
3835 readme_msg, strlen(readme_msg));
3836}
3837
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003838static const struct file_operations tracing_readme_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003839 .open = tracing_open_generic,
3840 .read = tracing_readme_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003841 .llseek = generic_file_llseek,
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003842};
3843
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003844static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
Avadh Patel69abe6a2009-04-10 16:04:48 -04003845{
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003846 unsigned int *ptr = v;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003847
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003848 if (*pos || m->count)
3849 ptr++;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003850
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003851 (*pos)++;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003852
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003853 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3854 ptr++) {
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003855 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
Avadh Patel69abe6a2009-04-10 16:04:48 -04003856 continue;
3857
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003858 return ptr;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003859 }
3860
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003861 return NULL;
3862}
Avadh Patel69abe6a2009-04-10 16:04:48 -04003863
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003864static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3865{
3866 void *v;
3867 loff_t l = 0;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003868
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04003869 preempt_disable();
3870 arch_spin_lock(&trace_cmdline_lock);
3871
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003872 v = &savedcmd->map_cmdline_to_pid[0];
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003873 while (l <= *pos) {
3874 v = saved_cmdlines_next(m, v, &l);
3875 if (!v)
3876 return NULL;
3877 }
3878
3879 return v;
3880}
3881
3882static void saved_cmdlines_stop(struct seq_file *m, void *v)
3883{
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04003884 arch_spin_unlock(&trace_cmdline_lock);
3885 preempt_enable();
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003886}
3887
3888static int saved_cmdlines_show(struct seq_file *m, void *v)
3889{
3890 char buf[TASK_COMM_LEN];
3891 unsigned int *pid = v;
3892
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04003893 __trace_find_cmdline(*pid, buf);
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003894 seq_printf(m, "%d %s\n", *pid, buf);
3895 return 0;
3896}
3897
3898static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3899 .start = saved_cmdlines_start,
3900 .next = saved_cmdlines_next,
3901 .stop = saved_cmdlines_stop,
3902 .show = saved_cmdlines_show,
3903};
3904
3905static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3906{
3907 if (tracing_disabled)
3908 return -ENODEV;
3909
3910 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
Avadh Patel69abe6a2009-04-10 16:04:48 -04003911}
3912
3913static const struct file_operations tracing_saved_cmdlines_fops = {
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003914 .open = tracing_saved_cmdlines_open,
3915 .read = seq_read,
3916 .llseek = seq_lseek,
3917 .release = seq_release,
Avadh Patel69abe6a2009-04-10 16:04:48 -04003918};
3919
3920static ssize_t
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003921tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3922 size_t cnt, loff_t *ppos)
3923{
3924 char buf[64];
3925 int r;
3926
3927 arch_spin_lock(&trace_cmdline_lock);
Namhyung Kima6af8fb2014-06-10 16:11:35 +09003928 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003929 arch_spin_unlock(&trace_cmdline_lock);
3930
3931 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3932}
3933
3934static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3935{
3936 kfree(s->saved_cmdlines);
3937 kfree(s->map_cmdline_to_pid);
3938 kfree(s);
3939}
3940
3941static int tracing_resize_saved_cmdlines(unsigned int val)
3942{
3943 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3944
Namhyung Kima6af8fb2014-06-10 16:11:35 +09003945 s = kmalloc(sizeof(*s), GFP_KERNEL);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003946 if (!s)
3947 return -ENOMEM;
3948
3949 if (allocate_cmdlines_buffer(val, s) < 0) {
3950 kfree(s);
3951 return -ENOMEM;
3952 }
3953
3954 arch_spin_lock(&trace_cmdline_lock);
3955 savedcmd_temp = savedcmd;
3956 savedcmd = s;
3957 arch_spin_unlock(&trace_cmdline_lock);
3958 free_saved_cmdlines_buffer(savedcmd_temp);
3959
3960 return 0;
3961}
3962
3963static ssize_t
3964tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3965 size_t cnt, loff_t *ppos)
3966{
3967 unsigned long val;
3968 int ret;
3969
3970 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3971 if (ret)
3972 return ret;
3973
3974 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3975 if (!val || val > PID_MAX_DEFAULT)
3976 return -EINVAL;
3977
3978 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3979 if (ret < 0)
3980 return ret;
3981
3982 *ppos += cnt;
3983
3984 return cnt;
3985}
3986
3987static const struct file_operations tracing_saved_cmdlines_size_fops = {
3988 .open = tracing_open_generic,
3989 .read = tracing_saved_cmdlines_size_read,
3990 .write = tracing_saved_cmdlines_size_write,
3991};
3992
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04003993#ifdef CONFIG_TRACE_ENUM_MAP_FILE
3994static union trace_enum_map_item *
3995update_enum_map(union trace_enum_map_item *ptr)
3996{
3997 if (!ptr->map.enum_string) {
3998 if (ptr->tail.next) {
3999 ptr = ptr->tail.next;
4000 /* Set ptr to the next real item (skip head) */
4001 ptr++;
4002 } else
4003 return NULL;
4004 }
4005 return ptr;
4006}
4007
4008static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
4009{
4010 union trace_enum_map_item *ptr = v;
4011
4012 /*
4013 * Paranoid! If ptr points to end, we don't want to increment past it.
4014 * This really should never happen.
4015 */
4016 ptr = update_enum_map(ptr);
4017 if (WARN_ON_ONCE(!ptr))
4018 return NULL;
4019
4020 ptr++;
4021
4022 (*pos)++;
4023
4024 ptr = update_enum_map(ptr);
4025
4026 return ptr;
4027}
4028
4029static void *enum_map_start(struct seq_file *m, loff_t *pos)
4030{
4031 union trace_enum_map_item *v;
4032 loff_t l = 0;
4033
4034 mutex_lock(&trace_enum_mutex);
4035
4036 v = trace_enum_maps;
4037 if (v)
4038 v++;
4039
4040 while (v && l < *pos) {
4041 v = enum_map_next(m, v, &l);
4042 }
4043
4044 return v;
4045}
4046
4047static void enum_map_stop(struct seq_file *m, void *v)
4048{
4049 mutex_unlock(&trace_enum_mutex);
4050}
4051
4052static int enum_map_show(struct seq_file *m, void *v)
4053{
4054 union trace_enum_map_item *ptr = v;
4055
4056 seq_printf(m, "%s %ld (%s)\n",
4057 ptr->map.enum_string, ptr->map.enum_value,
4058 ptr->map.system);
4059
4060 return 0;
4061}
4062
4063static const struct seq_operations tracing_enum_map_seq_ops = {
4064 .start = enum_map_start,
4065 .next = enum_map_next,
4066 .stop = enum_map_stop,
4067 .show = enum_map_show,
4068};
4069
4070static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4071{
4072 if (tracing_disabled)
4073 return -ENODEV;
4074
4075 return seq_open(filp, &tracing_enum_map_seq_ops);
4076}
4077
4078static const struct file_operations tracing_enum_map_fops = {
4079 .open = tracing_enum_map_open,
4080 .read = seq_read,
4081 .llseek = seq_lseek,
4082 .release = seq_release,
4083};
4084
4085static inline union trace_enum_map_item *
4086trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4087{
4088 /* Return tail of array given the head */
4089 return ptr + ptr->head.length + 1;
4090}
4091
4092static void
4093trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4094 int len)
4095{
4096 struct trace_enum_map **stop;
4097 struct trace_enum_map **map;
4098 union trace_enum_map_item *map_array;
4099 union trace_enum_map_item *ptr;
4100
4101 stop = start + len;
4102
4103 /*
4104 * The trace_enum_maps contains the map plus a head and tail item,
4105 * where the head holds the module and length of array, and the
4106 * tail holds a pointer to the next list.
4107 */
4108 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4109 if (!map_array) {
4110 pr_warning("Unable to allocate trace enum mapping\n");
4111 return;
4112 }
4113
4114 mutex_lock(&trace_enum_mutex);
4115
4116 if (!trace_enum_maps)
4117 trace_enum_maps = map_array;
4118 else {
4119 ptr = trace_enum_maps;
4120 for (;;) {
4121 ptr = trace_enum_jmp_to_tail(ptr);
4122 if (!ptr->tail.next)
4123 break;
4124 ptr = ptr->tail.next;
4125
4126 }
4127 ptr->tail.next = map_array;
4128 }
4129 map_array->head.mod = mod;
4130 map_array->head.length = len;
4131 map_array++;
4132
4133 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4134 map_array->map = **map;
4135 map_array++;
4136 }
4137 memset(map_array, 0, sizeof(*map_array));
4138
4139 mutex_unlock(&trace_enum_mutex);
4140}
4141
4142static void trace_create_enum_file(struct dentry *d_tracer)
4143{
4144 trace_create_file("enum_map", 0444, d_tracer,
4145 NULL, &tracing_enum_map_fops);
4146}
4147
4148#else /* CONFIG_TRACE_ENUM_MAP_FILE */
4149static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4150static inline void trace_insert_enum_map_file(struct module *mod,
4151 struct trace_enum_map **start, int len) { }
4152#endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4153
4154static void trace_insert_enum_map(struct module *mod,
4155 struct trace_enum_map **start, int len)
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04004156{
4157 struct trace_enum_map **map;
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04004158
4159 if (len <= 0)
4160 return;
4161
4162 map = start;
4163
4164 trace_event_enum_update(map, len);
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04004165
4166 trace_insert_enum_map_file(mod, start, len);
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04004167}
4168
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09004169static ssize_t
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004170tracing_set_trace_read(struct file *filp, char __user *ubuf,
4171 size_t cnt, loff_t *ppos)
4172{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004173 struct trace_array *tr = filp->private_data;
Li Zefanee6c2c12009-09-18 14:06:47 +08004174 char buf[MAX_TRACER_SIZE+2];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004175 int r;
4176
4177 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004178 r = sprintf(buf, "%s\n", tr->current_trace->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004179 mutex_unlock(&trace_types_lock);
4180
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004181 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004182}
4183
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02004184int tracer_init(struct tracer *t, struct trace_array *tr)
4185{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004186 tracing_reset_online_cpus(&tr->trace_buffer);
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02004187 return t->init(tr);
4188}
4189
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004190static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004191{
4192 int cpu;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05004193
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004194 for_each_tracing_cpu(cpu)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004195 per_cpu_ptr(buf->data, cpu)->entries = val;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004196}
4197
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004198#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookad60da502012-10-17 11:56:16 +09004199/* resize @tr's buffer to the size of @size_tr's entries */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004200static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4201 struct trace_buffer *size_buf, int cpu_id)
Hiraku Toyookad60da502012-10-17 11:56:16 +09004202{
4203 int cpu, ret = 0;
4204
4205 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4206 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004207 ret = ring_buffer_resize(trace_buf->buffer,
4208 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
Hiraku Toyookad60da502012-10-17 11:56:16 +09004209 if (ret < 0)
4210 break;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004211 per_cpu_ptr(trace_buf->data, cpu)->entries =
4212 per_cpu_ptr(size_buf->data, cpu)->entries;
Hiraku Toyookad60da502012-10-17 11:56:16 +09004213 }
4214 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004215 ret = ring_buffer_resize(trace_buf->buffer,
4216 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
Hiraku Toyookad60da502012-10-17 11:56:16 +09004217 if (ret == 0)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004218 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4219 per_cpu_ptr(size_buf->data, cpu_id)->entries;
Hiraku Toyookad60da502012-10-17 11:56:16 +09004220 }
4221
4222 return ret;
4223}
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004224#endif /* CONFIG_TRACER_MAX_TRACE */
Hiraku Toyookad60da502012-10-17 11:56:16 +09004225
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004226static int __tracing_resize_ring_buffer(struct trace_array *tr,
4227 unsigned long size, int cpu)
Steven Rostedt73c51622009-03-11 13:42:01 -04004228{
4229 int ret;
4230
4231 /*
4232 * If kernel or user changes the size of the ring buffer
Steven Rostedta123c522009-03-12 11:21:08 -04004233 * we use the size that was given, and we can forget about
4234 * expanding it later.
Steven Rostedt73c51622009-03-11 13:42:01 -04004235 */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05004236 ring_buffer_expanded = true;
Steven Rostedt73c51622009-03-11 13:42:01 -04004237
Steven Rostedtb382ede62012-10-10 21:44:34 -04004238 /* May be called before buffers are initialized */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004239 if (!tr->trace_buffer.buffer)
Steven Rostedtb382ede62012-10-10 21:44:34 -04004240 return 0;
4241
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004242 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04004243 if (ret < 0)
4244 return ret;
4245
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004246#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004247 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4248 !tr->current_trace->use_max_tr)
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004249 goto out;
4250
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004251 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04004252 if (ret < 0) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004253 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4254 &tr->trace_buffer, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04004255 if (r < 0) {
Steven Rostedta123c522009-03-12 11:21:08 -04004256 /*
4257 * AARGH! We are left with different
4258 * size max buffer!!!!
4259 * The max buffer is our "snapshot" buffer.
4260 * When a tracer needs a snapshot (one of the
4261 * latency tracers), it swaps the max buffer
4262 * with the saved snap shot. We succeeded to
4263 * update the size of the main buffer, but failed to
4264 * update the size of the max buffer. But when we tried
4265 * to reset the main buffer to the original size, we
4266 * failed there too. This is very unlikely to
4267 * happen, but if it does, warn and kill all
4268 * tracing.
4269 */
Steven Rostedt73c51622009-03-11 13:42:01 -04004270 WARN_ON(1);
4271 tracing_disabled = 1;
4272 }
4273 return ret;
4274 }
4275
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004276 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004277 set_buffer_entries(&tr->max_buffer, size);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004278 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004279 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004280
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004281 out:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004282#endif /* CONFIG_TRACER_MAX_TRACE */
4283
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004284 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004285 set_buffer_entries(&tr->trace_buffer, size);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004286 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004287 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
Steven Rostedt73c51622009-03-11 13:42:01 -04004288
4289 return ret;
4290}
4291
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004292static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4293 unsigned long size, int cpu_id)
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004294{
Vaibhav Nagarnaik83f40312012-05-03 18:59:50 -07004295 int ret = size;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004296
4297 mutex_lock(&trace_types_lock);
4298
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004299 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4300 /* make sure, this cpu is enabled in the mask */
4301 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4302 ret = -EINVAL;
4303 goto out;
4304 }
4305 }
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004306
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004307 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004308 if (ret < 0)
4309 ret = -ENOMEM;
4310
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004311out:
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004312 mutex_unlock(&trace_types_lock);
4313
4314 return ret;
4315}
4316
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004317
Steven Rostedt1852fcc2009-03-11 14:33:00 -04004318/**
4319 * tracing_update_buffers - used by tracing facility to expand ring buffers
4320 *
4321 * To save on memory when the tracing is never used on a system with it
4322 * configured in. The ring buffers are set to a minimum size. But once
4323 * a user starts to use the tracing facility, then they need to grow
4324 * to their default size.
4325 *
4326 * This function is to be called when a tracer is about to be used.
4327 */
4328int tracing_update_buffers(void)
4329{
4330 int ret = 0;
4331
Steven Rostedt1027fcb2009-03-12 11:33:20 -04004332 mutex_lock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04004333 if (!ring_buffer_expanded)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004334 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004335 RING_BUFFER_ALL_CPUS);
Steven Rostedt1027fcb2009-03-12 11:33:20 -04004336 mutex_unlock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04004337
4338 return ret;
4339}
4340
Steven Rostedt577b7852009-02-26 23:43:05 -05004341struct trace_option_dentry;
4342
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04004343static void
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004344create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
Steven Rostedt577b7852009-02-26 23:43:05 -05004345
Steven Rostedt (Red Hat)6b450d22014-01-14 08:43:01 -05004346/*
4347 * Used to clear out the tracer before deletion of an instance.
4348 * Must have trace_types_lock held.
4349 */
4350static void tracing_set_nop(struct trace_array *tr)
4351{
4352 if (tr->current_trace == &nop_trace)
4353 return;
4354
Steven Rostedt (Red Hat)50512ab2014-01-14 08:52:35 -05004355 tr->current_trace->enabled--;
Steven Rostedt (Red Hat)6b450d22014-01-14 08:43:01 -05004356
4357 if (tr->current_trace->reset)
4358 tr->current_trace->reset(tr);
4359
4360 tr->current_trace = &nop_trace;
4361}
4362
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04004363static void add_tracer_options(struct trace_array *tr, struct tracer *t)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004364{
Steven Rostedt (Red Hat)09d23a12015-02-03 12:45:53 -05004365 /* Only enable if the directory has been created already. */
4366 if (!tr->dir)
4367 return;
4368
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04004369 create_trace_option_files(tr, t);
Steven Rostedt (Red Hat)09d23a12015-02-03 12:45:53 -05004370}
4371
4372static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4373{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004374 struct tracer *t;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004375#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt34600f02013-01-22 13:35:11 -05004376 bool had_max_tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004377#endif
Peter Zijlstrad9e54072008-11-01 19:57:37 +01004378 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004379
Steven Rostedt1027fcb2009-03-12 11:33:20 -04004380 mutex_lock(&trace_types_lock);
4381
Steven Rostedt73c51622009-03-11 13:42:01 -04004382 if (!ring_buffer_expanded) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004383 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004384 RING_BUFFER_ALL_CPUS);
Steven Rostedt73c51622009-03-11 13:42:01 -04004385 if (ret < 0)
Frederic Weisbecker59f586d2009-03-15 22:10:39 +01004386 goto out;
Steven Rostedt73c51622009-03-11 13:42:01 -04004387 ret = 0;
4388 }
4389
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004390 for (t = trace_types; t; t = t->next) {
4391 if (strcmp(t->name, buf) == 0)
4392 break;
4393 }
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02004394 if (!t) {
4395 ret = -EINVAL;
4396 goto out;
4397 }
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004398 if (t == tr->current_trace)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004399 goto out;
4400
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05004401 /* Some tracers are only allowed for the top level buffer */
4402 if (!trace_ok_for_array(t, tr)) {
4403 ret = -EINVAL;
4404 goto out;
4405 }
4406
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05004407 /* If trace pipe files are being read, we can't change the tracer */
4408 if (tr->current_trace->ref) {
4409 ret = -EBUSY;
4410 goto out;
4411 }
4412
Steven Rostedt9f029e82008-11-12 15:24:24 -05004413 trace_branch_disable();
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04004414
Steven Rostedt (Red Hat)50512ab2014-01-14 08:52:35 -05004415 tr->current_trace->enabled--;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04004416
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004417 if (tr->current_trace->reset)
4418 tr->current_trace->reset(tr);
Steven Rostedt34600f02013-01-22 13:35:11 -05004419
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004420 /* Current trace needs to be nop_trace before synchronize_sched */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004421 tr->current_trace = &nop_trace;
Steven Rostedt34600f02013-01-22 13:35:11 -05004422
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05004423#ifdef CONFIG_TRACER_MAX_TRACE
4424 had_max_tr = tr->allocated_snapshot;
Steven Rostedt34600f02013-01-22 13:35:11 -05004425
4426 if (had_max_tr && !t->use_max_tr) {
4427 /*
4428 * We need to make sure that the update_max_tr sees that
4429 * current_trace changed to nop_trace to keep it from
4430 * swapping the buffers after we resize it.
4431 * The update_max_tr is called from interrupts disabled
4432 * so a synchronized_sched() is sufficient.
4433 */
4434 synchronize_sched();
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04004435 free_snapshot(tr);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004436 }
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004437#endif
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004438
4439#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt34600f02013-01-22 13:35:11 -05004440 if (t->use_max_tr && !had_max_tr) {
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04004441 ret = alloc_snapshot(tr);
Hiraku Toyookad60da502012-10-17 11:56:16 +09004442 if (ret < 0)
4443 goto out;
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004444 }
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004445#endif
Steven Rostedt577b7852009-02-26 23:43:05 -05004446
Frederic Weisbecker1c800252008-11-16 05:57:26 +01004447 if (t->init) {
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02004448 ret = tracer_init(t, tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +01004449 if (ret)
4450 goto out;
4451 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004452
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004453 tr->current_trace = t;
Steven Rostedt (Red Hat)50512ab2014-01-14 08:52:35 -05004454 tr->current_trace->enabled++;
Steven Rostedt9f029e82008-11-12 15:24:24 -05004455 trace_branch_enable(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004456 out:
4457 mutex_unlock(&trace_types_lock);
4458
Peter Zijlstrad9e54072008-11-01 19:57:37 +01004459 return ret;
4460}
4461
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004462static ssize_t
4463tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4464 size_t cnt, loff_t *ppos)
4465{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05004466 struct trace_array *tr = filp->private_data;
Li Zefanee6c2c12009-09-18 14:06:47 +08004467 char buf[MAX_TRACER_SIZE+1];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004468 int i;
4469 size_t ret;
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01004470 int err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004471
Steven Rostedt60063a62008-10-28 10:44:24 -04004472 ret = cnt;
4473
Li Zefanee6c2c12009-09-18 14:06:47 +08004474 if (cnt > MAX_TRACER_SIZE)
4475 cnt = MAX_TRACER_SIZE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004476
4477 if (copy_from_user(&buf, ubuf, cnt))
4478 return -EFAULT;
4479
4480 buf[cnt] = 0;
4481
4482 /* strip ending whitespace. */
4483 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4484 buf[i] = 0;
4485
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05004486 err = tracing_set_tracer(tr, buf);
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01004487 if (err)
4488 return err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004489
Jiri Olsacf8517c2009-10-23 19:36:16 -04004490 *ppos += ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004491
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02004492 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004493}
4494
4495static ssize_t
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04004496tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4497 size_t cnt, loff_t *ppos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004498{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004499 char buf[64];
4500 int r;
4501
Steven Rostedtcffae432008-05-12 21:21:00 +02004502 r = snprintf(buf, sizeof(buf), "%ld\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004503 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
Steven Rostedtcffae432008-05-12 21:21:00 +02004504 if (r > sizeof(buf))
4505 r = sizeof(buf);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004506 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004507}
4508
4509static ssize_t
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04004510tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4511 size_t cnt, loff_t *ppos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004512{
Hannes Eder5e398412009-02-10 19:44:34 +01004513 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02004514 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004515
Peter Huewe22fe9b52011-06-07 21:58:27 +02004516 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4517 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02004518 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004519
4520 *ptr = val * 1000;
4521
4522 return cnt;
4523}
4524
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04004525static ssize_t
4526tracing_thresh_read(struct file *filp, char __user *ubuf,
4527 size_t cnt, loff_t *ppos)
4528{
4529 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4530}
4531
4532static ssize_t
4533tracing_thresh_write(struct file *filp, const char __user *ubuf,
4534 size_t cnt, loff_t *ppos)
4535{
4536 struct trace_array *tr = filp->private_data;
4537 int ret;
4538
4539 mutex_lock(&trace_types_lock);
4540 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4541 if (ret < 0)
4542 goto out;
4543
4544 if (tr->current_trace->update_thresh) {
4545 ret = tr->current_trace->update_thresh(tr);
4546 if (ret < 0)
4547 goto out;
4548 }
4549
4550 ret = cnt;
4551out:
4552 mutex_unlock(&trace_types_lock);
4553
4554 return ret;
4555}
4556
4557static ssize_t
4558tracing_max_lat_read(struct file *filp, char __user *ubuf,
4559 size_t cnt, loff_t *ppos)
4560{
4561 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4562}
4563
4564static ssize_t
4565tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4566 size_t cnt, loff_t *ppos)
4567{
4568 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4569}
4570
Steven Rostedtb3806b42008-05-12 21:20:46 +02004571static int tracing_open_pipe(struct inode *inode, struct file *filp)
4572{
Oleg Nesterov15544202013-07-23 17:25:57 +02004573 struct trace_array *tr = inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004574 struct trace_iterator *iter;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004575 int ret = 0;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004576
4577 if (tracing_disabled)
4578 return -ENODEV;
4579
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004580 if (trace_array_get(tr) < 0)
4581 return -ENODEV;
4582
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004583 mutex_lock(&trace_types_lock);
4584
Steven Rostedtb3806b42008-05-12 21:20:46 +02004585 /* create a buffer to store the information to pass to userspace */
4586 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004587 if (!iter) {
4588 ret = -ENOMEM;
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07004589 __trace_array_put(tr);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004590 goto out;
4591 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02004592
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04004593 trace_seq_init(&iter->seq);
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05004594 iter->trace = tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004595
4596 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4597 ret = -ENOMEM;
4598 goto fail;
Rusty Russell44623442009-01-01 10:12:23 +10304599 }
4600
Steven Rostedta3097202008-11-07 22:36:02 -05004601 /* trace pipe does not show start of buffer */
Rusty Russell44623442009-01-01 10:12:23 +10304602 cpumask_setall(iter->started);
Steven Rostedta3097202008-11-07 22:36:02 -05004603
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04004604 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
Steven Rostedt112f38a72009-06-01 15:16:05 -04004605 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4606
David Sharp8be07092012-11-13 12:18:22 -08004607 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09004608 if (trace_clocks[tr->clock_id].in_ns)
David Sharp8be07092012-11-13 12:18:22 -08004609 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4610
Oleg Nesterov15544202013-07-23 17:25:57 +02004611 iter->tr = tr;
4612 iter->trace_buffer = &tr->trace_buffer;
4613 iter->cpu_file = tracing_get_cpu(inode);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004614 mutex_init(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004615 filp->private_data = iter;
4616
Steven Rostedt107bad82008-05-12 21:21:01 +02004617 if (iter->trace->pipe_open)
4618 iter->trace->pipe_open(iter);
Steven Rostedt107bad82008-05-12 21:21:01 +02004619
Arnd Bergmannb4447862010-07-07 23:40:11 +02004620 nonseekable_open(inode, filp);
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05004621
4622 tr->current_trace->ref++;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004623out:
4624 mutex_unlock(&trace_types_lock);
4625 return ret;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004626
4627fail:
4628 kfree(iter->trace);
4629 kfree(iter);
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004630 __trace_array_put(tr);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004631 mutex_unlock(&trace_types_lock);
4632 return ret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004633}
4634
4635static int tracing_release_pipe(struct inode *inode, struct file *file)
4636{
4637 struct trace_iterator *iter = file->private_data;
Oleg Nesterov15544202013-07-23 17:25:57 +02004638 struct trace_array *tr = inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004639
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004640 mutex_lock(&trace_types_lock);
4641
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05004642 tr->current_trace->ref--;
4643
Steven Rostedt29bf4a52009-12-09 12:37:43 -05004644 if (iter->trace->pipe_close)
Steven Rostedtc521efd2009-12-07 09:06:24 -05004645 iter->trace->pipe_close(iter);
4646
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004647 mutex_unlock(&trace_types_lock);
4648
Rusty Russell44623442009-01-01 10:12:23 +10304649 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004650 mutex_destroy(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004651 kfree(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004652
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004653 trace_array_put(tr);
4654
Steven Rostedtb3806b42008-05-12 21:20:46 +02004655 return 0;
4656}
4657
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004658static unsigned int
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004659trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004660{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04004661 struct trace_array *tr = iter->tr;
4662
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004663 /* Iterators are static, they should be filled or empty */
4664 if (trace_buffer_iter(iter, iter->cpu_file))
4665 return POLLIN | POLLRDNORM;
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004666
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04004667 if (tr->trace_flags & TRACE_ITER_BLOCK)
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004668 /*
4669 * Always select as readable when in blocking mode
4670 */
4671 return POLLIN | POLLRDNORM;
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004672 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004673 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004674 filp, poll_table);
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004675}
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004676
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004677static unsigned int
4678tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4679{
4680 struct trace_iterator *iter = filp->private_data;
4681
4682 return trace_poll(iter, filp, poll_table);
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004683}
4684
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05004685/* Must be called with iter->mutex held. */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004686static int tracing_wait_pipe(struct file *filp)
4687{
4688 struct trace_iterator *iter = filp->private_data;
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04004689 int ret;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004690
4691 while (trace_empty(iter)) {
4692
4693 if ((filp->f_flags & O_NONBLOCK)) {
4694 return -EAGAIN;
4695 }
4696
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004697 /*
Liu Bo250bfd32013-01-14 10:54:11 +08004698 * We block until we read something and tracing is disabled.
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004699 * We still block if tracing is disabled, but we have never
4700 * read anything. This allows a user to cat this file, and
4701 * then enable tracing. But after we have read something,
4702 * we give an EOF when tracing is again disabled.
4703 *
4704 * iter->pos will be 0 if we haven't read anything.
4705 */
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04004706 if (!tracing_is_on() && iter->pos)
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004707 break;
Steven Rostedt (Red Hat)f4874262014-04-29 16:07:28 -04004708
4709 mutex_unlock(&iter->mutex);
4710
Rabin Vincente30f53a2014-11-10 19:46:34 +01004711 ret = wait_on_pipe(iter, false);
Steven Rostedt (Red Hat)f4874262014-04-29 16:07:28 -04004712
4713 mutex_lock(&iter->mutex);
4714
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04004715 if (ret)
4716 return ret;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004717 }
4718
4719 return 1;
4720}
4721
Steven Rostedtb3806b42008-05-12 21:20:46 +02004722/*
4723 * Consumer reader.
4724 */
4725static ssize_t
4726tracing_read_pipe(struct file *filp, char __user *ubuf,
4727 size_t cnt, loff_t *ppos)
4728{
4729 struct trace_iterator *iter = filp->private_data;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004730 ssize_t sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004731
4732 /* return any leftover data */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004733 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4734 if (sret != -EBUSY)
4735 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004736
Steven Rostedtf9520752009-03-02 14:04:40 -05004737 trace_seq_init(&iter->seq);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004738
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004739 /*
4740 * Avoid more than one consumer on a single file descriptor
4741 * This is just a matter of traces coherency, the ring buffer itself
4742 * is protected.
4743 */
4744 mutex_lock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02004745 if (iter->trace->read) {
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004746 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4747 if (sret)
Steven Rostedt107bad82008-05-12 21:21:01 +02004748 goto out;
Steven Rostedt107bad82008-05-12 21:21:01 +02004749 }
4750
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004751waitagain:
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004752 sret = tracing_wait_pipe(filp);
4753 if (sret <= 0)
4754 goto out;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004755
4756 /* stop when tracing is finished */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004757 if (trace_empty(iter)) {
4758 sret = 0;
Steven Rostedt107bad82008-05-12 21:21:01 +02004759 goto out;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004760 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02004761
4762 if (cnt >= PAGE_SIZE)
4763 cnt = PAGE_SIZE - 1;
4764
Steven Rostedt53d0aa72008-05-12 21:21:01 +02004765 /* reset all but tr, trace, and overruns */
Steven Rostedt53d0aa72008-05-12 21:21:01 +02004766 memset(&iter->seq, 0,
4767 sizeof(struct trace_iterator) -
4768 offsetof(struct trace_iterator, seq));
Andrew Vagined5467d2013-08-02 21:16:43 +04004769 cpumask_clear(iter->started);
Steven Rostedt4823ed72008-05-12 21:21:01 +02004770 iter->pos = -1;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004771
Lai Jiangshan4f535962009-05-18 19:35:34 +08004772 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004773 trace_access_lock(iter->cpu_file);
Jason Wessel955b61e2010-08-05 09:22:23 -05004774 while (trace_find_next_entry_inc(iter) != NULL) {
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02004775 enum print_line_t ret;
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004776 int save_len = iter->seq.seq.len;
Steven Rostedt088b1e422008-05-12 21:20:48 +02004777
Ingo Molnarf9896bf2008-05-12 21:20:47 +02004778 ret = print_trace_line(iter);
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02004779 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt088b1e422008-05-12 21:20:48 +02004780 /* don't print partial lines */
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004781 iter->seq.seq.len = save_len;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004782 break;
Steven Rostedt088b1e422008-05-12 21:20:48 +02004783 }
Frederic Weisbeckerb91facc2009-02-06 18:30:44 +01004784 if (ret != TRACE_TYPE_NO_CONSUME)
4785 trace_consume(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004786
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004787 if (trace_seq_used(&iter->seq) >= cnt)
Steven Rostedtb3806b42008-05-12 21:20:46 +02004788 break;
Jiri Olsaee5e51f2011-03-25 12:05:18 +01004789
4790 /*
4791 * Setting the full flag means we reached the trace_seq buffer
4792 * size and we should leave by partial output condition above.
4793 * One of the trace_seq_* functions is not used properly.
4794 */
4795 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4796 iter->ent->type);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004797 }
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004798 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004799 trace_event_read_unlock();
Steven Rostedtb3806b42008-05-12 21:20:46 +02004800
Steven Rostedtb3806b42008-05-12 21:20:46 +02004801 /* Now copy what we have to the user */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004802 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004803 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
Steven Rostedtf9520752009-03-02 14:04:40 -05004804 trace_seq_init(&iter->seq);
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004805
4806 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004807 * If there was nothing to send to user, in spite of consuming trace
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004808 * entries, go back to wait for more entries.
4809 */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004810 if (sret == -EBUSY)
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004811 goto waitagain;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004812
Steven Rostedt107bad82008-05-12 21:21:01 +02004813out:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004814 mutex_unlock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02004815
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004816 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004817}
4818
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004819static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4820 unsigned int idx)
4821{
4822 __free_page(spd->pages[idx]);
4823}
4824
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08004825static const struct pipe_buf_operations tracing_pipe_buf_ops = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05004826 .can_merge = 0,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004827 .confirm = generic_pipe_buf_confirm,
Al Viro92fdd982014-01-17 07:53:39 -05004828 .release = generic_pipe_buf_release,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004829 .steal = generic_pipe_buf_steal,
4830 .get = generic_pipe_buf_get,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004831};
4832
Steven Rostedt34cd4992009-02-09 12:06:29 -05004833static size_t
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01004834tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004835{
4836 size_t count;
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004837 int save_len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004838 int ret;
4839
4840 /* Seq buffer is page-sized, exactly what we need. */
4841 for (;;) {
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004842 save_len = iter->seq.seq.len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004843 ret = print_trace_line(iter);
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004844
4845 if (trace_seq_has_overflowed(&iter->seq)) {
4846 iter->seq.seq.len = save_len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004847 break;
4848 }
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004849
4850 /*
4851 * This should not be hit, because it should only
4852 * be set if the iter->seq overflowed. But check it
4853 * anyway to be safe.
4854 */
Steven Rostedt34cd4992009-02-09 12:06:29 -05004855 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004856 iter->seq.seq.len = save_len;
4857 break;
4858 }
4859
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004860 count = trace_seq_used(&iter->seq) - save_len;
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004861 if (rem < count) {
4862 rem = 0;
4863 iter->seq.seq.len = save_len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004864 break;
4865 }
4866
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08004867 if (ret != TRACE_TYPE_NO_CONSUME)
4868 trace_consume(iter);
Steven Rostedt34cd4992009-02-09 12:06:29 -05004869 rem -= count;
Jason Wessel955b61e2010-08-05 09:22:23 -05004870 if (!trace_find_next_entry_inc(iter)) {
Steven Rostedt34cd4992009-02-09 12:06:29 -05004871 rem = 0;
4872 iter->ent = NULL;
4873 break;
4874 }
4875 }
4876
4877 return rem;
4878}
4879
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004880static ssize_t tracing_splice_read_pipe(struct file *filp,
4881 loff_t *ppos,
4882 struct pipe_inode_info *pipe,
4883 size_t len,
4884 unsigned int flags)
4885{
Jens Axboe35f3d142010-05-20 10:43:18 +02004886 struct page *pages_def[PIPE_DEF_BUFFERS];
4887 struct partial_page partial_def[PIPE_DEF_BUFFERS];
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004888 struct trace_iterator *iter = filp->private_data;
4889 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02004890 .pages = pages_def,
4891 .partial = partial_def,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004892 .nr_pages = 0, /* This gets updated below. */
Eric Dumazet047fe362012-06-12 15:24:40 +02004893 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004894 .flags = flags,
4895 .ops = &tracing_pipe_buf_ops,
4896 .spd_release = tracing_spd_release_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004897 };
4898 ssize_t ret;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004899 size_t rem;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004900 unsigned int i;
4901
Jens Axboe35f3d142010-05-20 10:43:18 +02004902 if (splice_grow_spd(pipe, &spd))
4903 return -ENOMEM;
4904
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004905 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004906
4907 if (iter->trace->splice_read) {
4908 ret = iter->trace->splice_read(iter, filp,
4909 ppos, pipe, len, flags);
4910 if (ret)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004911 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004912 }
4913
4914 ret = tracing_wait_pipe(filp);
4915 if (ret <= 0)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004916 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004917
Jason Wessel955b61e2010-08-05 09:22:23 -05004918 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004919 ret = -EFAULT;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004920 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004921 }
4922
Lai Jiangshan4f535962009-05-18 19:35:34 +08004923 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004924 trace_access_lock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004925
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004926 /* Fill as many pages as possible. */
Al Viroa786c062014-04-11 12:01:03 -04004927 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004928 spd.pages[i] = alloc_page(GFP_KERNEL);
4929 if (!spd.pages[i])
Steven Rostedt34cd4992009-02-09 12:06:29 -05004930 break;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004931
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01004932 rem = tracing_fill_pipe_page(rem, iter);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004933
4934 /* Copy the data into the page, so we can start over. */
4935 ret = trace_seq_to_buffer(&iter->seq,
Jens Axboe35f3d142010-05-20 10:43:18 +02004936 page_address(spd.pages[i]),
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004937 trace_seq_used(&iter->seq));
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004938 if (ret < 0) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004939 __free_page(spd.pages[i]);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004940 break;
4941 }
Jens Axboe35f3d142010-05-20 10:43:18 +02004942 spd.partial[i].offset = 0;
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004943 spd.partial[i].len = trace_seq_used(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004944
Steven Rostedtf9520752009-03-02 14:04:40 -05004945 trace_seq_init(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004946 }
4947
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004948 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004949 trace_event_read_unlock();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004950 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004951
4952 spd.nr_pages = i;
4953
Jens Axboe35f3d142010-05-20 10:43:18 +02004954 ret = splice_to_pipe(pipe, &spd);
4955out:
Eric Dumazet047fe362012-06-12 15:24:40 +02004956 splice_shrink_spd(&spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02004957 return ret;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004958
Steven Rostedt34cd4992009-02-09 12:06:29 -05004959out_err:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004960 mutex_unlock(&iter->mutex);
Jens Axboe35f3d142010-05-20 10:43:18 +02004961 goto out;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004962}
4963
Steven Rostedta98a3c32008-05-12 21:20:59 +02004964static ssize_t
4965tracing_entries_read(struct file *filp, char __user *ubuf,
4966 size_t cnt, loff_t *ppos)
4967{
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02004968 struct inode *inode = file_inode(filp);
4969 struct trace_array *tr = inode->i_private;
4970 int cpu = tracing_get_cpu(inode);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004971 char buf[64];
4972 int r = 0;
4973 ssize_t ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004974
Steven Rostedtdb526ca2009-03-12 13:53:25 -04004975 mutex_lock(&trace_types_lock);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004976
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02004977 if (cpu == RING_BUFFER_ALL_CPUS) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004978 int cpu, buf_size_same;
4979 unsigned long size;
4980
4981 size = 0;
4982 buf_size_same = 1;
4983 /* check if all cpu sizes are same */
4984 for_each_tracing_cpu(cpu) {
4985 /* fill in the size from first enabled cpu */
4986 if (size == 0)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004987 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4988 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004989 buf_size_same = 0;
4990 break;
4991 }
4992 }
4993
4994 if (buf_size_same) {
4995 if (!ring_buffer_expanded)
4996 r = sprintf(buf, "%lu (expanded: %lu)\n",
4997 size >> 10,
4998 trace_buf_size >> 10);
4999 else
5000 r = sprintf(buf, "%lu\n", size >> 10);
5001 } else
5002 r = sprintf(buf, "X\n");
5003 } else
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005004 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08005005
Steven Rostedtdb526ca2009-03-12 13:53:25 -04005006 mutex_unlock(&trace_types_lock);
5007
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08005008 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5009 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005010}
5011
5012static ssize_t
5013tracing_entries_write(struct file *filp, const char __user *ubuf,
5014 size_t cnt, loff_t *ppos)
5015{
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005016 struct inode *inode = file_inode(filp);
5017 struct trace_array *tr = inode->i_private;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005018 unsigned long val;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005019 int ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005020
Peter Huewe22fe9b52011-06-07 21:58:27 +02005021 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5022 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02005023 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005024
5025 /* must have at least 1 entry */
5026 if (!val)
5027 return -EINVAL;
5028
Steven Rostedt1696b2b2008-11-13 00:09:35 -05005029 /* value is in KB */
5030 val <<= 10;
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005031 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005032 if (ret < 0)
5033 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005034
Jiri Olsacf8517c2009-10-23 19:36:16 -04005035 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005036
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005037 return cnt;
5038}
Steven Rostedtbf5e6512008-11-10 21:46:00 -05005039
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005040static ssize_t
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005041tracing_total_entries_read(struct file *filp, char __user *ubuf,
5042 size_t cnt, loff_t *ppos)
5043{
5044 struct trace_array *tr = filp->private_data;
5045 char buf[64];
5046 int r, cpu;
5047 unsigned long size = 0, expanded_size = 0;
5048
5049 mutex_lock(&trace_types_lock);
5050 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005051 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005052 if (!ring_buffer_expanded)
5053 expanded_size += trace_buf_size >> 10;
5054 }
5055 if (ring_buffer_expanded)
5056 r = sprintf(buf, "%lu\n", size);
5057 else
5058 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5059 mutex_unlock(&trace_types_lock);
5060
5061 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5062}
5063
5064static ssize_t
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005065tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5066 size_t cnt, loff_t *ppos)
5067{
5068 /*
5069 * There is no need to read what the user has written, this function
5070 * is just to make sure that there is no error when "echo" is used
5071 */
5072
5073 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005074
5075 return cnt;
5076}
5077
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005078static int
5079tracing_free_buffer_release(struct inode *inode, struct file *filp)
5080{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005081 struct trace_array *tr = inode->i_private;
5082
Steven Rostedtcf30cf62011-06-14 22:44:07 -04005083 /* disable tracing ? */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04005084 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
Alexander Z Lam711e1242013-08-02 18:36:15 -07005085 tracer_tracing_off(tr);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005086 /* resize the ring buffer to 0 */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005087 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005088
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005089 trace_array_put(tr);
5090
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005091 return 0;
5092}
5093
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005094static ssize_t
5095tracing_mark_write(struct file *filp, const char __user *ubuf,
5096 size_t cnt, loff_t *fpos)
5097{
Steven Rostedtd696b582011-09-22 11:50:27 -04005098 unsigned long addr = (unsigned long)ubuf;
Alexander Z Lam2d716192013-07-01 15:31:24 -07005099 struct trace_array *tr = filp->private_data;
Steven Rostedtd696b582011-09-22 11:50:27 -04005100 struct ring_buffer_event *event;
5101 struct ring_buffer *buffer;
5102 struct print_entry *entry;
5103 unsigned long irq_flags;
5104 struct page *pages[2];
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005105 void *map_page[2];
Steven Rostedtd696b582011-09-22 11:50:27 -04005106 int nr_pages = 1;
5107 ssize_t written;
Steven Rostedtd696b582011-09-22 11:50:27 -04005108 int offset;
5109 int size;
5110 int len;
5111 int ret;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005112 int i;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005113
Steven Rostedtc76f0692008-11-07 22:36:02 -05005114 if (tracing_disabled)
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005115 return -EINVAL;
5116
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04005117 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
Mandeep Singh Baines5224c3a2012-09-07 18:12:19 -07005118 return -EINVAL;
5119
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005120 if (cnt > TRACE_BUF_SIZE)
5121 cnt = TRACE_BUF_SIZE;
5122
Steven Rostedtd696b582011-09-22 11:50:27 -04005123 /*
5124 * Userspace is injecting traces into the kernel trace buffer.
5125 * We want to be as non intrusive as possible.
5126 * To do so, we do not want to allocate any special buffers
5127 * or take any locks, but instead write the userspace data
5128 * straight into the ring buffer.
5129 *
5130 * First we need to pin the userspace buffer into memory,
5131 * which, most likely it is, because it just referenced it.
5132 * But there's no guarantee that it is. By using get_user_pages_fast()
5133 * and kmap_atomic/kunmap_atomic() we can get access to the
5134 * pages directly. We then write the data directly into the
5135 * ring buffer.
5136 */
5137 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005138
Steven Rostedtd696b582011-09-22 11:50:27 -04005139 /* check if we cross pages */
5140 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5141 nr_pages = 2;
5142
5143 offset = addr & (PAGE_SIZE - 1);
5144 addr &= PAGE_MASK;
5145
5146 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5147 if (ret < nr_pages) {
5148 while (--ret >= 0)
5149 put_page(pages[ret]);
5150 written = -EFAULT;
5151 goto out;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005152 }
5153
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005154 for (i = 0; i < nr_pages; i++)
5155 map_page[i] = kmap_atomic(pages[i]);
Steven Rostedtd696b582011-09-22 11:50:27 -04005156
5157 local_save_flags(irq_flags);
5158 size = sizeof(*entry) + cnt + 2; /* possible \n added */
Alexander Z Lam2d716192013-07-01 15:31:24 -07005159 buffer = tr->trace_buffer.buffer;
Steven Rostedtd696b582011-09-22 11:50:27 -04005160 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5161 irq_flags, preempt_count());
5162 if (!event) {
5163 /* Ring buffer disabled, return as if not open for write */
5164 written = -EBADF;
5165 goto out_unlock;
5166 }
5167
5168 entry = ring_buffer_event_data(event);
5169 entry->ip = _THIS_IP_;
5170
5171 if (nr_pages == 2) {
5172 len = PAGE_SIZE - offset;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005173 memcpy(&entry->buf, map_page[0] + offset, len);
5174 memcpy(&entry->buf[len], map_page[1], cnt - len);
Steven Rostedtd696b582011-09-22 11:50:27 -04005175 } else
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005176 memcpy(&entry->buf, map_page[0] + offset, cnt);
Steven Rostedtd696b582011-09-22 11:50:27 -04005177
5178 if (entry->buf[cnt - 1] != '\n') {
5179 entry->buf[cnt] = '\n';
5180 entry->buf[cnt + 1] = '\0';
5181 } else
5182 entry->buf[cnt] = '\0';
5183
Steven Rostedt7ffbd482012-10-11 12:14:25 -04005184 __buffer_unlock_commit(buffer, event);
Steven Rostedtd696b582011-09-22 11:50:27 -04005185
5186 written = cnt;
5187
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02005188 *fpos += written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005189
Steven Rostedtd696b582011-09-22 11:50:27 -04005190 out_unlock:
Vikram Mulukutla72158532014-12-17 18:50:56 -08005191 for (i = nr_pages - 1; i >= 0; i--) {
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005192 kunmap_atomic(map_page[i]);
5193 put_page(pages[i]);
5194 }
Steven Rostedtd696b582011-09-22 11:50:27 -04005195 out:
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02005196 return written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005197}
5198
Li Zefan13f16d22009-12-08 11:16:11 +08005199static int tracing_clock_show(struct seq_file *m, void *v)
Zhaolei5079f322009-08-25 16:12:56 +08005200{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005201 struct trace_array *tr = m->private;
Zhaolei5079f322009-08-25 16:12:56 +08005202 int i;
5203
5204 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
Li Zefan13f16d22009-12-08 11:16:11 +08005205 seq_printf(m,
Zhaolei5079f322009-08-25 16:12:56 +08005206 "%s%s%s%s", i ? " " : "",
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005207 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5208 i == tr->clock_id ? "]" : "");
Li Zefan13f16d22009-12-08 11:16:11 +08005209 seq_putc(m, '\n');
Zhaolei5079f322009-08-25 16:12:56 +08005210
Li Zefan13f16d22009-12-08 11:16:11 +08005211 return 0;
Zhaolei5079f322009-08-25 16:12:56 +08005212}
5213
Steven Rostedte1e232c2014-02-10 23:38:46 -05005214static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
Zhaolei5079f322009-08-25 16:12:56 +08005215{
Zhaolei5079f322009-08-25 16:12:56 +08005216 int i;
5217
Zhaolei5079f322009-08-25 16:12:56 +08005218 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5219 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5220 break;
5221 }
5222 if (i == ARRAY_SIZE(trace_clocks))
5223 return -EINVAL;
5224
Zhaolei5079f322009-08-25 16:12:56 +08005225 mutex_lock(&trace_types_lock);
5226
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005227 tr->clock_id = i;
5228
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005229 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
Zhaolei5079f322009-08-25 16:12:56 +08005230
David Sharp60303ed2012-10-11 16:27:52 -07005231 /*
5232 * New clock may not be consistent with the previous clock.
5233 * Reset the buffer so that it doesn't have incomparable timestamps.
5234 */
Alexander Z Lam94571582013-08-02 18:36:16 -07005235 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005236
5237#ifdef CONFIG_TRACER_MAX_TRACE
5238 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5239 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
Alexander Z Lam94571582013-08-02 18:36:16 -07005240 tracing_reset_online_cpus(&tr->max_buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005241#endif
David Sharp60303ed2012-10-11 16:27:52 -07005242
Zhaolei5079f322009-08-25 16:12:56 +08005243 mutex_unlock(&trace_types_lock);
5244
Steven Rostedte1e232c2014-02-10 23:38:46 -05005245 return 0;
5246}
5247
5248static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5249 size_t cnt, loff_t *fpos)
5250{
5251 struct seq_file *m = filp->private_data;
5252 struct trace_array *tr = m->private;
5253 char buf[64];
5254 const char *clockstr;
5255 int ret;
5256
5257 if (cnt >= sizeof(buf))
5258 return -EINVAL;
5259
5260 if (copy_from_user(&buf, ubuf, cnt))
5261 return -EFAULT;
5262
5263 buf[cnt] = 0;
5264
5265 clockstr = strstrip(buf);
5266
5267 ret = tracing_set_clock(tr, clockstr);
5268 if (ret)
5269 return ret;
5270
Zhaolei5079f322009-08-25 16:12:56 +08005271 *fpos += cnt;
5272
5273 return cnt;
5274}
5275
Li Zefan13f16d22009-12-08 11:16:11 +08005276static int tracing_clock_open(struct inode *inode, struct file *file)
5277{
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005278 struct trace_array *tr = inode->i_private;
5279 int ret;
5280
Li Zefan13f16d22009-12-08 11:16:11 +08005281 if (tracing_disabled)
5282 return -ENODEV;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005283
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005284 if (trace_array_get(tr))
5285 return -ENODEV;
5286
5287 ret = single_open(file, tracing_clock_show, inode->i_private);
5288 if (ret < 0)
5289 trace_array_put(tr);
5290
5291 return ret;
Li Zefan13f16d22009-12-08 11:16:11 +08005292}
5293
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005294struct ftrace_buffer_info {
5295 struct trace_iterator iter;
5296 void *spare;
5297 unsigned int read;
5298};
5299
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005300#ifdef CONFIG_TRACER_SNAPSHOT
5301static int tracing_snapshot_open(struct inode *inode, struct file *file)
5302{
Oleg Nesterov6484c712013-07-23 17:26:10 +02005303 struct trace_array *tr = inode->i_private;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005304 struct trace_iterator *iter;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005305 struct seq_file *m;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005306 int ret = 0;
5307
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005308 if (trace_array_get(tr) < 0)
5309 return -ENODEV;
5310
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005311 if (file->f_mode & FMODE_READ) {
Oleg Nesterov6484c712013-07-23 17:26:10 +02005312 iter = __tracing_open(inode, file, true);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005313 if (IS_ERR(iter))
5314 ret = PTR_ERR(iter);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005315 } else {
5316 /* Writes still need the seq_file to hold the private data */
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005317 ret = -ENOMEM;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005318 m = kzalloc(sizeof(*m), GFP_KERNEL);
5319 if (!m)
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005320 goto out;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005321 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5322 if (!iter) {
5323 kfree(m);
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005324 goto out;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005325 }
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005326 ret = 0;
5327
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005328 iter->tr = tr;
Oleg Nesterov6484c712013-07-23 17:26:10 +02005329 iter->trace_buffer = &tr->max_buffer;
5330 iter->cpu_file = tracing_get_cpu(inode);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005331 m->private = iter;
5332 file->private_data = m;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005333 }
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005334out:
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005335 if (ret < 0)
5336 trace_array_put(tr);
5337
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005338 return ret;
5339}
5340
5341static ssize_t
5342tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5343 loff_t *ppos)
5344{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005345 struct seq_file *m = filp->private_data;
5346 struct trace_iterator *iter = m->private;
5347 struct trace_array *tr = iter->tr;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005348 unsigned long val;
5349 int ret;
5350
5351 ret = tracing_update_buffers();
5352 if (ret < 0)
5353 return ret;
5354
5355 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5356 if (ret)
5357 return ret;
5358
5359 mutex_lock(&trace_types_lock);
5360
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005361 if (tr->current_trace->use_max_tr) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005362 ret = -EBUSY;
5363 goto out;
5364 }
5365
5366 switch (val) {
5367 case 0:
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005368 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5369 ret = -EINVAL;
5370 break;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005371 }
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04005372 if (tr->allocated_snapshot)
5373 free_snapshot(tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005374 break;
5375 case 1:
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005376/* Only allow per-cpu swap if the ring buffer supports it */
5377#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5378 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5379 ret = -EINVAL;
5380 break;
5381 }
5382#endif
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05005383 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04005384 ret = alloc_snapshot(tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005385 if (ret < 0)
5386 break;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005387 }
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005388 local_irq_disable();
5389 /* Now, we're going to swap */
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005390 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05005391 update_max_tr(tr, current, smp_processor_id());
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005392 else
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05005393 update_max_tr_single(tr, current, iter->cpu_file);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005394 local_irq_enable();
5395 break;
5396 default:
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05005397 if (tr->allocated_snapshot) {
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005398 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5399 tracing_reset_online_cpus(&tr->max_buffer);
5400 else
5401 tracing_reset(&tr->max_buffer, iter->cpu_file);
5402 }
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005403 break;
5404 }
5405
5406 if (ret >= 0) {
5407 *ppos += cnt;
5408 ret = cnt;
5409 }
5410out:
5411 mutex_unlock(&trace_types_lock);
5412 return ret;
5413}
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005414
5415static int tracing_snapshot_release(struct inode *inode, struct file *file)
5416{
5417 struct seq_file *m = file->private_data;
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005418 int ret;
5419
5420 ret = tracing_release(inode, file);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005421
5422 if (file->f_mode & FMODE_READ)
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005423 return ret;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005424
5425 /* If write only, the seq_file is just a stub */
5426 if (m)
5427 kfree(m->private);
5428 kfree(m);
5429
5430 return 0;
5431}
5432
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005433static int tracing_buffers_open(struct inode *inode, struct file *filp);
5434static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5435 size_t count, loff_t *ppos);
5436static int tracing_buffers_release(struct inode *inode, struct file *file);
5437static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5438 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5439
5440static int snapshot_raw_open(struct inode *inode, struct file *filp)
5441{
5442 struct ftrace_buffer_info *info;
5443 int ret;
5444
5445 ret = tracing_buffers_open(inode, filp);
5446 if (ret < 0)
5447 return ret;
5448
5449 info = filp->private_data;
5450
5451 if (info->iter.trace->use_max_tr) {
5452 tracing_buffers_release(inode, filp);
5453 return -EBUSY;
5454 }
5455
5456 info->iter.snapshot = true;
5457 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5458
5459 return ret;
5460}
5461
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005462#endif /* CONFIG_TRACER_SNAPSHOT */
5463
5464
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04005465static const struct file_operations tracing_thresh_fops = {
5466 .open = tracing_open_generic,
5467 .read = tracing_thresh_read,
5468 .write = tracing_thresh_write,
5469 .llseek = generic_file_llseek,
5470};
5471
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005472static const struct file_operations tracing_max_lat_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005473 .open = tracing_open_generic,
5474 .read = tracing_max_lat_read,
5475 .write = tracing_max_lat_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005476 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005477};
5478
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005479static const struct file_operations set_tracer_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005480 .open = tracing_open_generic,
5481 .read = tracing_set_trace_read,
5482 .write = tracing_set_trace_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005483 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005484};
5485
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005486static const struct file_operations tracing_pipe_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005487 .open = tracing_open_pipe,
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02005488 .poll = tracing_poll_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005489 .read = tracing_read_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02005490 .splice_read = tracing_splice_read_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005491 .release = tracing_release_pipe,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005492 .llseek = no_llseek,
Steven Rostedtb3806b42008-05-12 21:20:46 +02005493};
5494
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005495static const struct file_operations tracing_entries_fops = {
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005496 .open = tracing_open_generic_tr,
Steven Rostedta98a3c32008-05-12 21:20:59 +02005497 .read = tracing_entries_read,
5498 .write = tracing_entries_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005499 .llseek = generic_file_llseek,
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005500 .release = tracing_release_generic_tr,
Steven Rostedta98a3c32008-05-12 21:20:59 +02005501};
5502
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005503static const struct file_operations tracing_total_entries_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005504 .open = tracing_open_generic_tr,
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005505 .read = tracing_total_entries_read,
5506 .llseek = generic_file_llseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005507 .release = tracing_release_generic_tr,
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005508};
5509
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005510static const struct file_operations tracing_free_buffer_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005511 .open = tracing_open_generic_tr,
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005512 .write = tracing_free_buffer_write,
5513 .release = tracing_free_buffer_release,
5514};
5515
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005516static const struct file_operations tracing_mark_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005517 .open = tracing_open_generic_tr,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005518 .write = tracing_mark_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005519 .llseek = generic_file_llseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005520 .release = tracing_release_generic_tr,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005521};
5522
Zhaolei5079f322009-08-25 16:12:56 +08005523static const struct file_operations trace_clock_fops = {
Li Zefan13f16d22009-12-08 11:16:11 +08005524 .open = tracing_clock_open,
5525 .read = seq_read,
5526 .llseek = seq_lseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005527 .release = tracing_single_release_tr,
Zhaolei5079f322009-08-25 16:12:56 +08005528 .write = tracing_clock_write,
5529};
5530
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005531#ifdef CONFIG_TRACER_SNAPSHOT
5532static const struct file_operations snapshot_fops = {
5533 .open = tracing_snapshot_open,
5534 .read = seq_read,
5535 .write = tracing_snapshot_write,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05005536 .llseek = tracing_lseek,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005537 .release = tracing_snapshot_release,
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005538};
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005539
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005540static const struct file_operations snapshot_raw_fops = {
5541 .open = snapshot_raw_open,
5542 .read = tracing_buffers_read,
5543 .release = tracing_buffers_release,
5544 .splice_read = tracing_buffers_splice_read,
5545 .llseek = no_llseek,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005546};
5547
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005548#endif /* CONFIG_TRACER_SNAPSHOT */
5549
Steven Rostedt2cadf912008-12-01 22:20:19 -05005550static int tracing_buffers_open(struct inode *inode, struct file *filp)
5551{
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02005552 struct trace_array *tr = inode->i_private;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005553 struct ftrace_buffer_info *info;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005554 int ret;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005555
5556 if (tracing_disabled)
5557 return -ENODEV;
5558
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005559 if (trace_array_get(tr) < 0)
5560 return -ENODEV;
5561
Steven Rostedt2cadf912008-12-01 22:20:19 -05005562 info = kzalloc(sizeof(*info), GFP_KERNEL);
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005563 if (!info) {
5564 trace_array_put(tr);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005565 return -ENOMEM;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005566 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05005567
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005568 mutex_lock(&trace_types_lock);
5569
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005570 info->iter.tr = tr;
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02005571 info->iter.cpu_file = tracing_get_cpu(inode);
Steven Rostedtb6273442013-02-28 13:44:11 -05005572 info->iter.trace = tr->current_trace;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005573 info->iter.trace_buffer = &tr->trace_buffer;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005574 info->spare = NULL;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005575 /* Force reading ring buffer for first read */
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005576 info->read = (unsigned int)-1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005577
5578 filp->private_data = info;
5579
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05005580 tr->current_trace->ref++;
5581
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005582 mutex_unlock(&trace_types_lock);
5583
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005584 ret = nonseekable_open(inode, filp);
5585 if (ret < 0)
5586 trace_array_put(tr);
5587
5588 return ret;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005589}
5590
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005591static unsigned int
5592tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5593{
5594 struct ftrace_buffer_info *info = filp->private_data;
5595 struct trace_iterator *iter = &info->iter;
5596
5597 return trace_poll(iter, filp, poll_table);
5598}
5599
Steven Rostedt2cadf912008-12-01 22:20:19 -05005600static ssize_t
5601tracing_buffers_read(struct file *filp, char __user *ubuf,
5602 size_t count, loff_t *ppos)
5603{
5604 struct ftrace_buffer_info *info = filp->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005605 struct trace_iterator *iter = &info->iter;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005606 ssize_t ret;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005607 ssize_t size;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005608
Steven Rostedt2dc5d122009-03-04 19:10:05 -05005609 if (!count)
5610 return 0;
5611
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005612#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005613 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5614 return -EBUSY;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005615#endif
5616
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005617 if (!info->spare)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005618 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5619 iter->cpu_file);
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005620 if (!info->spare)
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005621 return -ENOMEM;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005622
Steven Rostedt2cadf912008-12-01 22:20:19 -05005623 /* Do we have previous read data to read? */
5624 if (info->read < PAGE_SIZE)
5625 goto read;
5626
Steven Rostedtb6273442013-02-28 13:44:11 -05005627 again:
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005628 trace_access_lock(iter->cpu_file);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005629 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005630 &info->spare,
5631 count,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005632 iter->cpu_file, 0);
5633 trace_access_unlock(iter->cpu_file);
Steven Rostedtb6273442013-02-28 13:44:11 -05005634
5635 if (ret < 0) {
5636 if (trace_empty(iter)) {
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005637 if ((filp->f_flags & O_NONBLOCK))
5638 return -EAGAIN;
5639
Rabin Vincente30f53a2014-11-10 19:46:34 +01005640 ret = wait_on_pipe(iter, false);
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005641 if (ret)
5642 return ret;
5643
Steven Rostedtb6273442013-02-28 13:44:11 -05005644 goto again;
5645 }
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005646 return 0;
Steven Rostedtb6273442013-02-28 13:44:11 -05005647 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05005648
Steven Rostedt436fc282011-10-14 10:44:25 -04005649 info->read = 0;
Steven Rostedtb6273442013-02-28 13:44:11 -05005650 read:
Steven Rostedt2cadf912008-12-01 22:20:19 -05005651 size = PAGE_SIZE - info->read;
5652 if (size > count)
5653 size = count;
5654
5655 ret = copy_to_user(ubuf, info->spare + info->read, size);
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005656 if (ret == size)
5657 return -EFAULT;
5658
Steven Rostedt2dc5d122009-03-04 19:10:05 -05005659 size -= ret;
5660
Steven Rostedt2cadf912008-12-01 22:20:19 -05005661 *ppos += size;
5662 info->read += size;
5663
5664 return size;
5665}
5666
5667static int tracing_buffers_release(struct inode *inode, struct file *file)
5668{
5669 struct ftrace_buffer_info *info = file->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005670 struct trace_iterator *iter = &info->iter;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005671
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005672 mutex_lock(&trace_types_lock);
5673
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05005674 iter->tr->current_trace->ref--;
5675
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005676 __trace_array_put(iter->tr);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005677
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005678 if (info->spare)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005679 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005680 kfree(info);
5681
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005682 mutex_unlock(&trace_types_lock);
5683
Steven Rostedt2cadf912008-12-01 22:20:19 -05005684 return 0;
5685}
5686
5687struct buffer_ref {
5688 struct ring_buffer *buffer;
5689 void *page;
5690 int ref;
5691};
5692
5693static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5694 struct pipe_buffer *buf)
5695{
5696 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5697
5698 if (--ref->ref)
5699 return;
5700
5701 ring_buffer_free_read_page(ref->buffer, ref->page);
5702 kfree(ref);
5703 buf->private = 0;
5704}
5705
Steven Rostedt2cadf912008-12-01 22:20:19 -05005706static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5707 struct pipe_buffer *buf)
5708{
5709 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5710
5711 ref->ref++;
5712}
5713
5714/* Pipe buffer operations for a buffer. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08005715static const struct pipe_buf_operations buffer_pipe_buf_ops = {
Steven Rostedt2cadf912008-12-01 22:20:19 -05005716 .can_merge = 0,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005717 .confirm = generic_pipe_buf_confirm,
5718 .release = buffer_pipe_buf_release,
Masami Hiramatsud55cb6c2012-08-09 21:31:10 +09005719 .steal = generic_pipe_buf_steal,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005720 .get = buffer_pipe_buf_get,
5721};
5722
5723/*
5724 * Callback from splice_to_pipe(), if we need to release some pages
5725 * at the end of the spd in case we error'ed out in filling the pipe.
5726 */
5727static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5728{
5729 struct buffer_ref *ref =
5730 (struct buffer_ref *)spd->partial[i].private;
5731
5732 if (--ref->ref)
5733 return;
5734
5735 ring_buffer_free_read_page(ref->buffer, ref->page);
5736 kfree(ref);
5737 spd->partial[i].private = 0;
5738}
5739
5740static ssize_t
5741tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5742 struct pipe_inode_info *pipe, size_t len,
5743 unsigned int flags)
5744{
5745 struct ftrace_buffer_info *info = file->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005746 struct trace_iterator *iter = &info->iter;
Jens Axboe35f3d142010-05-20 10:43:18 +02005747 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5748 struct page *pages_def[PIPE_DEF_BUFFERS];
Steven Rostedt2cadf912008-12-01 22:20:19 -05005749 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02005750 .pages = pages_def,
5751 .partial = partial_def,
Eric Dumazet047fe362012-06-12 15:24:40 +02005752 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005753 .flags = flags,
5754 .ops = &buffer_pipe_buf_ops,
5755 .spd_release = buffer_spd_release,
5756 };
5757 struct buffer_ref *ref;
Steven Rostedt93459c62009-04-29 00:23:13 -04005758 int entries, size, i;
Rabin Vincent07906da2014-11-06 22:26:07 +01005759 ssize_t ret = 0;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005760
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005761#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005762 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5763 return -EBUSY;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005764#endif
5765
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005766 if (splice_grow_spd(pipe, &spd))
5767 return -ENOMEM;
Jens Axboe35f3d142010-05-20 10:43:18 +02005768
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005769 if (*ppos & (PAGE_SIZE - 1))
5770 return -EINVAL;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005771
5772 if (len & (PAGE_SIZE - 1)) {
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005773 if (len < PAGE_SIZE)
5774 return -EINVAL;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005775 len &= PAGE_MASK;
5776 }
5777
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005778 again:
5779 trace_access_lock(iter->cpu_file);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005780 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt93459c62009-04-29 00:23:13 -04005781
Al Viroa786c062014-04-11 12:01:03 -04005782 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05005783 struct page *page;
5784 int r;
5785
5786 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
Rabin Vincent07906da2014-11-06 22:26:07 +01005787 if (!ref) {
5788 ret = -ENOMEM;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005789 break;
Rabin Vincent07906da2014-11-06 22:26:07 +01005790 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05005791
Steven Rostedt7267fa62009-04-29 00:16:21 -04005792 ref->ref = 1;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005793 ref->buffer = iter->trace_buffer->buffer;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005794 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005795 if (!ref->page) {
Rabin Vincent07906da2014-11-06 22:26:07 +01005796 ret = -ENOMEM;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005797 kfree(ref);
5798 break;
5799 }
5800
5801 r = ring_buffer_read_page(ref->buffer, &ref->page,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005802 len, iter->cpu_file, 1);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005803 if (r < 0) {
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07005804 ring_buffer_free_read_page(ref->buffer, ref->page);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005805 kfree(ref);
5806 break;
5807 }
5808
5809 /*
5810 * zero out any left over data, this is going to
5811 * user land.
5812 */
5813 size = ring_buffer_page_len(ref->page);
5814 if (size < PAGE_SIZE)
5815 memset(ref->page + size, 0, PAGE_SIZE - size);
5816
5817 page = virt_to_page(ref->page);
5818
5819 spd.pages[i] = page;
5820 spd.partial[i].len = PAGE_SIZE;
5821 spd.partial[i].offset = 0;
5822 spd.partial[i].private = (unsigned long)ref;
5823 spd.nr_pages++;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005824 *ppos += PAGE_SIZE;
Steven Rostedt93459c62009-04-29 00:23:13 -04005825
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005826 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005827 }
5828
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005829 trace_access_unlock(iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005830 spd.nr_pages = i;
5831
5832 /* did we read anything? */
5833 if (!spd.nr_pages) {
Rabin Vincent07906da2014-11-06 22:26:07 +01005834 if (ret)
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005835 return ret;
Rabin Vincent07906da2014-11-06 22:26:07 +01005836
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005837 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
5838 return -EAGAIN;
5839
Rabin Vincente30f53a2014-11-10 19:46:34 +01005840 ret = wait_on_pipe(iter, true);
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04005841 if (ret)
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005842 return ret;
Rabin Vincente30f53a2014-11-10 19:46:34 +01005843
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005844 goto again;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005845 }
5846
5847 ret = splice_to_pipe(pipe, &spd);
Eric Dumazet047fe362012-06-12 15:24:40 +02005848 splice_shrink_spd(&spd);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005849
Steven Rostedt2cadf912008-12-01 22:20:19 -05005850 return ret;
5851}
5852
5853static const struct file_operations tracing_buffers_fops = {
5854 .open = tracing_buffers_open,
5855 .read = tracing_buffers_read,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005856 .poll = tracing_buffers_poll,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005857 .release = tracing_buffers_release,
5858 .splice_read = tracing_buffers_splice_read,
5859 .llseek = no_llseek,
5860};
5861
Steven Rostedtc8d77182009-04-29 18:03:45 -04005862static ssize_t
5863tracing_stats_read(struct file *filp, char __user *ubuf,
5864 size_t count, loff_t *ppos)
5865{
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005866 struct inode *inode = file_inode(filp);
5867 struct trace_array *tr = inode->i_private;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005868 struct trace_buffer *trace_buf = &tr->trace_buffer;
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005869 int cpu = tracing_get_cpu(inode);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005870 struct trace_seq *s;
5871 unsigned long cnt;
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005872 unsigned long long t;
5873 unsigned long usec_rem;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005874
Li Zefane4f2d102009-06-15 10:57:28 +08005875 s = kmalloc(sizeof(*s), GFP_KERNEL);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005876 if (!s)
Roel Kluina6463652009-11-11 22:26:35 +01005877 return -ENOMEM;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005878
5879 trace_seq_init(s);
5880
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005881 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005882 trace_seq_printf(s, "entries: %ld\n", cnt);
5883
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005884 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005885 trace_seq_printf(s, "overrun: %ld\n", cnt);
5886
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005887 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005888 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5889
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005890 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005891 trace_seq_printf(s, "bytes: %ld\n", cnt);
5892
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09005893 if (trace_clocks[tr->clock_id].in_ns) {
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005894 /* local or global for trace_clock */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005895 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005896 usec_rem = do_div(t, USEC_PER_SEC);
5897 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5898 t, usec_rem);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005899
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005900 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005901 usec_rem = do_div(t, USEC_PER_SEC);
5902 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5903 } else {
5904 /* counter or tsc mode for trace_clock */
5905 trace_seq_printf(s, "oldest event ts: %llu\n",
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005906 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005907
5908 trace_seq_printf(s, "now ts: %llu\n",
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005909 ring_buffer_time_stamp(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005910 }
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005911
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005912 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
Slava Pestov884bfe82011-07-15 14:23:58 -07005913 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5914
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005915 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
Steven Rostedt (Red Hat)ad964702013-01-29 17:45:49 -05005916 trace_seq_printf(s, "read events: %ld\n", cnt);
5917
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05005918 count = simple_read_from_buffer(ubuf, count, ppos,
5919 s->buffer, trace_seq_used(s));
Steven Rostedtc8d77182009-04-29 18:03:45 -04005920
5921 kfree(s);
5922
5923 return count;
5924}
5925
5926static const struct file_operations tracing_stats_fops = {
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005927 .open = tracing_open_generic_tr,
Steven Rostedtc8d77182009-04-29 18:03:45 -04005928 .read = tracing_stats_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005929 .llseek = generic_file_llseek,
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005930 .release = tracing_release_generic_tr,
Steven Rostedtc8d77182009-04-29 18:03:45 -04005931};
5932
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005933#ifdef CONFIG_DYNAMIC_FTRACE
5934
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005935int __weak ftrace_arch_read_dyn_info(char *buf, int size)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005936{
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005937 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005938}
5939
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005940static ssize_t
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005941tracing_read_dyn_info(struct file *filp, char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005942 size_t cnt, loff_t *ppos)
5943{
Steven Rostedta26a2a22008-10-31 00:03:22 -04005944 static char ftrace_dyn_info_buffer[1024];
5945 static DEFINE_MUTEX(dyn_info_mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005946 unsigned long *p = filp->private_data;
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005947 char *buf = ftrace_dyn_info_buffer;
Steven Rostedta26a2a22008-10-31 00:03:22 -04005948 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005949 int r;
5950
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005951 mutex_lock(&dyn_info_mutex);
5952 r = sprintf(buf, "%ld ", *p);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005953
Steven Rostedta26a2a22008-10-31 00:03:22 -04005954 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005955 buf[r++] = '\n';
5956
5957 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5958
5959 mutex_unlock(&dyn_info_mutex);
5960
5961 return r;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005962}
5963
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005964static const struct file_operations tracing_dyn_info_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005965 .open = tracing_open_generic,
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005966 .read = tracing_read_dyn_info,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005967 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005968};
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005969#endif /* CONFIG_DYNAMIC_FTRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005970
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005971#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5972static void
5973ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005974{
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005975 tracing_snapshot();
5976}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005977
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005978static void
5979ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5980{
5981 unsigned long *count = (long *)data;
5982
5983 if (!*count)
5984 return;
5985
5986 if (*count != -1)
5987 (*count)--;
5988
5989 tracing_snapshot();
5990}
5991
5992static int
5993ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5994 struct ftrace_probe_ops *ops, void *data)
5995{
5996 long count = (long)data;
5997
5998 seq_printf(m, "%ps:", (void *)ip);
5999
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01006000 seq_puts(m, "snapshot");
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04006001
6002 if (count == -1)
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01006003 seq_puts(m, ":unlimited\n");
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04006004 else
6005 seq_printf(m, ":count=%ld\n", count);
6006
6007 return 0;
6008}
6009
6010static struct ftrace_probe_ops snapshot_probe_ops = {
6011 .func = ftrace_snapshot,
6012 .print = ftrace_snapshot_print,
6013};
6014
6015static struct ftrace_probe_ops snapshot_count_probe_ops = {
6016 .func = ftrace_count_snapshot,
6017 .print = ftrace_snapshot_print,
6018};
6019
6020static int
6021ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
6022 char *glob, char *cmd, char *param, int enable)
6023{
6024 struct ftrace_probe_ops *ops;
6025 void *count = (void *)-1;
6026 char *number;
6027 int ret;
6028
6029 /* hash funcs only work with set_ftrace_filter */
6030 if (!enable)
6031 return -EINVAL;
6032
6033 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6034
6035 if (glob[0] == '!') {
6036 unregister_ftrace_function_probe_func(glob+1, ops);
6037 return 0;
6038 }
6039
6040 if (!param)
6041 goto out_reg;
6042
6043 number = strsep(&param, ":");
6044
6045 if (!strlen(number))
6046 goto out_reg;
6047
6048 /*
6049 * We use the callback data field (which is a pointer)
6050 * as our counter.
6051 */
6052 ret = kstrtoul(number, 0, (unsigned long *)&count);
6053 if (ret)
6054 return ret;
6055
6056 out_reg:
6057 ret = register_ftrace_function_probe(glob, ops, count);
6058
6059 if (ret >= 0)
6060 alloc_snapshot(&global_trace);
6061
6062 return ret < 0 ? ret : 0;
6063}
6064
6065static struct ftrace_func_command ftrace_snapshot_cmd = {
6066 .name = "snapshot",
6067 .func = ftrace_trace_snapshot_callback,
6068};
6069
Tom Zanussi38de93a2013-10-24 08:34:18 -05006070static __init int register_snapshot_cmd(void)
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04006071{
6072 return register_ftrace_command(&ftrace_snapshot_cmd);
6073}
6074#else
Tom Zanussi38de93a2013-10-24 08:34:18 -05006075static inline __init int register_snapshot_cmd(void) { return 0; }
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04006076#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006077
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006078static struct dentry *tracing_get_dentry(struct trace_array *tr)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006079{
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006080 if (WARN_ON(!tr->dir))
6081 return ERR_PTR(-ENODEV);
6082
6083 /* Top directory uses NULL as the parent */
6084 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6085 return NULL;
6086
6087 /* All sub buffers have a descriptor */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006088 return tr->dir;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006089}
6090
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006091static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6092{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006093 struct dentry *d_tracer;
6094
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006095 if (tr->percpu_dir)
6096 return tr->percpu_dir;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006097
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006098 d_tracer = tracing_get_dentry(tr);
Steven Rostedt (Red Hat)14a5ae42015-01-20 11:14:16 -05006099 if (IS_ERR(d_tracer))
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006100 return NULL;
6101
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006102 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006103
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006104 WARN_ONCE(!tr->percpu_dir,
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006105 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006106
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006107 return tr->percpu_dir;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006108}
6109
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006110static struct dentry *
6111trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6112 void *data, long cpu, const struct file_operations *fops)
6113{
6114 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6115
6116 if (ret) /* See tracing_get_cpu() */
David Howells7682c912015-03-17 22:26:16 +00006117 d_inode(ret)->i_cdev = (void *)(cpu + 1);
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006118 return ret;
6119}
6120
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006121static void
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006122tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006123{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006124 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006125 struct dentry *d_cpu;
Steven Rostedtdd49a382010-10-20 21:51:26 -04006126 char cpu_dir[30]; /* 30 characters should be more than enough */
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006127
Namhyung Kim0a3d7ce2012-04-23 10:11:57 +09006128 if (!d_percpu)
6129 return;
6130
Steven Rostedtdd49a382010-10-20 21:51:26 -04006131 snprintf(cpu_dir, 30, "cpu%ld", cpu);
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006132 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01006133 if (!d_cpu) {
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006134 pr_warning("Could not create tracefs '%s' entry\n", cpu_dir);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01006135 return;
6136 }
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006137
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01006138 /* per cpu trace_pipe */
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006139 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
Oleg Nesterov15544202013-07-23 17:25:57 +02006140 tr, cpu, &tracing_pipe_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006141
6142 /* per cpu trace */
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006143 trace_create_cpu_file("trace", 0644, d_cpu,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006144 tr, cpu, &tracing_fops);
Steven Rostedt7f96f932009-03-13 00:37:42 -04006145
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006146 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02006147 tr, cpu, &tracing_buffers_fops);
Steven Rostedtc8d77182009-04-29 18:03:45 -04006148
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006149 trace_create_cpu_file("stats", 0444, d_cpu,
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02006150 tr, cpu, &tracing_stats_fops);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08006151
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006152 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02006153 tr, cpu, &tracing_entries_fops);
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05006154
6155#ifdef CONFIG_TRACER_SNAPSHOT
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006156 trace_create_cpu_file("snapshot", 0644, d_cpu,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006157 tr, cpu, &snapshot_fops);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05006158
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006159 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02006160 tr, cpu, &snapshot_raw_fops);
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05006161#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006162}
6163
Steven Rostedt60a11772008-05-12 21:20:44 +02006164#ifdef CONFIG_FTRACE_SELFTEST
6165/* Let selftest have access to static functions in this file */
6166#include "trace_selftest.c"
6167#endif
6168
Steven Rostedt577b7852009-02-26 23:43:05 -05006169static ssize_t
6170trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6171 loff_t *ppos)
6172{
6173 struct trace_option_dentry *topt = filp->private_data;
6174 char *buf;
6175
6176 if (topt->flags->val & topt->opt->bit)
6177 buf = "1\n";
6178 else
6179 buf = "0\n";
6180
6181 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6182}
6183
6184static ssize_t
6185trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6186 loff_t *ppos)
6187{
6188 struct trace_option_dentry *topt = filp->private_data;
6189 unsigned long val;
Steven Rostedt577b7852009-02-26 23:43:05 -05006190 int ret;
6191
Peter Huewe22fe9b52011-06-07 21:58:27 +02006192 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6193 if (ret)
Steven Rostedt577b7852009-02-26 23:43:05 -05006194 return ret;
6195
Li Zefan8d18eaa2009-12-08 11:17:06 +08006196 if (val != 0 && val != 1)
Steven Rostedt577b7852009-02-26 23:43:05 -05006197 return -EINVAL;
Li Zefan8d18eaa2009-12-08 11:17:06 +08006198
6199 if (!!(topt->flags->val & topt->opt->bit) != val) {
6200 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05006201 ret = __set_tracer_option(topt->tr, topt->flags,
Steven Rostedtc757bea2009-12-21 22:35:16 -05006202 topt->opt, !val);
Li Zefan8d18eaa2009-12-08 11:17:06 +08006203 mutex_unlock(&trace_types_lock);
6204 if (ret)
6205 return ret;
Steven Rostedt577b7852009-02-26 23:43:05 -05006206 }
6207
6208 *ppos += cnt;
6209
6210 return cnt;
6211}
6212
6213
6214static const struct file_operations trace_options_fops = {
6215 .open = tracing_open_generic,
6216 .read = trace_options_read,
6217 .write = trace_options_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02006218 .llseek = generic_file_llseek,
Steven Rostedt577b7852009-02-26 23:43:05 -05006219};
6220
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006221/*
6222 * In order to pass in both the trace_array descriptor as well as the index
6223 * to the flag that the trace option file represents, the trace_array
6224 * has a character array of trace_flags_index[], which holds the index
6225 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6226 * The address of this character array is passed to the flag option file
6227 * read/write callbacks.
6228 *
6229 * In order to extract both the index and the trace_array descriptor,
6230 * get_tr_index() uses the following algorithm.
6231 *
6232 * idx = *ptr;
6233 *
6234 * As the pointer itself contains the address of the index (remember
6235 * index[1] == 1).
6236 *
6237 * Then to get the trace_array descriptor, by subtracting that index
6238 * from the ptr, we get to the start of the index itself.
6239 *
6240 * ptr - idx == &index[0]
6241 *
6242 * Then a simple container_of() from that pointer gets us to the
6243 * trace_array descriptor.
6244 */
6245static void get_tr_index(void *data, struct trace_array **ptr,
6246 unsigned int *pindex)
6247{
6248 *pindex = *(unsigned char *)data;
6249
6250 *ptr = container_of(data - *pindex, struct trace_array,
6251 trace_flags_index);
6252}
6253
Steven Rostedta8259072009-02-26 22:19:12 -05006254static ssize_t
6255trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6256 loff_t *ppos)
6257{
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006258 void *tr_index = filp->private_data;
6259 struct trace_array *tr;
6260 unsigned int index;
Steven Rostedta8259072009-02-26 22:19:12 -05006261 char *buf;
6262
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006263 get_tr_index(tr_index, &tr, &index);
6264
6265 if (tr->trace_flags & (1 << index))
Steven Rostedta8259072009-02-26 22:19:12 -05006266 buf = "1\n";
6267 else
6268 buf = "0\n";
6269
6270 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6271}
6272
6273static ssize_t
6274trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6275 loff_t *ppos)
6276{
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006277 void *tr_index = filp->private_data;
6278 struct trace_array *tr;
6279 unsigned int index;
Steven Rostedta8259072009-02-26 22:19:12 -05006280 unsigned long val;
6281 int ret;
6282
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006283 get_tr_index(tr_index, &tr, &index);
6284
Peter Huewe22fe9b52011-06-07 21:58:27 +02006285 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6286 if (ret)
Steven Rostedta8259072009-02-26 22:19:12 -05006287 return ret;
6288
Zhaoleif2d84b62009-08-07 18:55:48 +08006289 if (val != 0 && val != 1)
Steven Rostedta8259072009-02-26 22:19:12 -05006290 return -EINVAL;
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04006291
6292 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006293 ret = set_tracer_flag(tr, 1 << index, val);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04006294 mutex_unlock(&trace_types_lock);
Steven Rostedta8259072009-02-26 22:19:12 -05006295
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04006296 if (ret < 0)
6297 return ret;
6298
Steven Rostedta8259072009-02-26 22:19:12 -05006299 *ppos += cnt;
6300
6301 return cnt;
6302}
6303
Steven Rostedta8259072009-02-26 22:19:12 -05006304static const struct file_operations trace_options_core_fops = {
6305 .open = tracing_open_generic,
6306 .read = trace_options_core_read,
6307 .write = trace_options_core_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02006308 .llseek = generic_file_llseek,
Steven Rostedta8259072009-02-26 22:19:12 -05006309};
6310
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006311struct dentry *trace_create_file(const char *name,
Al Virof4ae40a2011-07-24 04:33:43 -04006312 umode_t mode,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006313 struct dentry *parent,
6314 void *data,
6315 const struct file_operations *fops)
6316{
6317 struct dentry *ret;
6318
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006319 ret = tracefs_create_file(name, mode, parent, data, fops);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006320 if (!ret)
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006321 pr_warning("Could not create tracefs '%s' entry\n", name);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006322
6323 return ret;
6324}
6325
6326
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006327static struct dentry *trace_options_init_dentry(struct trace_array *tr)
Steven Rostedta8259072009-02-26 22:19:12 -05006328{
6329 struct dentry *d_tracer;
Steven Rostedta8259072009-02-26 22:19:12 -05006330
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006331 if (tr->options)
6332 return tr->options;
Steven Rostedta8259072009-02-26 22:19:12 -05006333
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006334 d_tracer = tracing_get_dentry(tr);
Steven Rostedt (Red Hat)14a5ae42015-01-20 11:14:16 -05006335 if (IS_ERR(d_tracer))
Steven Rostedta8259072009-02-26 22:19:12 -05006336 return NULL;
6337
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006338 tr->options = tracefs_create_dir("options", d_tracer);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006339 if (!tr->options) {
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006340 pr_warning("Could not create tracefs directory 'options'\n");
Steven Rostedta8259072009-02-26 22:19:12 -05006341 return NULL;
6342 }
6343
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006344 return tr->options;
Steven Rostedta8259072009-02-26 22:19:12 -05006345}
6346
Steven Rostedt577b7852009-02-26 23:43:05 -05006347static void
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006348create_trace_option_file(struct trace_array *tr,
6349 struct trace_option_dentry *topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05006350 struct tracer_flags *flags,
6351 struct tracer_opt *opt)
6352{
6353 struct dentry *t_options;
Steven Rostedt577b7852009-02-26 23:43:05 -05006354
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006355 t_options = trace_options_init_dentry(tr);
Steven Rostedt577b7852009-02-26 23:43:05 -05006356 if (!t_options)
6357 return;
6358
6359 topt->flags = flags;
6360 topt->opt = opt;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006361 topt->tr = tr;
Steven Rostedt577b7852009-02-26 23:43:05 -05006362
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006363 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05006364 &trace_options_fops);
6365
Steven Rostedt577b7852009-02-26 23:43:05 -05006366}
6367
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006368static void
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006369create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
Steven Rostedt577b7852009-02-26 23:43:05 -05006370{
6371 struct trace_option_dentry *topts;
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006372 struct trace_options *tr_topts;
Steven Rostedt577b7852009-02-26 23:43:05 -05006373 struct tracer_flags *flags;
6374 struct tracer_opt *opts;
6375 int cnt;
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006376 int i;
Steven Rostedt577b7852009-02-26 23:43:05 -05006377
6378 if (!tracer)
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006379 return;
Steven Rostedt577b7852009-02-26 23:43:05 -05006380
6381 flags = tracer->flags;
6382
6383 if (!flags || !flags->opts)
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006384 return;
6385
6386 /*
6387 * If this is an instance, only create flags for tracers
6388 * the instance may have.
6389 */
6390 if (!trace_ok_for_array(tracer, tr))
6391 return;
6392
6393 for (i = 0; i < tr->nr_topts; i++) {
6394 /*
6395 * Check if these flags have already been added.
6396 * Some tracers share flags.
6397 */
6398 if (tr->topts[i].tracer->flags == tracer->flags)
6399 return;
6400 }
Steven Rostedt577b7852009-02-26 23:43:05 -05006401
6402 opts = flags->opts;
6403
6404 for (cnt = 0; opts[cnt].name; cnt++)
6405 ;
6406
Steven Rostedt0cfe8242009-02-27 10:51:10 -05006407 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
Steven Rostedt577b7852009-02-26 23:43:05 -05006408 if (!topts)
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006409 return;
6410
6411 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
6412 GFP_KERNEL);
6413 if (!tr_topts) {
6414 kfree(topts);
6415 return;
6416 }
6417
6418 tr->topts = tr_topts;
6419 tr->topts[tr->nr_topts].tracer = tracer;
6420 tr->topts[tr->nr_topts].topts = topts;
6421 tr->nr_topts++;
Steven Rostedt577b7852009-02-26 23:43:05 -05006422
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04006423 for (cnt = 0; opts[cnt].name; cnt++) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006424 create_trace_option_file(tr, &topts[cnt], flags,
Steven Rostedt577b7852009-02-26 23:43:05 -05006425 &opts[cnt]);
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04006426 WARN_ONCE(topts[cnt].entry == NULL,
6427 "Failed to create trace option: %s",
6428 opts[cnt].name);
6429 }
Steven Rostedt577b7852009-02-26 23:43:05 -05006430}
6431
Steven Rostedta8259072009-02-26 22:19:12 -05006432static struct dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006433create_trace_option_core_file(struct trace_array *tr,
6434 const char *option, long index)
Steven Rostedta8259072009-02-26 22:19:12 -05006435{
6436 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05006437
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006438 t_options = trace_options_init_dentry(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05006439 if (!t_options)
6440 return NULL;
6441
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006442 return trace_create_file(option, 0644, t_options,
6443 (void *)&tr->trace_flags_index[index],
6444 &trace_options_core_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05006445}
6446
Steven Rostedt (Red Hat)16270142015-09-30 12:30:06 -04006447static void create_trace_options_dir(struct trace_array *tr)
Steven Rostedta8259072009-02-26 22:19:12 -05006448{
6449 struct dentry *t_options;
Steven Rostedt (Red Hat)16270142015-09-30 12:30:06 -04006450 bool top_level = tr == &global_trace;
Steven Rostedta8259072009-02-26 22:19:12 -05006451 int i;
6452
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006453 t_options = trace_options_init_dentry(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05006454 if (!t_options)
6455 return;
6456
Steven Rostedt (Red Hat)16270142015-09-30 12:30:06 -04006457 for (i = 0; trace_options[i]; i++) {
6458 if (top_level ||
6459 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
6460 create_trace_option_core_file(tr, trace_options[i], i);
6461 }
Steven Rostedta8259072009-02-26 22:19:12 -05006462}
6463
Steven Rostedt499e5472012-02-22 15:50:28 -05006464static ssize_t
6465rb_simple_read(struct file *filp, char __user *ubuf,
6466 size_t cnt, loff_t *ppos)
6467{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04006468 struct trace_array *tr = filp->private_data;
Steven Rostedt499e5472012-02-22 15:50:28 -05006469 char buf[64];
6470 int r;
6471
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04006472 r = tracer_tracing_is_on(tr);
Steven Rostedt499e5472012-02-22 15:50:28 -05006473 r = sprintf(buf, "%d\n", r);
6474
6475 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6476}
6477
6478static ssize_t
6479rb_simple_write(struct file *filp, const char __user *ubuf,
6480 size_t cnt, loff_t *ppos)
6481{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04006482 struct trace_array *tr = filp->private_data;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006483 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt499e5472012-02-22 15:50:28 -05006484 unsigned long val;
6485 int ret;
6486
6487 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6488 if (ret)
6489 return ret;
6490
6491 if (buffer) {
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05006492 mutex_lock(&trace_types_lock);
6493 if (val) {
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04006494 tracer_tracing_on(tr);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006495 if (tr->current_trace->start)
6496 tr->current_trace->start(tr);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05006497 } else {
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04006498 tracer_tracing_off(tr);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006499 if (tr->current_trace->stop)
6500 tr->current_trace->stop(tr);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05006501 }
6502 mutex_unlock(&trace_types_lock);
Steven Rostedt499e5472012-02-22 15:50:28 -05006503 }
6504
6505 (*ppos)++;
6506
6507 return cnt;
6508}
6509
6510static const struct file_operations rb_simple_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04006511 .open = tracing_open_generic_tr,
Steven Rostedt499e5472012-02-22 15:50:28 -05006512 .read = rb_simple_read,
6513 .write = rb_simple_write,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04006514 .release = tracing_release_generic_tr,
Steven Rostedt499e5472012-02-22 15:50:28 -05006515 .llseek = default_llseek,
6516};
6517
Steven Rostedt277ba042012-08-03 16:10:49 -04006518struct dentry *trace_instance_dir;
6519
6520static void
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006521init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
Steven Rostedt277ba042012-08-03 16:10:49 -04006522
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006523static int
6524allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
Steven Rostedt277ba042012-08-03 16:10:49 -04006525{
6526 enum ring_buffer_flags rb_flags;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006527
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04006528 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006529
Steven Rostedt (Red Hat)dced3412014-01-14 10:19:46 -05006530 buf->tr = tr;
6531
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006532 buf->buffer = ring_buffer_alloc(size, rb_flags);
6533 if (!buf->buffer)
6534 return -ENOMEM;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006535
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006536 buf->data = alloc_percpu(struct trace_array_cpu);
6537 if (!buf->data) {
6538 ring_buffer_free(buf->buffer);
6539 return -ENOMEM;
6540 }
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006541
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006542 /* Allocate the first page for all buffers */
6543 set_buffer_entries(&tr->trace_buffer,
6544 ring_buffer_size(tr->trace_buffer.buffer, 0));
6545
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006546 return 0;
6547}
6548
6549static int allocate_trace_buffers(struct trace_array *tr, int size)
6550{
6551 int ret;
6552
6553 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6554 if (ret)
6555 return ret;
6556
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006557#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006558 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6559 allocate_snapshot ? size : 1);
6560 if (WARN_ON(ret)) {
6561 ring_buffer_free(tr->trace_buffer.buffer);
6562 free_percpu(tr->trace_buffer.data);
6563 return -ENOMEM;
6564 }
6565 tr->allocated_snapshot = allocate_snapshot;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006566
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006567 /*
6568 * Only the top level trace array gets its snapshot allocated
6569 * from the kernel command line.
6570 */
6571 allocate_snapshot = false;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006572#endif
6573 return 0;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006574}
6575
Steven Rostedt (Red Hat)f0b70cc2014-06-10 12:06:30 -04006576static void free_trace_buffer(struct trace_buffer *buf)
6577{
6578 if (buf->buffer) {
6579 ring_buffer_free(buf->buffer);
6580 buf->buffer = NULL;
6581 free_percpu(buf->data);
6582 buf->data = NULL;
6583 }
6584}
6585
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006586static void free_trace_buffers(struct trace_array *tr)
6587{
6588 if (!tr)
6589 return;
6590
Steven Rostedt (Red Hat)f0b70cc2014-06-10 12:06:30 -04006591 free_trace_buffer(&tr->trace_buffer);
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006592
6593#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)f0b70cc2014-06-10 12:06:30 -04006594 free_trace_buffer(&tr->max_buffer);
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006595#endif
6596}
6597
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006598static void init_trace_flags_index(struct trace_array *tr)
6599{
6600 int i;
6601
6602 /* Used by the trace options files */
6603 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
6604 tr->trace_flags_index[i] = i;
6605}
6606
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006607static void __update_tracer_options(struct trace_array *tr)
6608{
6609 struct tracer *t;
6610
6611 for (t = trace_types; t; t = t->next)
6612 add_tracer_options(tr, t);
6613}
6614
6615static void update_tracer_options(struct trace_array *tr)
6616{
6617 mutex_lock(&trace_types_lock);
6618 __update_tracer_options(tr);
6619 mutex_unlock(&trace_types_lock);
6620}
6621
Steven Rostedt (Red Hat)eae47352015-01-21 10:01:39 -05006622static int instance_mkdir(const char *name)
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006623{
Steven Rostedt277ba042012-08-03 16:10:49 -04006624 struct trace_array *tr;
6625 int ret;
Steven Rostedt277ba042012-08-03 16:10:49 -04006626
6627 mutex_lock(&trace_types_lock);
6628
6629 ret = -EEXIST;
6630 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6631 if (tr->name && strcmp(tr->name, name) == 0)
6632 goto out_unlock;
6633 }
6634
6635 ret = -ENOMEM;
6636 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6637 if (!tr)
6638 goto out_unlock;
6639
6640 tr->name = kstrdup(name, GFP_KERNEL);
6641 if (!tr->name)
6642 goto out_free_tr;
6643
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006644 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6645 goto out_free_tr;
6646
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04006647 tr->trace_flags = global_trace.trace_flags;
6648
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006649 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6650
Steven Rostedt277ba042012-08-03 16:10:49 -04006651 raw_spin_lock_init(&tr->start_lock);
6652
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05006653 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6654
Steven Rostedt277ba042012-08-03 16:10:49 -04006655 tr->current_trace = &nop_trace;
6656
6657 INIT_LIST_HEAD(&tr->systems);
6658 INIT_LIST_HEAD(&tr->events);
6659
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006660 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
Steven Rostedt277ba042012-08-03 16:10:49 -04006661 goto out_free_tr;
6662
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006663 tr->dir = tracefs_create_dir(name, trace_instance_dir);
Steven Rostedt277ba042012-08-03 16:10:49 -04006664 if (!tr->dir)
6665 goto out_free_tr;
6666
6667 ret = event_trace_add_tracer(tr->dir, tr);
Alexander Z Lam609e85a2013-07-10 17:34:34 -07006668 if (ret) {
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006669 tracefs_remove_recursive(tr->dir);
Steven Rostedt277ba042012-08-03 16:10:49 -04006670 goto out_free_tr;
Alexander Z Lam609e85a2013-07-10 17:34:34 -07006671 }
Steven Rostedt277ba042012-08-03 16:10:49 -04006672
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006673 init_tracer_tracefs(tr, tr->dir);
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006674 init_trace_flags_index(tr);
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006675 __update_tracer_options(tr);
Steven Rostedt277ba042012-08-03 16:10:49 -04006676
6677 list_add(&tr->list, &ftrace_trace_arrays);
6678
6679 mutex_unlock(&trace_types_lock);
6680
6681 return 0;
6682
6683 out_free_tr:
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006684 free_trace_buffers(tr);
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006685 free_cpumask_var(tr->tracing_cpumask);
Steven Rostedt277ba042012-08-03 16:10:49 -04006686 kfree(tr->name);
6687 kfree(tr);
6688
6689 out_unlock:
6690 mutex_unlock(&trace_types_lock);
6691
6692 return ret;
6693
6694}
6695
Steven Rostedt (Red Hat)eae47352015-01-21 10:01:39 -05006696static int instance_rmdir(const char *name)
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006697{
6698 struct trace_array *tr;
6699 int found = 0;
6700 int ret;
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006701 int i;
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006702
6703 mutex_lock(&trace_types_lock);
6704
6705 ret = -ENODEV;
6706 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6707 if (tr->name && strcmp(tr->name, name) == 0) {
6708 found = 1;
6709 break;
6710 }
6711 }
6712 if (!found)
6713 goto out_unlock;
6714
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05006715 ret = -EBUSY;
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05006716 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05006717 goto out_unlock;
6718
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006719 list_del(&tr->list);
6720
Steven Rostedt (Red Hat)6b450d22014-01-14 08:43:01 -05006721 tracing_set_nop(tr);
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006722 event_trace_del_tracer(tr);
Steven Rostedt (Red Hat)591dffd2014-01-10 16:17:45 -05006723 ftrace_destroy_function_files(tr);
Jiaxing Wang681a4a22015-10-18 19:58:08 +08006724 tracefs_remove_recursive(tr->dir);
Steven Rostedt (Red Hat)a9fcaaa2014-06-06 23:17:28 -04006725 free_trace_buffers(tr);
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006726
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006727 for (i = 0; i < tr->nr_topts; i++) {
6728 kfree(tr->topts[i].topts);
6729 }
6730 kfree(tr->topts);
6731
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006732 kfree(tr->name);
6733 kfree(tr);
6734
6735 ret = 0;
6736
6737 out_unlock:
6738 mutex_unlock(&trace_types_lock);
6739
6740 return ret;
6741}
6742
Steven Rostedt277ba042012-08-03 16:10:49 -04006743static __init void create_trace_instances(struct dentry *d_tracer)
6744{
Steven Rostedt (Red Hat)eae47352015-01-21 10:01:39 -05006745 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
6746 instance_mkdir,
6747 instance_rmdir);
Steven Rostedt277ba042012-08-03 16:10:49 -04006748 if (WARN_ON(!trace_instance_dir))
6749 return;
Steven Rostedt277ba042012-08-03 16:10:49 -04006750}
6751
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006752static void
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006753init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006754{
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006755 int cpu;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006756
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05006757 trace_create_file("available_tracers", 0444, d_tracer,
6758 tr, &show_traces_fops);
6759
6760 trace_create_file("current_tracer", 0644, d_tracer,
6761 tr, &set_tracer_fops);
6762
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006763 trace_create_file("tracing_cpumask", 0644, d_tracer,
6764 tr, &tracing_cpumask_fops);
6765
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006766 trace_create_file("trace_options", 0644, d_tracer,
6767 tr, &tracing_iter_fops);
6768
6769 trace_create_file("trace", 0644, d_tracer,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006770 tr, &tracing_fops);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006771
6772 trace_create_file("trace_pipe", 0444, d_tracer,
Oleg Nesterov15544202013-07-23 17:25:57 +02006773 tr, &tracing_pipe_fops);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006774
6775 trace_create_file("buffer_size_kb", 0644, d_tracer,
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02006776 tr, &tracing_entries_fops);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006777
6778 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6779 tr, &tracing_total_entries_fops);
6780
Wang YanQing238ae932013-05-26 16:52:01 +08006781 trace_create_file("free_buffer", 0200, d_tracer,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006782 tr, &tracing_free_buffer_fops);
6783
6784 trace_create_file("trace_marker", 0220, d_tracer,
6785 tr, &tracing_mark_fops);
6786
6787 trace_create_file("trace_clock", 0644, d_tracer, tr,
6788 &trace_clock_fops);
6789
6790 trace_create_file("tracing_on", 0644, d_tracer,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006791 tr, &rb_simple_fops);
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05006792
Steven Rostedt (Red Hat)16270142015-09-30 12:30:06 -04006793 create_trace_options_dir(tr);
6794
Steven Rostedt (Red Hat)6d9b3fa2014-01-14 11:28:38 -05006795#ifdef CONFIG_TRACER_MAX_TRACE
6796 trace_create_file("tracing_max_latency", 0644, d_tracer,
6797 &tr->max_latency, &tracing_max_lat_fops);
6798#endif
6799
Steven Rostedt (Red Hat)591dffd2014-01-10 16:17:45 -05006800 if (ftrace_create_function_files(tr, d_tracer))
6801 WARN(1, "Could not allocate function filter files");
6802
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05006803#ifdef CONFIG_TRACER_SNAPSHOT
6804 trace_create_file("snapshot", 0644, d_tracer,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006805 tr, &snapshot_fops);
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05006806#endif
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006807
6808 for_each_tracing_cpu(cpu)
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006809 tracing_init_tracefs_percpu(tr, cpu);
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006810
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006811}
6812
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006813static struct vfsmount *trace_automount(void *ingore)
6814{
6815 struct vfsmount *mnt;
6816 struct file_system_type *type;
6817
6818 /*
6819 * To maintain backward compatibility for tools that mount
6820 * debugfs to get to the tracing facility, tracefs is automatically
6821 * mounted to the debugfs/tracing directory.
6822 */
6823 type = get_fs_type("tracefs");
6824 if (!type)
6825 return NULL;
6826 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
6827 put_filesystem(type);
6828 if (IS_ERR(mnt))
6829 return NULL;
6830 mntget(mnt);
6831
6832 return mnt;
6833}
6834
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006835/**
6836 * tracing_init_dentry - initialize top level trace array
6837 *
6838 * This is called when creating files or directories in the tracing
6839 * directory. It is called via fs_initcall() by any of the boot up code
6840 * and expects to return the dentry of the top level tracing directory.
6841 */
6842struct dentry *tracing_init_dentry(void)
6843{
6844 struct trace_array *tr = &global_trace;
6845
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006846 /* The top level trace array uses NULL as parent */
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006847 if (tr->dir)
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006848 return NULL;
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006849
6850 if (WARN_ON(!debugfs_initialized()))
6851 return ERR_PTR(-ENODEV);
6852
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006853 /*
6854 * As there may still be users that expect the tracing
6855 * files to exist in debugfs/tracing, we must automount
6856 * the tracefs file system there, so older tools still
6857 * work with the newer kerenl.
6858 */
6859 tr->dir = debugfs_create_automount("tracing", NULL,
6860 trace_automount, NULL);
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006861 if (!tr->dir) {
6862 pr_warn_once("Could not create debugfs directory 'tracing'\n");
6863 return ERR_PTR(-ENOMEM);
6864 }
6865
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006866 return NULL;
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006867}
6868
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04006869extern struct trace_enum_map *__start_ftrace_enum_maps[];
6870extern struct trace_enum_map *__stop_ftrace_enum_maps[];
6871
6872static void __init trace_enum_init(void)
6873{
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006874 int len;
6875
6876 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006877 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04006878}
6879
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006880#ifdef CONFIG_MODULES
6881static void trace_module_add_enums(struct module *mod)
6882{
6883 if (!mod->num_trace_enums)
6884 return;
6885
6886 /*
6887 * Modules with bad taint do not have events created, do
6888 * not bother with enums either.
6889 */
6890 if (trace_module_has_bad_taint(mod))
6891 return;
6892
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006893 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006894}
6895
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006896#ifdef CONFIG_TRACE_ENUM_MAP_FILE
6897static void trace_module_remove_enums(struct module *mod)
6898{
6899 union trace_enum_map_item *map;
6900 union trace_enum_map_item **last = &trace_enum_maps;
6901
6902 if (!mod->num_trace_enums)
6903 return;
6904
6905 mutex_lock(&trace_enum_mutex);
6906
6907 map = trace_enum_maps;
6908
6909 while (map) {
6910 if (map->head.mod == mod)
6911 break;
6912 map = trace_enum_jmp_to_tail(map);
6913 last = &map->tail.next;
6914 map = map->tail.next;
6915 }
6916 if (!map)
6917 goto out;
6918
6919 *last = trace_enum_jmp_to_tail(map)->tail.next;
6920 kfree(map);
6921 out:
6922 mutex_unlock(&trace_enum_mutex);
6923}
6924#else
6925static inline void trace_module_remove_enums(struct module *mod) { }
6926#endif /* CONFIG_TRACE_ENUM_MAP_FILE */
6927
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006928static int trace_module_notify(struct notifier_block *self,
6929 unsigned long val, void *data)
6930{
6931 struct module *mod = data;
6932
6933 switch (val) {
6934 case MODULE_STATE_COMING:
6935 trace_module_add_enums(mod);
6936 break;
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006937 case MODULE_STATE_GOING:
6938 trace_module_remove_enums(mod);
6939 break;
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006940 }
6941
6942 return 0;
6943}
6944
6945static struct notifier_block trace_module_nb = {
6946 .notifier_call = trace_module_notify,
6947 .priority = 0,
6948};
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006949#endif /* CONFIG_MODULES */
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006950
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006951static __init int tracer_init_tracefs(void)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006952{
6953 struct dentry *d_tracer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006954
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08006955 trace_access_lock_init();
6956
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006957 d_tracer = tracing_init_dentry();
Steven Rostedt (Red Hat)14a5ae42015-01-20 11:14:16 -05006958 if (IS_ERR(d_tracer))
Namhyung Kimed6f1c92013-04-10 09:18:12 +09006959 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006960
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006961 init_tracer_tracefs(&global_trace, d_tracer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006962
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006963 trace_create_file("tracing_thresh", 0644, d_tracer,
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04006964 &global_trace, &tracing_thresh_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006965
Li Zefan339ae5d2009-04-17 10:34:30 +08006966 trace_create_file("README", 0444, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006967 NULL, &tracing_readme_fops);
Ingo Molnar7bd2f242008-05-12 21:20:45 +02006968
Avadh Patel69abe6a2009-04-10 16:04:48 -04006969 trace_create_file("saved_cmdlines", 0444, d_tracer,
6970 NULL, &tracing_saved_cmdlines_fops);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03006971
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09006972 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6973 NULL, &tracing_saved_cmdlines_size_fops);
6974
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04006975 trace_enum_init();
6976
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006977 trace_create_enum_file(d_tracer);
6978
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006979#ifdef CONFIG_MODULES
6980 register_module_notifier(&trace_module_nb);
6981#endif
6982
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006983#ifdef CONFIG_DYNAMIC_FTRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006984 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6985 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006986#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006987
Steven Rostedt277ba042012-08-03 16:10:49 -04006988 create_trace_instances(d_tracer);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09006989
Steven Rostedt (Red Hat)37aea982015-09-30 14:27:31 -04006990 update_tracer_options(&global_trace);
Steven Rostedt (Red Hat)09d23a12015-02-03 12:45:53 -05006991
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01006992 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006993}
6994
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006995static int trace_panic_handler(struct notifier_block *this,
6996 unsigned long event, void *unused)
6997{
Steven Rostedt944ac422008-10-23 19:26:08 -04006998 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006999 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007000 return NOTIFY_OK;
7001}
7002
7003static struct notifier_block trace_panic_notifier = {
7004 .notifier_call = trace_panic_handler,
7005 .next = NULL,
7006 .priority = 150 /* priority: INT_MAX >= x >= 0 */
7007};
7008
7009static int trace_die_handler(struct notifier_block *self,
7010 unsigned long val,
7011 void *data)
7012{
7013 switch (val) {
7014 case DIE_OOPS:
Steven Rostedt944ac422008-10-23 19:26:08 -04007015 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007016 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007017 break;
7018 default:
7019 break;
7020 }
7021 return NOTIFY_OK;
7022}
7023
7024static struct notifier_block trace_die_notifier = {
7025 .notifier_call = trace_die_handler,
7026 .priority = 200
7027};
7028
7029/*
7030 * printk is set to max of 1024, we really don't need it that big.
7031 * Nothing should be printing 1000 characters anyway.
7032 */
7033#define TRACE_MAX_PRINT 1000
7034
7035/*
7036 * Define here KERN_TRACE so that we have one place to modify
7037 * it if we decide to change what log level the ftrace dump
7038 * should be at.
7039 */
Steven Rostedt428aee12009-01-14 12:24:42 -05007040#define KERN_TRACE KERN_EMERG
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007041
Jason Wessel955b61e2010-08-05 09:22:23 -05007042void
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007043trace_printk_seq(struct trace_seq *s)
7044{
7045 /* Probably should print a warning here. */
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04007046 if (s->seq.len >= TRACE_MAX_PRINT)
7047 s->seq.len = TRACE_MAX_PRINT;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007048
Steven Rostedt (Red Hat)820b75f2014-11-19 10:56:41 -05007049 /*
7050 * More paranoid code. Although the buffer size is set to
7051 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7052 * an extra layer of protection.
7053 */
7054 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
7055 s->seq.len = s->seq.size - 1;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007056
7057 /* should be zero ended, but we are paranoid. */
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04007058 s->buffer[s->seq.len] = 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007059
7060 printk(KERN_TRACE "%s", s->buffer);
7061
Steven Rostedtf9520752009-03-02 14:04:40 -05007062 trace_seq_init(s);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007063}
7064
Jason Wessel955b61e2010-08-05 09:22:23 -05007065void trace_init_global_iter(struct trace_iterator *iter)
7066{
7067 iter->tr = &global_trace;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04007068 iter->trace = iter->tr->current_trace;
Steven Rostedtae3b5092013-01-23 15:22:59 -05007069 iter->cpu_file = RING_BUFFER_ALL_CPUS;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05007070 iter->trace_buffer = &global_trace.trace_buffer;
Cody P Schaferb2f974d2013-10-23 11:49:57 -07007071
7072 if (iter->trace && iter->trace->open)
7073 iter->trace->open(iter);
7074
7075 /* Annotate start of buffers if we had overruns */
7076 if (ring_buffer_overruns(iter->trace_buffer->buffer))
7077 iter->iter_flags |= TRACE_FILE_ANNOTATE;
7078
7079 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7080 if (trace_clocks[iter->tr->clock_id].in_ns)
7081 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
Jason Wessel955b61e2010-08-05 09:22:23 -05007082}
7083
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007084void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007085{
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007086 /* use static because iter can be a bit big for the stack */
7087 static struct trace_iterator iter;
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007088 static atomic_t dump_running;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007089 struct trace_array *tr = &global_trace;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007090 unsigned int old_userobj;
Steven Rostedtd7690412008-10-01 00:29:53 -04007091 unsigned long flags;
7092 int cnt = 0, cpu;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007093
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007094 /* Only allow one dump user at a time. */
7095 if (atomic_inc_return(&dump_running) != 1) {
7096 atomic_dec(&dump_running);
7097 return;
Steven Rostedte0a413f2011-09-29 21:26:16 -04007098 }
7099
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007100 /*
7101 * Always turn off tracing when we dump.
7102 * We don't need to show trace output of what happens
7103 * between multiple crashes.
7104 *
7105 * If the user does a sysrq-z, then they can re-enable
7106 * tracing with echo 1 > tracing_on.
7107 */
7108 tracing_off();
7109
7110 local_irq_save(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007111
Jovi Zhang38dbe0b2013-01-25 18:03:07 +08007112 /* Simulate the iterator */
Jason Wessel955b61e2010-08-05 09:22:23 -05007113 trace_init_global_iter(&iter);
7114
Steven Rostedtd7690412008-10-01 00:29:53 -04007115 for_each_tracing_cpu(cpu) {
Umesh Tiwari5e2d5ef2015-06-22 16:55:06 +05307116 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -04007117 }
7118
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007119 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007120
Török Edwinb54d3de2008-11-22 13:28:48 +02007121 /* don't look at user memory in panic mode */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007122 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
Török Edwinb54d3de2008-11-22 13:28:48 +02007123
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007124 switch (oops_dump_mode) {
7125 case DUMP_ALL:
Steven Rostedtae3b5092013-01-23 15:22:59 -05007126 iter.cpu_file = RING_BUFFER_ALL_CPUS;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007127 break;
7128 case DUMP_ORIG:
7129 iter.cpu_file = raw_smp_processor_id();
7130 break;
7131 case DUMP_NONE:
7132 goto out_enable;
7133 default:
7134 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
Steven Rostedtae3b5092013-01-23 15:22:59 -05007135 iter.cpu_file = RING_BUFFER_ALL_CPUS;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007136 }
7137
7138 printk(KERN_TRACE "Dumping ftrace buffer:\n");
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007139
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007140 /* Did function tracer already get disabled? */
7141 if (ftrace_is_dead()) {
7142 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7143 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7144 }
7145
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007146 /*
7147 * We need to stop all tracing on all CPUS to read the
7148 * the next buffer. This is a bit expensive, but is
7149 * not done often. We fill all what we can read,
7150 * and then release the locks again.
7151 */
7152
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007153 while (!trace_empty(&iter)) {
7154
7155 if (!cnt)
7156 printk(KERN_TRACE "---------------------------------\n");
7157
7158 cnt++;
7159
7160 /* reset all but tr, trace, and overruns */
7161 memset(&iter.seq, 0,
7162 sizeof(struct trace_iterator) -
7163 offsetof(struct trace_iterator, seq));
7164 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7165 iter.pos = -1;
7166
Jason Wessel955b61e2010-08-05 09:22:23 -05007167 if (trace_find_next_entry_inc(&iter) != NULL) {
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08007168 int ret;
7169
7170 ret = print_trace_line(&iter);
7171 if (ret != TRACE_TYPE_NO_CONSUME)
7172 trace_consume(&iter);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007173 }
Steven Rostedtb892e5c2012-03-01 22:06:48 -05007174 touch_nmi_watchdog();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007175
7176 trace_printk_seq(&iter.seq);
7177 }
7178
7179 if (!cnt)
7180 printk(KERN_TRACE " (ftrace buffer empty)\n");
7181 else
7182 printk(KERN_TRACE "---------------------------------\n");
7183
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007184 out_enable:
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007185 tr->trace_flags |= old_userobj;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007186
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007187 for_each_tracing_cpu(cpu) {
7188 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007189 }
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007190 atomic_dec(&dump_running);
Steven Rostedtcd891ae2009-04-28 11:39:34 -04007191 local_irq_restore(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007192}
Paul E. McKenneya8eecf22011-10-02 11:01:15 -07007193EXPORT_SYMBOL_GPL(ftrace_dump);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007194
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02007195__init static int tracer_alloc_buffers(void)
7196{
Steven Rostedt73c51622009-03-11 13:42:01 -04007197 int ring_buf_size;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307198 int ret = -ENOMEM;
7199
Steven Rostedt (Red Hat)b5e87c02015-09-29 18:13:33 -04007200 /*
7201 * Make sure we don't accidently add more trace options
7202 * than we have bits for.
7203 */
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04007204 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
Steven Rostedt (Red Hat)b5e87c02015-09-29 18:13:33 -04007205
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307206 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7207 goto out;
7208
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07007209 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307210 goto out_free_buffer_mask;
7211
Steven Rostedt07d777f2011-09-22 14:01:55 -04007212 /* Only allocate trace_printk buffers if a trace_printk exists */
7213 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
Steven Rostedt81698832012-10-11 10:15:05 -04007214 /* Must be called before global_trace.buffer is allocated */
Steven Rostedt07d777f2011-09-22 14:01:55 -04007215 trace_printk_init_buffers();
7216
Steven Rostedt73c51622009-03-11 13:42:01 -04007217 /* To save memory, keep the ring buffer size to its minimum */
7218 if (ring_buffer_expanded)
7219 ring_buf_size = trace_buf_size;
7220 else
7221 ring_buf_size = 1;
7222
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307223 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07007224 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02007225
Steven Rostedt2b6080f2012-05-11 13:29:49 -04007226 raw_spin_lock_init(&global_trace.start_lock);
7227
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04007228 /* Used for event triggers */
7229 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7230 if (!temp_buffer)
7231 goto out_free_cpumask;
7232
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09007233 if (trace_create_savedcmd() < 0)
7234 goto out_free_temp_buffer;
7235
Steven Rostedtab464282008-05-12 21:21:00 +02007236 /* TODO: make the number of buffers hot pluggable with CPUS */
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05007237 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007238 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7239 WARN_ON(1);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09007240 goto out_free_savedcmd;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007241 }
Steven Rostedta7603ff2012-08-06 16:24:11 -04007242
Steven Rostedt499e5472012-02-22 15:50:28 -05007243 if (global_trace.buffer_disabled)
7244 tracing_off();
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007245
Steven Rostedte1e232c2014-02-10 23:38:46 -05007246 if (trace_boot_clock) {
7247 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7248 if (ret < 0)
7249 pr_warning("Trace clock %s not defined, going back to default\n",
7250 trace_boot_clock);
7251 }
7252
Steven Rostedt (Red Hat)ca164312013-05-23 11:51:10 -04007253 /*
7254 * register_tracer() might reference current_trace, so it
7255 * needs to be set before we register anything. This is
7256 * just a bootstrap of current_trace anyway.
7257 */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04007258 global_trace.current_trace = &nop_trace;
7259
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05007260 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7261
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -05007262 ftrace_init_global_array_ops(&global_trace);
7263
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04007264 init_trace_flags_index(&global_trace);
7265
Steven Rostedt (Red Hat)ca164312013-05-23 11:51:10 -04007266 register_tracer(&nop_trace);
7267
Steven Rostedt60a11772008-05-12 21:20:44 +02007268 /* All seems OK, enable tracing */
7269 tracing_disabled = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007270
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007271 atomic_notifier_chain_register(&panic_notifier_list,
7272 &trace_panic_notifier);
7273
7274 register_die_notifier(&trace_die_notifier);
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01007275
Steven Rostedtae63b312012-05-03 23:09:03 -04007276 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7277
7278 INIT_LIST_HEAD(&global_trace.systems);
7279 INIT_LIST_HEAD(&global_trace.events);
7280 list_add(&global_trace.list, &ftrace_trace_arrays);
7281
Jiaxing Wanga4d1e682015-11-04 09:14:29 +08007282 apply_trace_boot_options();
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04007283
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04007284 register_snapshot_cmd();
7285
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01007286 return 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007287
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09007288out_free_savedcmd:
7289 free_saved_cmdlines_buffer(savedcmd);
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04007290out_free_temp_buffer:
7291 ring_buffer_free(temp_buffer);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307292out_free_cpumask:
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07007293 free_cpumask_var(global_trace.tracing_cpumask);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307294out_free_buffer_mask:
7295 free_cpumask_var(tracing_buffer_mask);
7296out:
7297 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02007298}
Steven Rostedtb2821ae2009-02-02 21:38:32 -05007299
Steven Rostedt (Red Hat)5f893b22014-12-12 20:05:10 -05007300void __init trace_init(void)
7301{
Steven Rostedt (Red Hat)0daa23022014-12-12 22:27:10 -05007302 if (tracepoint_printk) {
7303 tracepoint_print_iter =
7304 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7305 if (WARN_ON(!tracepoint_print_iter))
7306 tracepoint_printk = 0;
7307 }
Steven Rostedt (Red Hat)5f893b22014-12-12 20:05:10 -05007308 tracer_alloc_buffers();
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04007309 trace_event_init();
Steven Rostedt (Red Hat)5f893b22014-12-12 20:05:10 -05007310}
7311
Steven Rostedtb2821ae2009-02-02 21:38:32 -05007312__init static int clear_boot_tracer(void)
7313{
7314 /*
7315 * The default tracer at boot buffer is an init section.
7316 * This function is called in lateinit. If we did not
7317 * find the boot tracer, then clear it out, to prevent
7318 * later registration from accessing the buffer that is
7319 * about to be freed.
7320 */
7321 if (!default_bootup_tracer)
7322 return 0;
7323
7324 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7325 default_bootup_tracer);
7326 default_bootup_tracer = NULL;
7327
7328 return 0;
7329}
7330
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05007331fs_initcall(tracer_init_tracefs);
Steven Rostedtb2821ae2009-02-02 21:38:32 -05007332late_initcall(clear_boot_tracer);