blob: 51697b41f5d47bc5528cee710337c6742dbc7b05 [file] [log] [blame]
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001/*
2 * ring buffer based function tracer
3 *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6 *
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
9 *
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
Nadia Yvette Chambers6d49e352012-12-06 10:39:54 +010012 * Copyright (C) 2004 Nadia Yvette Chambers
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020013 */
Steven Rostedt2cadf912008-12-01 22:20:19 -050014#include <linux/ring_buffer.h>
Sam Ravnborg273b2812009-10-18 00:52:28 +020015#include <generated/utsrelease.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050016#include <linux/stacktrace.h>
17#include <linux/writeback.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020018#include <linux/kallsyms.h>
19#include <linux/seq_file.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040020#include <linux/notifier.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050021#include <linux/irqflags.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020022#include <linux/debugfs.h>
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -050023#include <linux/tracefs.h>
Steven Rostedt4c11d7a2008-05-12 21:20:43 +020024#include <linux/pagemap.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020025#include <linux/hardirq.h>
26#include <linux/linkage.h>
27#include <linux/uaccess.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050028#include <linux/kprobes.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020029#include <linux/ftrace.h>
30#include <linux/module.h>
31#include <linux/percpu.h>
Steven Rostedt2cadf912008-12-01 22:20:19 -050032#include <linux/splice.h>
Steven Rostedt3f5a54e2008-07-30 22:36:46 -040033#include <linux/kdebug.h>
Frederic Weisbecker5f0c6c02009-03-27 14:22:10 +010034#include <linux/string.h>
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -050035#include <linux/mount.h>
Lai Jiangshan7e53bd42010-01-06 20:08:50 +080036#include <linux/rwsem.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090037#include <linux/slab.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020038#include <linux/ctype.h>
39#include <linux/init.h>
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +020040#include <linux/poll.h>
Steven Rostedtb892e5c2012-03-01 22:06:48 -050041#include <linux/nmi.h>
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020042#include <linux/fs.h>
Clark Williams8bd75c72013-02-07 09:47:07 -060043#include <linux/sched/rt.h>
Ingo Molnar86387f72008-05-12 21:20:51 +020044
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020045#include "trace.h"
Steven Rostedtf0868d12008-12-23 23:24:12 -050046#include "trace_output.h"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +020047
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010048/*
Steven Rostedt73c51622009-03-11 13:42:01 -040049 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
51 */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -050052bool ring_buffer_expanded;
Steven Rostedt73c51622009-03-11 13:42:01 -040053
54/*
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010055 * We need to change this state when a selftest is running.
Frederic Weisbeckerff325042008-12-04 23:47:35 +010056 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
Ingo Molnar5e1607a2009-03-05 10:24:48 +010058 * insertions into the ring-buffer such as trace_printk could occurred
Frederic Weisbeckerff325042008-12-04 23:47:35 +010059 * at the same time, giving false positive or negative results.
60 */
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +010061static bool __read_mostly tracing_selftest_running;
Frederic Weisbeckerff325042008-12-04 23:47:35 +010062
Steven Rostedtb2821ae2009-02-02 21:38:32 -050063/*
64 * If a tracer is running, we do not want to run SELFTEST.
65 */
Li Zefan020e5f82009-07-01 10:47:05 +080066bool __read_mostly tracing_selftest_disabled;
Steven Rostedtb2821ae2009-02-02 21:38:32 -050067
Steven Rostedt (Red Hat)0daa23022014-12-12 22:27:10 -050068/* Pipe tracepoints to printk */
69struct trace_iterator *tracepoint_print_iter;
70int tracepoint_printk;
71
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010072/* For tracers that don't implement custom flags */
73static struct tracer_opt dummy_tracer_opt[] = {
74 { }
75};
76
77static struct tracer_flags dummy_tracer_flags = {
78 .val = 0,
79 .opts = dummy_tracer_opt
80};
81
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -050082static int
83dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +010084{
85 return 0;
86}
Steven Rostedt0f048702008-11-05 16:05:44 -050087
88/*
Steven Rostedt7ffbd482012-10-11 12:14:25 -040089 * To prevent the comm cache from being overwritten when no
90 * tracing is active, only save the comm when a trace event
91 * occurred.
92 */
93static DEFINE_PER_CPU(bool, trace_cmdline_save);
94
95/*
Steven Rostedt0f048702008-11-05 16:05:44 -050096 * Kill all tracing for good (never come back).
97 * It is initialized to 1 but will turn to zero if the initialization
98 * of the tracer is successful. But that is the only place that sets
99 * this back to zero.
100 */
Hannes Eder4fd27352009-02-10 19:44:12 +0100101static int tracing_disabled = 1;
Steven Rostedt0f048702008-11-05 16:05:44 -0500102
Christoph Lameter9288f992009-10-07 19:17:45 -0400103DEFINE_PER_CPU(int, ftrace_cpu_disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -0400104
Jason Wessel955b61e2010-08-05 09:22:23 -0500105cpumask_var_t __read_mostly tracing_buffer_mask;
Steven Rostedtab464282008-05-12 21:21:00 +0200106
Steven Rostedt944ac422008-10-23 19:26:08 -0400107/*
108 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
109 *
110 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
111 * is set, then ftrace_dump is called. This will output the contents
112 * of the ftrace buffers to the console. This is very useful for
113 * capturing traces that lead to crashes and outputing it to a
114 * serial console.
115 *
116 * It is default off, but you can enable it with either specifying
117 * "ftrace_dump_on_oops" in the kernel command line, or setting
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200118 * /proc/sys/kernel/ftrace_dump_on_oops
119 * Set 1 if you want to dump buffers of all CPUs
120 * Set 2 if you want to dump the buffer of the CPU that triggered oops
Steven Rostedt944ac422008-10-23 19:26:08 -0400121 */
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200122
123enum ftrace_dump_mode ftrace_dump_on_oops;
Steven Rostedt944ac422008-10-23 19:26:08 -0400124
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400125/* When set, tracing will stop when a WARN*() is hit */
126int __disable_trace_on_warning;
127
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -0400128#ifdef CONFIG_TRACE_ENUM_MAP_FILE
129/* Map of enums to their values, for "enum_map" file */
130struct trace_enum_map_head {
131 struct module *mod;
132 unsigned long length;
133};
134
135union trace_enum_map_item;
136
137struct trace_enum_map_tail {
138 /*
139 * "end" is first and points to NULL as it must be different
140 * than "mod" or "enum_string"
141 */
142 union trace_enum_map_item *next;
143 const char *end; /* points to NULL */
144};
145
146static DEFINE_MUTEX(trace_enum_mutex);
147
148/*
149 * The trace_enum_maps are saved in an array with two extra elements,
150 * one at the beginning, and one at the end. The beginning item contains
151 * the count of the saved maps (head.length), and the module they
152 * belong to if not built in (head.mod). The ending item contains a
153 * pointer to the next array of saved enum_map items.
154 */
155union trace_enum_map_item {
156 struct trace_enum_map map;
157 struct trace_enum_map_head head;
158 struct trace_enum_map_tail tail;
159};
160
161static union trace_enum_map_item *trace_enum_maps;
162#endif /* CONFIG_TRACE_ENUM_MAP_FILE */
163
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -0500164static int tracing_set_tracer(struct trace_array *tr, const char *buf);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500165
Li Zefanee6c2c12009-09-18 14:06:47 +0800166#define MAX_TRACER_SIZE 100
167static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500168static char *default_bootup_tracer;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100169
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500170static bool allocate_snapshot;
171
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200172static int __init set_cmdline_ftrace(char *str)
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100173{
Chen Gang67012ab2013-04-08 12:06:44 +0800174 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
Steven Rostedtb2821ae2009-02-02 21:38:32 -0500175 default_bootup_tracer = bootup_tracer_buf;
Steven Rostedt73c51622009-03-11 13:42:01 -0400176 /* We are using ftrace early, expand it */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500177 ring_buffer_expanded = true;
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100178 return 1;
179}
Frederic Weisbecker1beee962009-10-14 20:50:32 +0200180__setup("ftrace=", set_cmdline_ftrace);
Peter Zijlstrad9e54072008-11-01 19:57:37 +0100181
Steven Rostedt944ac422008-10-23 19:26:08 -0400182static int __init set_ftrace_dump_on_oops(char *str)
183{
Frederic Weisbeckercecbca92010-04-18 19:08:41 +0200184 if (*str++ != '=' || !*str) {
185 ftrace_dump_on_oops = DUMP_ALL;
186 return 1;
187 }
188
189 if (!strcmp("orig_cpu", str)) {
190 ftrace_dump_on_oops = DUMP_ORIG;
191 return 1;
192 }
193
194 return 0;
Steven Rostedt944ac422008-10-23 19:26:08 -0400195}
196__setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
Steven Rostedt60a11772008-05-12 21:20:44 +0200197
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400198static int __init stop_trace_on_warning(char *str)
199{
Luis Claudio R. Goncalves933ff9f2014-11-12 21:14:00 -0200200 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
201 __disable_trace_on_warning = 1;
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400202 return 1;
203}
Luis Claudio R. Goncalves933ff9f2014-11-12 21:14:00 -0200204__setup("traceoff_on_warning", stop_trace_on_warning);
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400205
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400206static int __init boot_alloc_snapshot(char *str)
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500207{
208 allocate_snapshot = true;
209 /* We also need the main ring buffer expanded */
210 ring_buffer_expanded = true;
211 return 1;
212}
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400213__setup("alloc_snapshot", boot_alloc_snapshot);
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -0500214
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400215
216static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
217static char *trace_boot_options __initdata;
218
219static int __init set_trace_boot_options(char *str)
220{
Chen Gang67012ab2013-04-08 12:06:44 +0800221 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
Steven Rostedt7bcfaf52012-11-01 22:56:07 -0400222 trace_boot_options = trace_boot_options_buf;
223 return 0;
224}
225__setup("trace_options=", set_trace_boot_options);
226
Steven Rostedte1e232c2014-02-10 23:38:46 -0500227static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
228static char *trace_boot_clock __initdata;
229
230static int __init set_trace_boot_clock(char *str)
231{
232 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
233 trace_boot_clock = trace_boot_clock_buf;
234 return 0;
235}
236__setup("trace_clock=", set_trace_boot_clock);
237
Steven Rostedt (Red Hat)0daa23022014-12-12 22:27:10 -0500238static int __init set_tracepoint_printk(char *str)
239{
240 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
241 tracepoint_printk = 1;
242 return 1;
243}
244__setup("tp_printk", set_tracepoint_printk);
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400245
Lai Jiangshancf8e3472009-03-30 13:48:00 +0800246unsigned long long ns2usecs(cycle_t nsec)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200247{
248 nsec += 500;
249 do_div(nsec, 1000);
250 return nsec;
251}
252
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400253/* trace_flags holds trace_options default values */
254#define TRACE_DEFAULT_FLAGS \
255 (FUNCTION_DEFAULT_FLAGS | \
256 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
257 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
258 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
259 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
260
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200261/*
262 * The global_trace is the descriptor that holds the tracing
263 * buffers for the live tracing. For each CPU, it contains
264 * a link list of pages that will store trace entries. The
265 * page descriptor of the pages in the memory is used to hold
266 * the link list by linking the lru item in the page descriptor
267 * to each of the pages in the buffer per CPU.
268 *
269 * For each active CPU there is a data field that holds the
270 * pages for the buffer for that CPU. Each CPU has the same number
271 * of pages allocated for its buffer.
272 */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400273static struct trace_array global_trace = {
274 .trace_flags = TRACE_DEFAULT_FLAGS,
275};
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200276
Steven Rostedtae63b312012-05-03 23:09:03 -0400277LIST_HEAD(ftrace_trace_arrays);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200278
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -0400279int trace_array_get(struct trace_array *this_tr)
280{
281 struct trace_array *tr;
282 int ret = -ENODEV;
283
284 mutex_lock(&trace_types_lock);
285 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
286 if (tr == this_tr) {
287 tr->ref++;
288 ret = 0;
289 break;
290 }
291 }
292 mutex_unlock(&trace_types_lock);
293
294 return ret;
295}
296
297static void __trace_array_put(struct trace_array *this_tr)
298{
299 WARN_ON(!this_tr->ref);
300 this_tr->ref--;
301}
302
303void trace_array_put(struct trace_array *this_tr)
304{
305 mutex_lock(&trace_types_lock);
306 __trace_array_put(this_tr);
307 mutex_unlock(&trace_types_lock);
308}
309
Steven Rostedt (Red Hat)7f1d2f82015-05-05 10:09:53 -0400310int filter_check_discard(struct trace_event_file *file, void *rec,
Tom Zanussif306cc82013-10-24 08:34:17 -0500311 struct ring_buffer *buffer,
312 struct ring_buffer_event *event)
Tom Zanussieb02ce02009-04-08 03:15:54 -0500313{
Steven Rostedt (Red Hat)5d6ad962015-05-13 15:12:33 -0400314 if (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
Tom Zanussif306cc82013-10-24 08:34:17 -0500315 !filter_match_preds(file->filter, rec)) {
316 ring_buffer_discard_commit(buffer, event);
317 return 1;
318 }
319
320 return 0;
Tom Zanussieb02ce02009-04-08 03:15:54 -0500321}
Tom Zanussif306cc82013-10-24 08:34:17 -0500322EXPORT_SYMBOL_GPL(filter_check_discard);
323
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -0400324int call_filter_check_discard(struct trace_event_call *call, void *rec,
Tom Zanussif306cc82013-10-24 08:34:17 -0500325 struct ring_buffer *buffer,
326 struct ring_buffer_event *event)
327{
328 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
329 !filter_match_preds(call->filter, rec)) {
330 ring_buffer_discard_commit(buffer, event);
331 return 1;
332 }
333
334 return 0;
335}
336EXPORT_SYMBOL_GPL(call_filter_check_discard);
Tom Zanussieb02ce02009-04-08 03:15:54 -0500337
Fabian Frederickad1438a2014-04-17 21:44:42 +0200338static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
Steven Rostedt37886f62009-03-17 17:22:06 -0400339{
340 u64 ts;
341
342 /* Early boot up does not have a buffer yet */
Alexander Z Lam94571582013-08-02 18:36:16 -0700343 if (!buf->buffer)
Steven Rostedt37886f62009-03-17 17:22:06 -0400344 return trace_clock_local();
345
Alexander Z Lam94571582013-08-02 18:36:16 -0700346 ts = ring_buffer_time_stamp(buf->buffer, cpu);
347 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
Steven Rostedt37886f62009-03-17 17:22:06 -0400348
349 return ts;
350}
351
Alexander Z Lam94571582013-08-02 18:36:16 -0700352cycle_t ftrace_now(int cpu)
353{
354 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
355}
356
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400357/**
358 * tracing_is_enabled - Show if global_trace has been disabled
359 *
360 * Shows if the global trace has been enabled or not. It uses the
361 * mirror flag "buffer_disabled" to be used in fast paths such as for
362 * the irqsoff tracer. But it may be inaccurate due to races. If you
363 * need to know the accurate state, use tracing_is_on() which is a little
364 * slower, but accurate.
365 */
Steven Rostedt90369902008-11-05 16:05:44 -0500366int tracing_is_enabled(void)
367{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400368 /*
369 * For quick access (irqsoff uses this in fast path), just
370 * return the mirror variable of the state of the ring buffer.
371 * It's a little racy, but we don't really care.
372 */
373 smp_rmb();
374 return !global_trace.buffer_disabled;
Steven Rostedt90369902008-11-05 16:05:44 -0500375}
376
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200377/*
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400378 * trace_buf_size is the size in bytes that is allocated
379 * for a buffer. Note, the number of bytes is always rounded
380 * to page size.
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400381 *
382 * This number is purposely set to a low number of 16384.
383 * If the dump on oops happens, it will be much appreciated
384 * to not have to wait for all that output. Anyway this can be
385 * boot time and run time configurable.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200386 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400387#define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
Steven Rostedt3f5a54e2008-07-30 22:36:46 -0400388
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400389static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200390
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200391/* trace_types holds a link list of available tracers. */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200392static struct tracer *trace_types __read_mostly;
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200393
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200394/*
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200395 * trace_types_lock is used to protect the trace_types list.
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200396 */
Alexander Z Lama8227412013-07-01 19:37:54 -0700397DEFINE_MUTEX(trace_types_lock);
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200398
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800399/*
400 * serialize the access of the ring buffer
401 *
402 * ring buffer serializes readers, but it is low level protection.
403 * The validity of the events (which returns by ring_buffer_peek() ..etc)
404 * are not protected by ring buffer.
405 *
406 * The content of events may become garbage if we allow other process consumes
407 * these events concurrently:
408 * A) the page of the consumed events may become a normal page
409 * (not reader page) in ring buffer, and this page will be rewrited
410 * by events producer.
411 * B) The page of the consumed events may become a page for splice_read,
412 * and this page will be returned to system.
413 *
414 * These primitives allow multi process access to different cpu ring buffer
415 * concurrently.
416 *
417 * These primitives don't distinguish read-only and read-consume access.
418 * Multi read-only access are also serialized.
419 */
420
421#ifdef CONFIG_SMP
422static DECLARE_RWSEM(all_cpu_access_lock);
423static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
424
425static inline void trace_access_lock(int cpu)
426{
Steven Rostedtae3b5092013-01-23 15:22:59 -0500427 if (cpu == RING_BUFFER_ALL_CPUS) {
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800428 /* gain it for accessing the whole ring buffer. */
429 down_write(&all_cpu_access_lock);
430 } else {
431 /* gain it for accessing a cpu ring buffer. */
432
Steven Rostedtae3b5092013-01-23 15:22:59 -0500433 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800434 down_read(&all_cpu_access_lock);
435
436 /* Secondly block other access to this @cpu ring buffer. */
437 mutex_lock(&per_cpu(cpu_access_lock, cpu));
438 }
439}
440
441static inline void trace_access_unlock(int cpu)
442{
Steven Rostedtae3b5092013-01-23 15:22:59 -0500443 if (cpu == RING_BUFFER_ALL_CPUS) {
Lai Jiangshan7e53bd42010-01-06 20:08:50 +0800444 up_write(&all_cpu_access_lock);
445 } else {
446 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
447 up_read(&all_cpu_access_lock);
448 }
449}
450
451static inline void trace_access_lock_init(void)
452{
453 int cpu;
454
455 for_each_possible_cpu(cpu)
456 mutex_init(&per_cpu(cpu_access_lock, cpu));
457}
458
459#else
460
461static DEFINE_MUTEX(access_lock);
462
463static inline void trace_access_lock(int cpu)
464{
465 (void)cpu;
466 mutex_lock(&access_lock);
467}
468
469static inline void trace_access_unlock(int cpu)
470{
471 (void)cpu;
472 mutex_unlock(&access_lock);
473}
474
475static inline void trace_access_lock_init(void)
476{
477}
478
479#endif
480
Steven Rostedt (Red Hat)d78a4612015-09-25 13:30:47 -0400481#ifdef CONFIG_STACKTRACE
482static void __ftrace_trace_stack(struct ring_buffer *buffer,
483 unsigned long flags,
484 int skip, int pc, struct pt_regs *regs);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400485static inline void ftrace_trace_stack(struct trace_array *tr,
486 struct ring_buffer *buffer,
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -0400487 unsigned long flags,
488 int skip, int pc, struct pt_regs *regs);
Steven Rostedt (Red Hat)ca475e82015-09-28 09:41:11 -0400489
Steven Rostedt (Red Hat)d78a4612015-09-25 13:30:47 -0400490#else
491static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
492 unsigned long flags,
493 int skip, int pc, struct pt_regs *regs)
494{
495}
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400496static inline void ftrace_trace_stack(struct trace_array *tr,
497 struct ring_buffer *buffer,
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -0400498 unsigned long flags,
499 int skip, int pc, struct pt_regs *regs)
Steven Rostedt (Red Hat)ca475e82015-09-28 09:41:11 -0400500{
501}
502
Steven Rostedt (Red Hat)d78a4612015-09-25 13:30:47 -0400503#endif
504
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400505static void tracer_tracing_on(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400506{
507 if (tr->trace_buffer.buffer)
508 ring_buffer_record_on(tr->trace_buffer.buffer);
509 /*
510 * This flag is looked at when buffers haven't been allocated
511 * yet, or by some tracers (like irqsoff), that just want to
512 * know if the ring buffer has been disabled, but it can handle
513 * races of where it gets disabled but we still do a record.
514 * As the check is in the fast path of the tracers, it is more
515 * important to be fast than accurate.
516 */
517 tr->buffer_disabled = 0;
518 /* Make the flag seen by readers */
519 smp_wmb();
520}
521
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200522/**
Steven Rostedt499e5472012-02-22 15:50:28 -0500523 * tracing_on - enable tracing buffers
524 *
525 * This function enables tracing buffers that may have been
526 * disabled with tracing_off.
527 */
528void tracing_on(void)
529{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400530 tracer_tracing_on(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500531}
532EXPORT_SYMBOL_GPL(tracing_on);
533
534/**
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500535 * __trace_puts - write a constant string into the trace buffer.
536 * @ip: The address of the caller
537 * @str: The constant string to write
538 * @size: The size of the string.
539 */
540int __trace_puts(unsigned long ip, const char *str, int size)
541{
542 struct ring_buffer_event *event;
543 struct ring_buffer *buffer;
544 struct print_entry *entry;
545 unsigned long irq_flags;
546 int alloc;
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800547 int pc;
548
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400549 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
zhangwei(Jovi)f0160a52013-07-18 16:31:18 +0800550 return 0;
551
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800552 pc = preempt_count();
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500553
Steven Rostedt (Red Hat)3132e102014-01-23 12:27:59 -0500554 if (unlikely(tracing_selftest_running || tracing_disabled))
555 return 0;
556
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500557 alloc = sizeof(*entry) + size + 2; /* possible \n added */
558
559 local_save_flags(irq_flags);
560 buffer = global_trace.trace_buffer.buffer;
561 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800562 irq_flags, pc);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500563 if (!event)
564 return 0;
565
566 entry = ring_buffer_event_data(event);
567 entry->ip = ip;
568
569 memcpy(&entry->buf, str, size);
570
571 /* Add a newline if necessary */
572 if (entry->buf[size - 1] != '\n') {
573 entry->buf[size] = '\n';
574 entry->buf[size + 1] = '\0';
575 } else
576 entry->buf[size] = '\0';
577
578 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400579 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500580
581 return size;
582}
583EXPORT_SYMBOL_GPL(__trace_puts);
584
585/**
586 * __trace_bputs - write the pointer to a constant string into trace buffer
587 * @ip: The address of the caller
588 * @str: The constant string to write to the buffer to
589 */
590int __trace_bputs(unsigned long ip, const char *str)
591{
592 struct ring_buffer_event *event;
593 struct ring_buffer *buffer;
594 struct bputs_entry *entry;
595 unsigned long irq_flags;
596 int size = sizeof(struct bputs_entry);
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800597 int pc;
598
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -0400599 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
zhangwei(Jovi)f0160a52013-07-18 16:31:18 +0800600 return 0;
601
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800602 pc = preempt_count();
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500603
Steven Rostedt (Red Hat)3132e102014-01-23 12:27:59 -0500604 if (unlikely(tracing_selftest_running || tracing_disabled))
605 return 0;
606
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500607 local_save_flags(irq_flags);
608 buffer = global_trace.trace_buffer.buffer;
609 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
zhangwei(Jovi)8abfb872013-07-18 16:31:05 +0800610 irq_flags, pc);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500611 if (!event)
612 return 0;
613
614 entry = ring_buffer_event_data(event);
615 entry->ip = ip;
616 entry->str = str;
617
618 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -0400619 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -0500620
621 return 1;
622}
623EXPORT_SYMBOL_GPL(__trace_bputs);
624
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500625#ifdef CONFIG_TRACER_SNAPSHOT
626/**
627 * trace_snapshot - take a snapshot of the current buffer.
628 *
629 * This causes a swap between the snapshot buffer and the current live
630 * tracing buffer. You can use this to take snapshots of the live
631 * trace when some condition is triggered, but continue to trace.
632 *
633 * Note, make sure to allocate the snapshot with either
634 * a tracing_snapshot_alloc(), or by doing it manually
635 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
636 *
637 * If the snapshot buffer is not allocated, it will stop tracing.
638 * Basically making a permanent snapshot.
639 */
640void tracing_snapshot(void)
641{
642 struct trace_array *tr = &global_trace;
643 struct tracer *tracer = tr->current_trace;
644 unsigned long flags;
645
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500646 if (in_nmi()) {
647 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
648 internal_trace_puts("*** snapshot is being ignored ***\n");
649 return;
650 }
651
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500652 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)ca268da2013-03-09 00:40:58 -0500653 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
654 internal_trace_puts("*** stopping trace here! ***\n");
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500655 tracing_off();
656 return;
657 }
658
659 /* Note, snapshot can not be used when the tracer uses it */
660 if (tracer->use_max_tr) {
Steven Rostedt (Red Hat)ca268da2013-03-09 00:40:58 -0500661 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
662 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500663 return;
664 }
665
666 local_irq_save(flags);
667 update_max_tr(tr, current, smp_processor_id());
668 local_irq_restore(flags);
669}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500670EXPORT_SYMBOL_GPL(tracing_snapshot);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500671
672static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
673 struct trace_buffer *size_buf, int cpu_id);
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400674static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
675
676static int alloc_snapshot(struct trace_array *tr)
677{
678 int ret;
679
680 if (!tr->allocated_snapshot) {
681
682 /* allocate spare buffer */
683 ret = resize_buffer_duplicate_size(&tr->max_buffer,
684 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
685 if (ret < 0)
686 return ret;
687
688 tr->allocated_snapshot = true;
689 }
690
691 return 0;
692}
693
Fabian Frederickad1438a2014-04-17 21:44:42 +0200694static void free_snapshot(struct trace_array *tr)
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400695{
696 /*
697 * We don't free the ring buffer. instead, resize it because
698 * The max_tr ring buffer has some state (e.g. ring->clock) and
699 * we want preserve it.
700 */
701 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
702 set_buffer_entries(&tr->max_buffer, 1);
703 tracing_reset_online_cpus(&tr->max_buffer);
704 tr->allocated_snapshot = false;
705}
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500706
707/**
Tom Zanussi93e31ff2013-10-24 08:59:26 -0500708 * tracing_alloc_snapshot - allocate snapshot buffer.
709 *
710 * This only allocates the snapshot buffer if it isn't already
711 * allocated - it doesn't also take a snapshot.
712 *
713 * This is meant to be used in cases where the snapshot buffer needs
714 * to be set up for events that can't sleep but need to be able to
715 * trigger a snapshot.
716 */
717int tracing_alloc_snapshot(void)
718{
719 struct trace_array *tr = &global_trace;
720 int ret;
721
722 ret = alloc_snapshot(tr);
723 WARN_ON(ret < 0);
724
725 return ret;
726}
727EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
728
729/**
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500730 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
731 *
732 * This is similar to trace_snapshot(), but it will allocate the
733 * snapshot buffer if it isn't already allocated. Use this only
734 * where it is safe to sleep, as the allocation may sleep.
735 *
736 * This causes a swap between the snapshot buffer and the current live
737 * tracing buffer. You can use this to take snapshots of the live
738 * trace when some condition is triggered, but continue to trace.
739 */
740void tracing_snapshot_alloc(void)
741{
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500742 int ret;
743
Tom Zanussi93e31ff2013-10-24 08:59:26 -0500744 ret = tracing_alloc_snapshot();
745 if (ret < 0)
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -0400746 return;
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500747
748 tracing_snapshot();
749}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500750EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500751#else
752void tracing_snapshot(void)
753{
754 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
755}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500756EXPORT_SYMBOL_GPL(tracing_snapshot);
Tom Zanussi93e31ff2013-10-24 08:59:26 -0500757int tracing_alloc_snapshot(void)
758{
759 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
760 return -ENODEV;
761}
762EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500763void tracing_snapshot_alloc(void)
764{
765 /* Give warning */
766 tracing_snapshot();
767}
Steven Rostedt (Red Hat)1b22e382013-03-09 00:56:08 -0500768EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
Steven Rostedt (Red Hat)ad909e22013-03-06 21:45:37 -0500769#endif /* CONFIG_TRACER_SNAPSHOT */
770
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400771static void tracer_tracing_off(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400772{
773 if (tr->trace_buffer.buffer)
774 ring_buffer_record_off(tr->trace_buffer.buffer);
775 /*
776 * This flag is looked at when buffers haven't been allocated
777 * yet, or by some tracers (like irqsoff), that just want to
778 * know if the ring buffer has been disabled, but it can handle
779 * races of where it gets disabled but we still do a record.
780 * As the check is in the fast path of the tracers, it is more
781 * important to be fast than accurate.
782 */
783 tr->buffer_disabled = 1;
784 /* Make the flag seen by readers */
785 smp_wmb();
786}
787
Steven Rostedt499e5472012-02-22 15:50:28 -0500788/**
789 * tracing_off - turn off tracing buffers
790 *
791 * This function stops the tracing buffers from recording data.
792 * It does not disable any overhead the tracers themselves may
793 * be causing. This function simply causes all recording to
794 * the ring buffers to fail.
795 */
796void tracing_off(void)
797{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400798 tracer_tracing_off(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500799}
800EXPORT_SYMBOL_GPL(tracing_off);
801
Steven Rostedt (Red Hat)de7edd32013-06-14 16:21:43 -0400802void disable_trace_on_warning(void)
803{
804 if (__disable_trace_on_warning)
805 tracing_off();
806}
807
Steven Rostedt499e5472012-02-22 15:50:28 -0500808/**
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400809 * tracer_tracing_is_on - show real state of ring buffer enabled
810 * @tr : the trace array to know if ring buffer is enabled
811 *
812 * Shows real state of the ring buffer if it is enabled or not.
813 */
Steven Rostedt (Red Hat)5280bce2013-07-02 19:59:57 -0400814static int tracer_tracing_is_on(struct trace_array *tr)
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400815{
816 if (tr->trace_buffer.buffer)
817 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
818 return !tr->buffer_disabled;
819}
820
Steven Rostedt499e5472012-02-22 15:50:28 -0500821/**
822 * tracing_is_on - show state of ring buffers enabled
823 */
824int tracing_is_on(void)
825{
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -0400826 return tracer_tracing_is_on(&global_trace);
Steven Rostedt499e5472012-02-22 15:50:28 -0500827}
828EXPORT_SYMBOL_GPL(tracing_is_on);
829
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400830static int __init set_buf_size(char *str)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200831{
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400832 unsigned long buf_size;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200833
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200834 if (!str)
835 return 0;
Li Zefan9d612be2009-06-24 17:33:15 +0800836 buf_size = memparse(str, &str);
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200837 /* nr_entries can not be zero */
Li Zefan9d612be2009-06-24 17:33:15 +0800838 if (buf_size == 0)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +0200839 return 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400840 trace_buf_size = buf_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200841 return 1;
842}
Steven Rostedt3928a8a2008-09-29 23:02:41 -0400843__setup("trace_buf_size=", set_buf_size);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200844
Tim Bird0e950172010-02-25 15:36:43 -0800845static int __init set_tracing_thresh(char *str)
846{
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800847 unsigned long threshold;
Tim Bird0e950172010-02-25 15:36:43 -0800848 int ret;
849
850 if (!str)
851 return 0;
Daniel Walterbcd83ea2012-09-26 22:08:38 +0200852 ret = kstrtoul(str, 0, &threshold);
Tim Bird0e950172010-02-25 15:36:43 -0800853 if (ret < 0)
854 return 0;
Wang Tianhong87abb3b2012-08-02 14:02:00 +0800855 tracing_thresh = threshold * 1000;
Tim Bird0e950172010-02-25 15:36:43 -0800856 return 1;
857}
858__setup("tracing_thresh=", set_tracing_thresh);
859
Steven Rostedt57f50be2008-05-12 21:20:44 +0200860unsigned long nsecs_to_usecs(unsigned long nsecs)
861{
862 return nsecs / 1000;
863}
864
Steven Rostedt (Red Hat)a3418a32015-09-29 09:43:30 -0400865/*
866 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
867 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
868 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
869 * of strings in the order that the enums were defined.
870 */
871#undef C
872#define C(a, b) b
873
Steven Rostedt4fcdae82008-05-12 21:21:00 +0200874/* These must match the bit postions in trace_iterator_flags */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200875static const char *trace_options[] = {
Steven Rostedt (Red Hat)a3418a32015-09-29 09:43:30 -0400876 TRACE_FLAGS
Steven Rostedtbc0c38d2008-05-12 21:20:42 +0200877 NULL
878};
879
Zhaolei5079f322009-08-25 16:12:56 +0800880static struct {
881 u64 (*func)(void);
882 const char *name;
David Sharp8be07092012-11-13 12:18:22 -0800883 int in_ns; /* is this clock in nanoseconds? */
Zhaolei5079f322009-08-25 16:12:56 +0800884} trace_clocks[] = {
Thomas Gleixner1b3e5c02014-07-16 21:05:25 +0000885 { trace_clock_local, "local", 1 },
886 { trace_clock_global, "global", 1 },
887 { trace_clock_counter, "counter", 0 },
Linus Torvaldse7fda6c2014-08-05 17:46:42 -0700888 { trace_clock_jiffies, "uptime", 0 },
Thomas Gleixner1b3e5c02014-07-16 21:05:25 +0000889 { trace_clock, "perf", 1 },
890 { ktime_get_mono_fast_ns, "mono", 1 },
Drew Richardsonaabfa5f2015-05-08 07:30:39 -0700891 { ktime_get_raw_fast_ns, "mono_raw", 1 },
David Sharp8cbd9cc2012-11-13 12:18:21 -0800892 ARCH_TRACE_CLOCKS
Zhaolei5079f322009-08-25 16:12:56 +0800893};
894
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200895/*
896 * trace_parser_get_init - gets the buffer for trace parser
897 */
898int trace_parser_get_init(struct trace_parser *parser, int size)
899{
900 memset(parser, 0, sizeof(*parser));
901
902 parser->buffer = kmalloc(size, GFP_KERNEL);
903 if (!parser->buffer)
904 return 1;
905
906 parser->size = size;
907 return 0;
908}
909
910/*
911 * trace_parser_put - frees the buffer for trace parser
912 */
913void trace_parser_put(struct trace_parser *parser)
914{
915 kfree(parser->buffer);
916}
917
918/*
919 * trace_get_user - reads the user input string separated by space
920 * (matched by isspace(ch))
921 *
922 * For each string found the 'struct trace_parser' is updated,
923 * and the function returns.
924 *
925 * Returns number of bytes read.
926 *
927 * See kernel/trace/trace.h for 'struct trace_parser' details.
928 */
929int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
930 size_t cnt, loff_t *ppos)
931{
932 char ch;
933 size_t read = 0;
934 ssize_t ret;
935
936 if (!*ppos)
937 trace_parser_clear(parser);
938
939 ret = get_user(ch, ubuf++);
940 if (ret)
941 goto out;
942
943 read++;
944 cnt--;
945
946 /*
947 * The parser is not finished with the last write,
948 * continue reading the user input without skipping spaces.
949 */
950 if (!parser->cont) {
951 /* skip white space */
952 while (cnt && isspace(ch)) {
953 ret = get_user(ch, ubuf++);
954 if (ret)
955 goto out;
956 read++;
957 cnt--;
958 }
959
960 /* only spaces were written */
961 if (isspace(ch)) {
962 *ppos += read;
963 ret = read;
964 goto out;
965 }
966
967 parser->idx = 0;
968 }
969
970 /* read the non-space input */
971 while (cnt && !isspace(ch)) {
Li Zefan3c235a32009-09-22 13:51:54 +0800972 if (parser->idx < parser->size - 1)
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200973 parser->buffer[parser->idx++] = ch;
974 else {
975 ret = -EINVAL;
976 goto out;
977 }
978 ret = get_user(ch, ubuf++);
979 if (ret)
980 goto out;
981 read++;
982 cnt--;
983 }
984
985 /* We either got finished input or we have to wait for another call. */
986 if (isspace(ch)) {
987 parser->buffer[parser->idx] = 0;
988 parser->cont = false;
Steven Rostedt057db842013-10-09 22:23:23 -0400989 } else if (parser->idx < parser->size - 1) {
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200990 parser->cont = true;
991 parser->buffer[parser->idx++] = ch;
Steven Rostedt057db842013-10-09 22:23:23 -0400992 } else {
993 ret = -EINVAL;
994 goto out;
jolsa@redhat.comb63f39e2009-09-11 17:29:27 +0200995 }
996
997 *ppos += read;
998 ret = read;
999
1000out:
1001 return ret;
1002}
1003
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04001004/* TODO add a seq_buf_to_buffer() */
Dmitri Vorobievb8b94262009-03-22 19:11:11 +02001005static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001006{
1007 int len;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001008
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05001009 if (trace_seq_used(s) <= s->seq.readpos)
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001010 return -EBUSY;
1011
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05001012 len = trace_seq_used(s) - s->seq.readpos;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001013 if (cnt > len)
1014 cnt = len;
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04001015 memcpy(buf, s->buffer + s->seq.readpos, cnt);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001016
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04001017 s->seq.readpos += cnt;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02001018 return cnt;
1019}
1020
Tim Bird0e950172010-02-25 15:36:43 -08001021unsigned long __read_mostly tracing_thresh;
1022
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001023#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001024/*
1025 * Copy the new maximum trace into the separate maximum-trace
1026 * structure. (this way the maximum trace is permanently saved,
1027 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1028 */
1029static void
1030__update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1031{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001032 struct trace_buffer *trace_buf = &tr->trace_buffer;
1033 struct trace_buffer *max_buf = &tr->max_buffer;
1034 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1035 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001036
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001037 max_buf->cpu = cpu;
1038 max_buf->time_start = data->preempt_timestamp;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001039
Steven Rostedt (Red Hat)6d9b3fa2014-01-14 11:28:38 -05001040 max_data->saved_latency = tr->max_latency;
Steven Rostedt8248ac02009-09-02 12:27:41 -04001041 max_data->critical_start = data->critical_start;
1042 max_data->critical_end = data->critical_end;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001043
Arnaldo Carvalho de Melo1acaa1b2010-03-05 18:23:50 -03001044 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
Steven Rostedt8248ac02009-09-02 12:27:41 -04001045 max_data->pid = tsk->pid;
Steven Rostedt (Red Hat)f17a5192013-05-30 21:10:37 -04001046 /*
1047 * If tsk == current, then use current_uid(), as that does not use
1048 * RCU. The irq tracer can be called out of RCU scope.
1049 */
1050 if (tsk == current)
1051 max_data->uid = current_uid();
1052 else
1053 max_data->uid = task_uid(tsk);
1054
Steven Rostedt8248ac02009-09-02 12:27:41 -04001055 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1056 max_data->policy = tsk->policy;
1057 max_data->rt_priority = tsk->rt_priority;
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001058
1059 /* record this tasks comm */
1060 tracing_record_cmdline(tsk);
1061}
1062
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001063/**
1064 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1065 * @tr: tracer
1066 * @tsk: the task with the latency
1067 * @cpu: The cpu that initiated the trace.
1068 *
1069 * Flip the buffers between the @tr and the max_tr and record information
1070 * about which task was the cause of this latency.
1071 */
Ingo Molnare309b412008-05-12 21:20:51 +02001072void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001073update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1074{
Steven Rostedt (Red Hat)2721e722013-03-12 11:32:32 -04001075 struct ring_buffer *buf;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001076
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001077 if (tr->stop_count)
Steven Rostedtb8de7bd12009-08-31 22:32:27 -04001078 return;
1079
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02001080 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt34600f02013-01-22 13:35:11 -05001081
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05001082 if (!tr->allocated_snapshot) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09001083 /* Only the nop tracer should hit this when disabling */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001084 WARN_ON_ONCE(tr->current_trace != &nop_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001085 return;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09001086 }
Steven Rostedt34600f02013-01-22 13:35:11 -05001087
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001088 arch_spin_lock(&tr->max_lock);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001089
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001090 buf = tr->trace_buffer.buffer;
1091 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1092 tr->max_buffer.buffer = buf;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001093
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001094 __update_max_tr(tr, tsk, cpu);
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001095 arch_spin_unlock(&tr->max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001096}
1097
1098/**
1099 * update_max_tr_single - only copy one trace over, and reset the rest
1100 * @tr - tracer
1101 * @tsk - task with the latency
1102 * @cpu - the cpu of the buffer to copy.
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001103 *
1104 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001105 */
Ingo Molnare309b412008-05-12 21:20:51 +02001106void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001107update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1108{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001109 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001110
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001111 if (tr->stop_count)
Steven Rostedtb8de7bd12009-08-31 22:32:27 -04001112 return;
1113
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02001114 WARN_ON_ONCE(!irqs_disabled());
Steven Rostedt6c244992013-04-29 20:08:14 -04001115 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)2930e042013-03-26 17:33:00 -04001116 /* Only the nop tracer should hit this when disabling */
Linus Torvalds9e8529a2013-04-29 13:55:38 -07001117 WARN_ON_ONCE(tr->current_trace != &nop_trace);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001118 return;
Steven Rostedt (Red Hat)2930e042013-03-26 17:33:00 -04001119 }
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09001120
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001121 arch_spin_lock(&tr->max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001122
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001123 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001124
Steven Rostedte8165db2009-09-03 19:13:05 -04001125 if (ret == -EBUSY) {
1126 /*
1127 * We failed to swap the buffer due to a commit taking
1128 * place on this CPU. We fail to record, but we reset
1129 * the max trace buffer (no one writes directly to it)
1130 * and flag that it failed.
1131 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001132 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
Steven Rostedte8165db2009-09-03 19:13:05 -04001133 "Failed to swap buffers due to commit in progress\n");
1134 }
1135
Steven Rostedte8165db2009-09-03 19:13:05 -04001136 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001137
1138 __update_max_tr(tr, tsk, cpu);
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001139 arch_spin_unlock(&tr->max_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001140}
Steven Rostedt5d4a9db2009-08-27 16:52:21 -04001141#endif /* CONFIG_TRACER_MAX_TRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001142
Rabin Vincente30f53a2014-11-10 19:46:34 +01001143static int wait_on_pipe(struct trace_iterator *iter, bool full)
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001144{
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05001145 /* Iterators are static, they should be filled or empty */
1146 if (trace_buffer_iter(iter, iter->cpu_file))
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04001147 return 0;
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001148
Rabin Vincente30f53a2014-11-10 19:46:34 +01001149 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1150 full);
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001151}
1152
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001153#ifdef CONFIG_FTRACE_STARTUP_TEST
1154static int run_tracer_selftest(struct tracer *type)
1155{
1156 struct trace_array *tr = &global_trace;
1157 struct tracer *saved_tracer = tr->current_trace;
1158 int ret;
1159
1160 if (!type->selftest || tracing_selftest_disabled)
1161 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001162
1163 /*
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001164 * Run a selftest on this tracer.
1165 * Here we reset the trace buffer, and set the current
1166 * tracer to be this tracer. The tracer can then run some
1167 * internal tracing to verify that everything is in order.
1168 * If we fail, we do not register this tracer.
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001169 */
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001170 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001171
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001172 tr->current_trace = type;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001173
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001174#ifdef CONFIG_TRACER_MAX_TRACE
1175 if (type->use_max_tr) {
1176 /* If we expanded the buffers, make sure the max is expanded too */
1177 if (ring_buffer_expanded)
1178 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1179 RING_BUFFER_ALL_CPUS);
1180 tr->allocated_snapshot = true;
1181 }
1182#endif
1183
1184 /* the test is responsible for initializing and enabling */
1185 pr_info("Testing tracer %s: ", type->name);
1186 ret = type->selftest(type, tr);
1187 /* the test is responsible for resetting too */
1188 tr->current_trace = saved_tracer;
1189 if (ret) {
1190 printk(KERN_CONT "FAILED!\n");
1191 /* Add the warning after printing 'FAILED' */
1192 WARN_ON(1);
1193 return -1;
1194 }
1195 /* Only reset on passing, to avoid touching corrupted buffers */
1196 tracing_reset_online_cpus(&tr->trace_buffer);
1197
1198#ifdef CONFIG_TRACER_MAX_TRACE
1199 if (type->use_max_tr) {
1200 tr->allocated_snapshot = false;
1201
1202 /* Shrink the max buffer again */
1203 if (ring_buffer_expanded)
1204 ring_buffer_resize(tr->max_buffer.buffer, 1,
1205 RING_BUFFER_ALL_CPUS);
1206 }
1207#endif
1208
1209 printk(KERN_CONT "PASSED\n");
1210 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001211}
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001212#else
1213static inline int run_tracer_selftest(struct tracer *type)
1214{
1215 return 0;
1216}
1217#endif /* CONFIG_FTRACE_STARTUP_TEST */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001218
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04001219static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1220
Steven Rostedt4fcdae82008-05-12 21:21:00 +02001221/**
1222 * register_tracer - register a tracer with the ftrace system.
1223 * @type - the plugin for the tracer
1224 *
1225 * Register a new plugin tracer.
1226 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001227int register_tracer(struct tracer *type)
1228{
1229 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001230 int ret = 0;
1231
1232 if (!type->name) {
1233 pr_info("Tracer must have a name\n");
1234 return -1;
1235 }
1236
Dan Carpenter24a461d2010-07-10 12:06:44 +02001237 if (strlen(type->name) >= MAX_TRACER_SIZE) {
Li Zefanee6c2c12009-09-18 14:06:47 +08001238 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1239 return -1;
1240 }
1241
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001242 mutex_lock(&trace_types_lock);
Ingo Molnar86fa2f62008-11-19 10:00:15 +01001243
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +01001244 tracing_selftest_running = true;
1245
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001246 for (t = trace_types; t; t = t->next) {
1247 if (strcmp(type->name, t->name) == 0) {
1248 /* already found */
Li Zefanee6c2c12009-09-18 14:06:47 +08001249 pr_info("Tracer %s already registered\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001250 type->name);
1251 ret = -1;
1252 goto out;
1253 }
1254 }
1255
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01001256 if (!type->set_flag)
1257 type->set_flag = &dummy_set_flag;
1258 if (!type->flags)
1259 type->flags = &dummy_tracer_flags;
1260 else
1261 if (!type->flags->opts)
1262 type->flags->opts = dummy_tracer_opt;
Frederic Weisbecker6eaaa5d2009-02-11 02:25:00 +01001263
Steven Rostedt (Red Hat)f4e781c2013-03-07 11:10:56 -05001264 ret = run_tracer_selftest(type);
1265 if (ret < 0)
1266 goto out;
Steven Rostedt60a11772008-05-12 21:20:44 +02001267
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001268 type->next = trace_types;
1269 trace_types = type;
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04001270 add_tracer_options(&global_trace, type);
Steven Rostedt60a11772008-05-12 21:20:44 +02001271
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001272 out:
Frederic Weisbecker8e1b82e2008-12-06 03:41:33 +01001273 tracing_selftest_running = false;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001274 mutex_unlock(&trace_types_lock);
1275
Steven Rostedtdac74942009-02-05 01:13:38 -05001276 if (ret || !default_bootup_tracer)
1277 goto out_unlock;
Steven Rostedtb2821ae2009-02-02 21:38:32 -05001278
Li Zefanee6c2c12009-09-18 14:06:47 +08001279 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
Steven Rostedtdac74942009-02-05 01:13:38 -05001280 goto out_unlock;
1281
1282 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1283 /* Do we want this tracer to start on bootup? */
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05001284 tracing_set_tracer(&global_trace, type->name);
Steven Rostedtdac74942009-02-05 01:13:38 -05001285 default_bootup_tracer = NULL;
1286 /* disable other selftests, since this will break it. */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05001287 tracing_selftest_disabled = true;
Steven Rostedtdac74942009-02-05 01:13:38 -05001288#ifdef CONFIG_FTRACE_STARTUP_TEST
1289 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1290 type->name);
1291#endif
1292
1293 out_unlock:
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001294 return ret;
1295}
1296
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001297void tracing_reset(struct trace_buffer *buf, int cpu)
Steven Rostedtf6339032009-09-04 12:35:16 -04001298{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001299 struct ring_buffer *buffer = buf->buffer;
Steven Rostedtf6339032009-09-04 12:35:16 -04001300
Hiraku Toyookaa5416412012-12-19 16:02:34 +09001301 if (!buffer)
1302 return;
1303
Steven Rostedtf6339032009-09-04 12:35:16 -04001304 ring_buffer_record_disable(buffer);
1305
1306 /* Make sure all commits have finished */
1307 synchronize_sched();
Steven Rostedt68179682012-05-08 20:57:53 -04001308 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedtf6339032009-09-04 12:35:16 -04001309
1310 ring_buffer_record_enable(buffer);
1311}
1312
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001313void tracing_reset_online_cpus(struct trace_buffer *buf)
Pekka J Enberg213cc062008-12-19 12:08:39 +02001314{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001315 struct ring_buffer *buffer = buf->buffer;
Pekka J Enberg213cc062008-12-19 12:08:39 +02001316 int cpu;
1317
Hiraku Toyookaa5416412012-12-19 16:02:34 +09001318 if (!buffer)
1319 return;
1320
Steven Rostedt621968c2009-09-04 12:02:35 -04001321 ring_buffer_record_disable(buffer);
1322
1323 /* Make sure all commits have finished */
1324 synchronize_sched();
1325
Alexander Z Lam94571582013-08-02 18:36:16 -07001326 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
Pekka J Enberg213cc062008-12-19 12:08:39 +02001327
1328 for_each_online_cpu(cpu)
Steven Rostedt68179682012-05-08 20:57:53 -04001329 ring_buffer_reset_cpu(buffer, cpu);
Steven Rostedt621968c2009-09-04 12:02:35 -04001330
1331 ring_buffer_record_enable(buffer);
Pekka J Enberg213cc062008-12-19 12:08:39 +02001332}
1333
Steven Rostedt (Red Hat)09d80912013-07-23 22:21:59 -04001334/* Must have trace_types_lock held */
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001335void tracing_reset_all_online_cpus(void)
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001336{
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001337 struct trace_array *tr;
1338
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001339 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001340 tracing_reset_online_cpus(&tr->trace_buffer);
1341#ifdef CONFIG_TRACER_MAX_TRACE
1342 tracing_reset_online_cpus(&tr->max_buffer);
1343#endif
Steven Rostedt (Red Hat)873c6422013-03-04 23:26:06 -05001344 }
Steven Rostedt9456f0f2009-05-06 21:54:09 -04001345}
1346
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001347#define SAVED_CMDLINES_DEFAULT 128
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001348#define NO_CMDLINE_MAP UINT_MAX
Thomas Gleixneredc35bd2009-12-03 12:38:57 +01001349static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001350struct saved_cmdlines_buffer {
1351 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1352 unsigned *map_cmdline_to_pid;
1353 unsigned cmdline_num;
1354 int cmdline_idx;
1355 char *saved_cmdlines;
1356};
1357static struct saved_cmdlines_buffer *savedcmd;
Steven Rostedt25b0b442008-05-12 21:21:00 +02001358
Steven Rostedt25b0b442008-05-12 21:21:00 +02001359/* temporary disable recording */
Hannes Eder4fd27352009-02-10 19:44:12 +01001360static atomic_t trace_record_cmdline_disabled __read_mostly;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001361
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001362static inline char *get_saved_cmdlines(int idx)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001363{
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001364 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1365}
1366
1367static inline void set_cmdline(int idx, const char *cmdline)
1368{
1369 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1370}
1371
1372static int allocate_cmdlines_buffer(unsigned int val,
1373 struct saved_cmdlines_buffer *s)
1374{
1375 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1376 GFP_KERNEL);
1377 if (!s->map_cmdline_to_pid)
1378 return -ENOMEM;
1379
1380 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1381 if (!s->saved_cmdlines) {
1382 kfree(s->map_cmdline_to_pid);
1383 return -ENOMEM;
1384 }
1385
1386 s->cmdline_idx = 0;
1387 s->cmdline_num = val;
1388 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1389 sizeof(s->map_pid_to_cmdline));
1390 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1391 val * sizeof(*s->map_cmdline_to_pid));
1392
1393 return 0;
1394}
1395
1396static int trace_create_savedcmd(void)
1397{
1398 int ret;
1399
Namhyung Kima6af8fb2014-06-10 16:11:35 +09001400 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001401 if (!savedcmd)
1402 return -ENOMEM;
1403
1404 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1405 if (ret < 0) {
1406 kfree(savedcmd);
1407 savedcmd = NULL;
1408 return -ENOMEM;
1409 }
1410
1411 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001412}
1413
Carsten Emdeb5130b12009-09-13 01:43:07 +02001414int is_tracing_stopped(void)
1415{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001416 return global_trace.stop_count;
Carsten Emdeb5130b12009-09-13 01:43:07 +02001417}
1418
Steven Rostedt0f048702008-11-05 16:05:44 -05001419/**
1420 * tracing_start - quick start of the tracer
1421 *
1422 * If tracing is enabled but was stopped by tracing_stop,
1423 * this will start the tracer back up.
1424 */
1425void tracing_start(void)
1426{
1427 struct ring_buffer *buffer;
1428 unsigned long flags;
1429
1430 if (tracing_disabled)
1431 return;
1432
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001433 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1434 if (--global_trace.stop_count) {
1435 if (global_trace.stop_count < 0) {
Steven Rostedtb06a8302009-01-22 14:26:15 -05001436 /* Someone screwed up their debugging */
1437 WARN_ON_ONCE(1);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001438 global_trace.stop_count = 0;
Steven Rostedtb06a8302009-01-22 14:26:15 -05001439 }
Steven Rostedt0f048702008-11-05 16:05:44 -05001440 goto out;
1441 }
1442
Steven Rostedta2f80712010-03-12 19:56:00 -05001443 /* Prevent the buffers from switching */
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001444 arch_spin_lock(&global_trace.max_lock);
Steven Rostedt0f048702008-11-05 16:05:44 -05001445
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001446 buffer = global_trace.trace_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001447 if (buffer)
1448 ring_buffer_record_enable(buffer);
1449
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001450#ifdef CONFIG_TRACER_MAX_TRACE
1451 buffer = global_trace.max_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001452 if (buffer)
1453 ring_buffer_record_enable(buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001454#endif
Steven Rostedt0f048702008-11-05 16:05:44 -05001455
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001456 arch_spin_unlock(&global_trace.max_lock);
Steven Rostedta2f80712010-03-12 19:56:00 -05001457
Steven Rostedt0f048702008-11-05 16:05:44 -05001458 out:
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001459 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1460}
1461
1462static void tracing_start_tr(struct trace_array *tr)
1463{
1464 struct ring_buffer *buffer;
1465 unsigned long flags;
1466
1467 if (tracing_disabled)
1468 return;
1469
1470 /* If global, we need to also start the max tracer */
1471 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1472 return tracing_start();
1473
1474 raw_spin_lock_irqsave(&tr->start_lock, flags);
1475
1476 if (--tr->stop_count) {
1477 if (tr->stop_count < 0) {
1478 /* Someone screwed up their debugging */
1479 WARN_ON_ONCE(1);
1480 tr->stop_count = 0;
1481 }
1482 goto out;
1483 }
1484
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001485 buffer = tr->trace_buffer.buffer;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001486 if (buffer)
1487 ring_buffer_record_enable(buffer);
1488
1489 out:
1490 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001491}
1492
1493/**
1494 * tracing_stop - quick stop of the tracer
1495 *
1496 * Light weight way to stop tracing. Use in conjunction with
1497 * tracing_start.
1498 */
1499void tracing_stop(void)
1500{
1501 struct ring_buffer *buffer;
1502 unsigned long flags;
1503
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001504 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1505 if (global_trace.stop_count++)
Steven Rostedt0f048702008-11-05 16:05:44 -05001506 goto out;
1507
Steven Rostedta2f80712010-03-12 19:56:00 -05001508 /* Prevent the buffers from switching */
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001509 arch_spin_lock(&global_trace.max_lock);
Steven Rostedta2f80712010-03-12 19:56:00 -05001510
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001511 buffer = global_trace.trace_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001512 if (buffer)
1513 ring_buffer_record_disable(buffer);
1514
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001515#ifdef CONFIG_TRACER_MAX_TRACE
1516 buffer = global_trace.max_buffer.buffer;
Steven Rostedt0f048702008-11-05 16:05:44 -05001517 if (buffer)
1518 ring_buffer_record_disable(buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001519#endif
Steven Rostedt0f048702008-11-05 16:05:44 -05001520
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05001521 arch_spin_unlock(&global_trace.max_lock);
Steven Rostedta2f80712010-03-12 19:56:00 -05001522
Steven Rostedt0f048702008-11-05 16:05:44 -05001523 out:
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001524 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1525}
1526
1527static void tracing_stop_tr(struct trace_array *tr)
1528{
1529 struct ring_buffer *buffer;
1530 unsigned long flags;
1531
1532 /* If global, we need to also stop the max tracer */
1533 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1534 return tracing_stop();
1535
1536 raw_spin_lock_irqsave(&tr->start_lock, flags);
1537 if (tr->stop_count++)
1538 goto out;
1539
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001540 buffer = tr->trace_buffer.buffer;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04001541 if (buffer)
1542 ring_buffer_record_disable(buffer);
1543
1544 out:
1545 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
Steven Rostedt0f048702008-11-05 16:05:44 -05001546}
1547
Ingo Molnare309b412008-05-12 21:20:51 +02001548void trace_stop_cmdline_recording(void);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001549
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001550static int trace_save_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001551{
Carsten Emdea635cf02009-03-18 09:00:41 +01001552 unsigned pid, idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001553
1554 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001555 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001556
1557 /*
1558 * It's not the end of the world if we don't get
1559 * the lock, but we also don't want to spin
1560 * nor do we want to disable interrupts,
1561 * so if we miss here, then better luck next time.
1562 */
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001563 if (!arch_spin_trylock(&trace_cmdline_lock))
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001564 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001565
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001566 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
Thomas Gleixner2c7eea42009-03-18 09:03:19 +01001567 if (idx == NO_CMDLINE_MAP) {
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001568 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001569
Carsten Emdea635cf02009-03-18 09:00:41 +01001570 /*
1571 * Check whether the cmdline buffer at idx has a pid
1572 * mapped. We are going to overwrite that entry so we
1573 * need to clear the map_pid_to_cmdline. Otherwise we
1574 * would read the new comm for the old pid.
1575 */
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001576 pid = savedcmd->map_cmdline_to_pid[idx];
Carsten Emdea635cf02009-03-18 09:00:41 +01001577 if (pid != NO_CMDLINE_MAP)
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001578 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001579
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001580 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1581 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001582
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001583 savedcmd->cmdline_idx = idx;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001584 }
1585
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001586 set_cmdline(idx, tsk->comm);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001587
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001588 arch_spin_unlock(&trace_cmdline_lock);
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001589
1590 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001591}
1592
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04001593static void __trace_find_cmdline(int pid, char comm[])
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001594{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001595 unsigned map;
1596
Steven Rostedt4ca53082009-03-16 19:20:15 -04001597 if (!pid) {
1598 strcpy(comm, "<idle>");
1599 return;
1600 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001601
Steven Rostedt74bf4072010-01-25 15:11:53 -05001602 if (WARN_ON_ONCE(pid < 0)) {
1603 strcpy(comm, "<XXX>");
1604 return;
1605 }
1606
Steven Rostedt4ca53082009-03-16 19:20:15 -04001607 if (pid > PID_MAX_DEFAULT) {
1608 strcpy(comm, "<...>");
1609 return;
1610 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001611
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001612 map = savedcmd->map_pid_to_cmdline[pid];
Thomas Gleixner50d88752009-03-18 08:58:44 +01001613 if (map != NO_CMDLINE_MAP)
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09001614 strcpy(comm, get_saved_cmdlines(map));
Thomas Gleixner50d88752009-03-18 08:58:44 +01001615 else
1616 strcpy(comm, "<...>");
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04001617}
1618
1619void trace_find_cmdline(int pid, char comm[])
1620{
1621 preempt_disable();
1622 arch_spin_lock(&trace_cmdline_lock);
1623
1624 __trace_find_cmdline(pid, comm);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001625
Thomas Gleixner0199c4e2009-12-02 20:01:25 +01001626 arch_spin_unlock(&trace_cmdline_lock);
Heiko Carstens5b6045a2009-05-26 17:28:02 +02001627 preempt_enable();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001628}
1629
Ingo Molnare309b412008-05-12 21:20:51 +02001630void tracing_record_cmdline(struct task_struct *tsk)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001631{
Steven Rostedt0fb96562012-05-11 14:25:30 -04001632 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001633 return;
1634
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001635 if (!__this_cpu_read(trace_cmdline_save))
1636 return;
1637
Steven Rostedt (Red Hat)379cfda2014-05-30 09:42:39 -04001638 if (trace_save_cmdline(tsk))
1639 __this_cpu_write(trace_cmdline_save, false);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001640}
1641
Pekka Paalanen45dcd8b2008-09-16 21:56:41 +03001642void
Steven Rostedt38697052008-10-01 13:14:09 -04001643tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1644 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001645{
1646 struct task_struct *tsk = current;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001647
Steven Rostedt777e2082008-09-29 23:02:42 -04001648 entry->preempt_count = pc & 0xff;
1649 entry->pid = (tsk) ? tsk->pid : 0;
1650 entry->flags =
Steven Rostedt92444892008-10-24 09:42:59 -04001651#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
Steven Rostedt2e2ca152008-08-01 12:26:40 -04001652 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
Steven Rostedt92444892008-10-24 09:42:59 -04001653#else
1654 TRACE_FLAG_IRQS_NOSUPPORT |
1655#endif
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001656 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1657 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
Peter Zijlstrae5137b52013-10-04 17:28:26 +02001658 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1659 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001660}
Frederic Weisbeckerf413cdb2009-08-07 01:25:54 +02001661EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001662
Steven Rostedte77405a2009-09-02 14:17:06 -04001663struct ring_buffer_event *
1664trace_buffer_lock_reserve(struct ring_buffer *buffer,
1665 int type,
1666 unsigned long len,
1667 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001668{
1669 struct ring_buffer_event *event;
1670
Steven Rostedte77405a2009-09-02 14:17:06 -04001671 event = ring_buffer_lock_reserve(buffer, len);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001672 if (event != NULL) {
1673 struct trace_entry *ent = ring_buffer_event_data(event);
1674
1675 tracing_generic_entry_update(ent, flags, pc);
1676 ent->type = type;
1677 }
1678
1679 return event;
1680}
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001681
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001682void
1683__buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1684{
1685 __this_cpu_write(trace_cmdline_save, true);
1686 ring_buffer_unlock_commit(buffer, event);
1687}
1688
Steven Rostedt (Red Hat)b7f0c952015-09-25 17:38:44 -04001689void trace_buffer_unlock_commit(struct trace_array *tr,
1690 struct ring_buffer *buffer,
1691 struct ring_buffer_event *event,
1692 unsigned long flags, int pc)
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001693{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001694 __buffer_unlock_commit(buffer, event);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001695
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001696 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
Steven Rostedte77405a2009-09-02 14:17:06 -04001697 ftrace_trace_userstack(buffer, flags, pc);
Frederic Weisbecker07edf712009-03-22 23:10:46 +01001698}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001699EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001700
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001701static struct ring_buffer *temp_buffer;
1702
Steven Rostedtef5580d2009-02-27 19:38:04 -05001703struct ring_buffer_event *
Steven Rostedtccb469a2012-08-02 10:32:10 -04001704trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
Steven Rostedt (Red Hat)7f1d2f82015-05-05 10:09:53 -04001705 struct trace_event_file *trace_file,
Steven Rostedtccb469a2012-08-02 10:32:10 -04001706 int type, unsigned long len,
1707 unsigned long flags, int pc)
1708{
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001709 struct ring_buffer_event *entry;
1710
Steven Rostedt (Red Hat)7f1d2f82015-05-05 10:09:53 -04001711 *current_rb = trace_file->tr->trace_buffer.buffer;
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001712 entry = trace_buffer_lock_reserve(*current_rb,
Steven Rostedtccb469a2012-08-02 10:32:10 -04001713 type, len, flags, pc);
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001714 /*
1715 * If tracing is off, but we have triggers enabled
1716 * we still need to look at the event data. Use the temp_buffer
1717 * to store the trace event for the tigger to use. It's recusive
1718 * safe and will not be recorded anywhere.
1719 */
Steven Rostedt (Red Hat)5d6ad962015-05-13 15:12:33 -04001720 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04001721 *current_rb = temp_buffer;
1722 entry = trace_buffer_lock_reserve(*current_rb,
1723 type, len, flags, pc);
1724 }
1725 return entry;
Steven Rostedtccb469a2012-08-02 10:32:10 -04001726}
1727EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1728
1729struct ring_buffer_event *
Steven Rostedte77405a2009-09-02 14:17:06 -04001730trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1731 int type, unsigned long len,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001732 unsigned long flags, int pc)
1733{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001734 *current_rb = global_trace.trace_buffer.buffer;
Steven Rostedte77405a2009-09-02 14:17:06 -04001735 return trace_buffer_lock_reserve(*current_rb,
Steven Rostedtef5580d2009-02-27 19:38:04 -05001736 type, len, flags, pc);
1737}
Steven Rostedt94487d62009-05-05 19:22:53 -04001738EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001739
Steven Rostedt (Red Hat)b7f0c952015-09-25 17:38:44 -04001740void trace_buffer_unlock_commit_regs(struct trace_array *tr,
1741 struct ring_buffer *buffer,
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001742 struct ring_buffer_event *event,
1743 unsigned long flags, int pc,
1744 struct pt_regs *regs)
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001745{
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001746 __buffer_unlock_commit(buffer, event);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001747
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001748 ftrace_trace_stack(tr, buffer, flags, 6, pc, regs);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001749 ftrace_trace_userstack(buffer, flags, pc);
1750}
Steven Rostedt0d5c6e12012-11-01 20:54:21 -04001751EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001752
Steven Rostedte77405a2009-09-02 14:17:06 -04001753void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1754 struct ring_buffer_event *event)
Steven Rostedt77d9f462009-04-02 01:16:59 -04001755{
Steven Rostedte77405a2009-09-02 14:17:06 -04001756 ring_buffer_discard_commit(buffer, event);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001757}
Steven Rostedt12acd472009-04-17 16:01:56 -04001758EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
Steven Rostedtef5580d2009-02-27 19:38:04 -05001759
Ingo Molnare309b412008-05-12 21:20:51 +02001760void
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001761trace_function(struct trace_array *tr,
Steven Rostedt38697052008-10-01 13:14:09 -04001762 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1763 int pc)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001764{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04001765 struct trace_event_call *call = &event_function;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001766 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001767 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001768 struct ftrace_entry *entry;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001769
Steven Rostedtd7690412008-10-01 00:29:53 -04001770 /* If we are reading the ring buffer, don't trace */
Rusty Russelldd17c8f2009-10-29 22:34:15 +09001771 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
Steven Rostedtd7690412008-10-01 00:29:53 -04001772 return;
1773
Steven Rostedte77405a2009-09-02 14:17:06 -04001774 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001775 flags, pc);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001776 if (!event)
1777 return;
1778 entry = ring_buffer_event_data(event);
Steven Rostedt777e2082008-09-29 23:02:42 -04001779 entry->ip = ip;
1780 entry->parent_ip = parent_ip;
Tom Zanussie1112b42009-03-31 00:48:49 -05001781
Tom Zanussif306cc82013-10-24 08:34:17 -05001782 if (!call_filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001783 __buffer_unlock_commit(buffer, event);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02001784}
1785
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001786#ifdef CONFIG_STACKTRACE
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001787
1788#define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1789struct ftrace_stack {
1790 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1791};
1792
1793static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1794static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1795
Steven Rostedte77405a2009-09-02 14:17:06 -04001796static void __ftrace_trace_stack(struct ring_buffer *buffer,
Steven Rostedt53614992009-01-15 19:12:40 -05001797 unsigned long flags,
Masami Hiramatsu1fd8df22011-06-08 16:09:34 +09001798 int skip, int pc, struct pt_regs *regs)
Ingo Molnar86387f72008-05-12 21:20:51 +02001799{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04001800 struct trace_event_call *call = &event_kernel_stack;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04001801 struct ring_buffer_event *event;
Steven Rostedt777e2082008-09-29 23:02:42 -04001802 struct stack_entry *entry;
Ingo Molnar86387f72008-05-12 21:20:51 +02001803 struct stack_trace trace;
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001804 int use_stack;
1805 int size = FTRACE_STACK_ENTRIES;
Ingo Molnar86387f72008-05-12 21:20:51 +02001806
1807 trace.nr_entries = 0;
Ingo Molnar86387f72008-05-12 21:20:51 +02001808 trace.skip = skip;
Ingo Molnar86387f72008-05-12 21:20:51 +02001809
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001810 /*
1811 * Since events can happen in NMIs there's no safe way to
1812 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1813 * or NMI comes in, it will just have to use the default
1814 * FTRACE_STACK_SIZE.
1815 */
1816 preempt_disable_notrace();
1817
Shan Wei82146522012-11-19 13:21:01 +08001818 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001819 /*
1820 * We don't need any atomic variables, just a barrier.
1821 * If an interrupt comes in, we don't care, because it would
1822 * have exited and put the counter back to what we want.
1823 * We just need a barrier to keep gcc from moving things
1824 * around.
1825 */
1826 barrier();
1827 if (use_stack == 1) {
Christoph Lameterbdffd892014-04-29 14:17:40 -05001828 trace.entries = this_cpu_ptr(ftrace_stack.calls);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001829 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1830
1831 if (regs)
1832 save_stack_trace_regs(regs, &trace);
1833 else
1834 save_stack_trace(&trace);
1835
1836 if (trace.nr_entries > size)
1837 size = trace.nr_entries;
1838 } else
1839 /* From now on, use_stack is a boolean */
1840 use_stack = 0;
1841
1842 size *= sizeof(unsigned long);
1843
1844 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1845 sizeof(*entry) + size, flags, pc);
1846 if (!event)
1847 goto out;
1848 entry = ring_buffer_event_data(event);
1849
1850 memset(&entry->caller, 0, size);
1851
1852 if (use_stack)
1853 memcpy(&entry->caller, trace.entries,
1854 trace.nr_entries * sizeof(unsigned long));
1855 else {
1856 trace.max_entries = FTRACE_STACK_ENTRIES;
1857 trace.entries = entry->caller;
1858 if (regs)
1859 save_stack_trace_regs(regs, &trace);
1860 else
1861 save_stack_trace(&trace);
1862 }
1863
1864 entry->size = trace.nr_entries;
1865
Tom Zanussif306cc82013-10-24 08:34:17 -05001866 if (!call_filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001867 __buffer_unlock_commit(buffer, event);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001868
1869 out:
1870 /* Again, don't let gcc optimize things here */
1871 barrier();
Shan Wei82146522012-11-19 13:21:01 +08001872 __this_cpu_dec(ftrace_stack_reserve);
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04001873 preempt_enable_notrace();
1874
Ingo Molnarf0a920d2008-05-12 21:20:47 +02001875}
1876
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001877static inline void ftrace_trace_stack(struct trace_array *tr,
1878 struct ring_buffer *buffer,
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -04001879 unsigned long flags,
1880 int skip, int pc, struct pt_regs *regs)
Steven Rostedt53614992009-01-15 19:12:40 -05001881{
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04001882 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
Steven Rostedt53614992009-01-15 19:12:40 -05001883 return;
1884
Steven Rostedt (Red Hat)73dddbb2015-09-29 15:38:55 -04001885 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
Steven Rostedt53614992009-01-15 19:12:40 -05001886}
1887
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001888void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1889 int pc)
Steven Rostedt38697052008-10-01 13:14:09 -04001890{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05001891 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
Steven Rostedt38697052008-10-01 13:14:09 -04001892}
1893
Steven Rostedt03889382009-12-11 09:48:22 -05001894/**
1895 * trace_dump_stack - record a stack back trace in the trace buffer
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001896 * @skip: Number of functions to skip (helper handlers)
Steven Rostedt03889382009-12-11 09:48:22 -05001897 */
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001898void trace_dump_stack(int skip)
Steven Rostedt03889382009-12-11 09:48:22 -05001899{
1900 unsigned long flags;
1901
1902 if (tracing_disabled || tracing_selftest_running)
Steven Rostedte36c5452009-12-14 15:58:33 -05001903 return;
Steven Rostedt03889382009-12-11 09:48:22 -05001904
1905 local_save_flags(flags);
1906
Steven Rostedt (Red Hat)c142be82013-03-13 09:55:57 -04001907 /*
1908 * Skip 3 more, seems to get us at the caller of
1909 * this function.
1910 */
1911 skip += 3;
1912 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1913 flags, skip, preempt_count(), NULL);
Steven Rostedt03889382009-12-11 09:48:22 -05001914}
1915
Steven Rostedt91e86e52010-11-10 12:56:12 +01001916static DEFINE_PER_CPU(int, user_stack_count);
1917
Steven Rostedte77405a2009-09-02 14:17:06 -04001918void
1919ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
Török Edwin02b67512008-11-22 13:28:47 +02001920{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04001921 struct trace_event_call *call = &event_user_stack;
Török Edwin8d7c6a92008-11-23 12:39:06 +02001922 struct ring_buffer_event *event;
Török Edwin02b67512008-11-22 13:28:47 +02001923 struct userstack_entry *entry;
1924 struct stack_trace trace;
Török Edwin02b67512008-11-22 13:28:47 +02001925
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04001926 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
Török Edwin02b67512008-11-22 13:28:47 +02001927 return;
1928
Steven Rostedtb6345872010-03-12 20:03:30 -05001929 /*
1930 * NMIs can not handle page faults, even with fix ups.
1931 * The save user stack can (and often does) fault.
1932 */
1933 if (unlikely(in_nmi()))
1934 return;
1935
Steven Rostedt91e86e52010-11-10 12:56:12 +01001936 /*
1937 * prevent recursion, since the user stack tracing may
1938 * trigger other kernel events.
1939 */
1940 preempt_disable();
1941 if (__this_cpu_read(user_stack_count))
1942 goto out;
1943
1944 __this_cpu_inc(user_stack_count);
1945
Steven Rostedte77405a2009-09-02 14:17:06 -04001946 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
Arnaldo Carvalho de Melo51a763d2009-02-05 16:14:13 -02001947 sizeof(*entry), flags, pc);
Török Edwin02b67512008-11-22 13:28:47 +02001948 if (!event)
Li Zefan1dbd1952010-12-09 15:47:56 +08001949 goto out_drop_count;
Török Edwin02b67512008-11-22 13:28:47 +02001950 entry = ring_buffer_event_data(event);
Török Edwin02b67512008-11-22 13:28:47 +02001951
Steven Rostedt48659d32009-09-11 11:36:23 -04001952 entry->tgid = current->tgid;
Török Edwin02b67512008-11-22 13:28:47 +02001953 memset(&entry->caller, 0, sizeof(entry->caller));
1954
1955 trace.nr_entries = 0;
1956 trace.max_entries = FTRACE_STACK_ENTRIES;
1957 trace.skip = 0;
1958 trace.entries = entry->caller;
1959
1960 save_stack_trace_user(&trace);
Tom Zanussif306cc82013-10-24 08:34:17 -05001961 if (!call_filter_check_discard(call, entry, buffer, event))
Steven Rostedt7ffbd482012-10-11 12:14:25 -04001962 __buffer_unlock_commit(buffer, event);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001963
Li Zefan1dbd1952010-12-09 15:47:56 +08001964 out_drop_count:
Steven Rostedt91e86e52010-11-10 12:56:12 +01001965 __this_cpu_dec(user_stack_count);
Steven Rostedt91e86e52010-11-10 12:56:12 +01001966 out:
1967 preempt_enable();
Török Edwin02b67512008-11-22 13:28:47 +02001968}
1969
Hannes Eder4fd27352009-02-10 19:44:12 +01001970#ifdef UNUSED
1971static void __trace_userstack(struct trace_array *tr, unsigned long flags)
Török Edwin02b67512008-11-22 13:28:47 +02001972{
Arnaldo Carvalho de Melo7be42152009-02-05 01:13:37 -05001973 ftrace_trace_userstack(tr, flags, preempt_count());
Török Edwin02b67512008-11-22 13:28:47 +02001974}
Hannes Eder4fd27352009-02-10 19:44:12 +01001975#endif /* UNUSED */
Török Edwin02b67512008-11-22 13:28:47 +02001976
Frederic Weisbeckerc0a0d0d2009-07-29 17:51:13 +02001977#endif /* CONFIG_STACKTRACE */
1978
Steven Rostedt07d777f2011-09-22 14:01:55 -04001979/* created for use with alloc_percpu */
1980struct trace_buffer_struct {
1981 char buffer[TRACE_BUF_SIZE];
1982};
1983
1984static struct trace_buffer_struct *trace_percpu_buffer;
1985static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1986static struct trace_buffer_struct *trace_percpu_irq_buffer;
1987static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1988
1989/*
1990 * The buffer used is dependent on the context. There is a per cpu
1991 * buffer for normal context, softirq contex, hard irq context and
1992 * for NMI context. Thise allows for lockless recording.
1993 *
1994 * Note, if the buffers failed to be allocated, then this returns NULL
1995 */
1996static char *get_trace_buf(void)
1997{
1998 struct trace_buffer_struct *percpu_buffer;
Steven Rostedt07d777f2011-09-22 14:01:55 -04001999
2000 /*
2001 * If we have allocated per cpu buffers, then we do not
2002 * need to do any locking.
2003 */
2004 if (in_nmi())
2005 percpu_buffer = trace_percpu_nmi_buffer;
2006 else if (in_irq())
2007 percpu_buffer = trace_percpu_irq_buffer;
2008 else if (in_softirq())
2009 percpu_buffer = trace_percpu_sirq_buffer;
2010 else
2011 percpu_buffer = trace_percpu_buffer;
2012
2013 if (!percpu_buffer)
2014 return NULL;
2015
Shan Weid8a03492012-11-13 09:53:04 +08002016 return this_cpu_ptr(&percpu_buffer->buffer[0]);
Steven Rostedt07d777f2011-09-22 14:01:55 -04002017}
2018
2019static int alloc_percpu_trace_buffer(void)
2020{
2021 struct trace_buffer_struct *buffers;
2022 struct trace_buffer_struct *sirq_buffers;
2023 struct trace_buffer_struct *irq_buffers;
2024 struct trace_buffer_struct *nmi_buffers;
2025
2026 buffers = alloc_percpu(struct trace_buffer_struct);
2027 if (!buffers)
2028 goto err_warn;
2029
2030 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2031 if (!sirq_buffers)
2032 goto err_sirq;
2033
2034 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2035 if (!irq_buffers)
2036 goto err_irq;
2037
2038 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2039 if (!nmi_buffers)
2040 goto err_nmi;
2041
2042 trace_percpu_buffer = buffers;
2043 trace_percpu_sirq_buffer = sirq_buffers;
2044 trace_percpu_irq_buffer = irq_buffers;
2045 trace_percpu_nmi_buffer = nmi_buffers;
2046
2047 return 0;
2048
2049 err_nmi:
2050 free_percpu(irq_buffers);
2051 err_irq:
2052 free_percpu(sirq_buffers);
2053 err_sirq:
2054 free_percpu(buffers);
2055 err_warn:
2056 WARN(1, "Could not allocate percpu trace_printk buffer");
2057 return -ENOMEM;
2058}
2059
Steven Rostedt81698832012-10-11 10:15:05 -04002060static int buffers_allocated;
2061
Steven Rostedt07d777f2011-09-22 14:01:55 -04002062void trace_printk_init_buffers(void)
2063{
Steven Rostedt07d777f2011-09-22 14:01:55 -04002064 if (buffers_allocated)
2065 return;
2066
2067 if (alloc_percpu_trace_buffer())
2068 return;
2069
Steven Rostedt2184db42014-05-28 13:14:40 -04002070 /* trace_printk() is for debug use only. Don't use it in production. */
2071
Borislav Petkov69a1c992015-01-27 17:17:20 +01002072 pr_warning("\n");
2073 pr_warning("**********************************************************\n");
Steven Rostedt2184db42014-05-28 13:14:40 -04002074 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2075 pr_warning("** **\n");
2076 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2077 pr_warning("** **\n");
2078 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
Frans Klavereff264e2014-11-07 15:53:44 +01002079 pr_warning("** unsafe for production use. **\n");
Steven Rostedt2184db42014-05-28 13:14:40 -04002080 pr_warning("** **\n");
2081 pr_warning("** If you see this message and you are not debugging **\n");
2082 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2083 pr_warning("** **\n");
2084 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2085 pr_warning("**********************************************************\n");
Steven Rostedt07d777f2011-09-22 14:01:55 -04002086
Steven Rostedtb382ede62012-10-10 21:44:34 -04002087 /* Expand the buffers to set size */
2088 tracing_update_buffers();
2089
Steven Rostedt07d777f2011-09-22 14:01:55 -04002090 buffers_allocated = 1;
Steven Rostedt81698832012-10-11 10:15:05 -04002091
2092 /*
2093 * trace_printk_init_buffers() can be called by modules.
2094 * If that happens, then we need to start cmdline recording
2095 * directly here. If the global_trace.buffer is already
2096 * allocated here, then this was called by module code.
2097 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002098 if (global_trace.trace_buffer.buffer)
Steven Rostedt81698832012-10-11 10:15:05 -04002099 tracing_start_cmdline_record();
2100}
2101
2102void trace_printk_start_comm(void)
2103{
2104 /* Start tracing comms if trace printk is set */
2105 if (!buffers_allocated)
2106 return;
2107 tracing_start_cmdline_record();
2108}
2109
2110static void trace_printk_start_stop_comm(int enabled)
2111{
2112 if (!buffers_allocated)
2113 return;
2114
2115 if (enabled)
2116 tracing_start_cmdline_record();
2117 else
2118 tracing_stop_cmdline_record();
Steven Rostedt07d777f2011-09-22 14:01:55 -04002119}
2120
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002121/**
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002122 * trace_vbprintk - write binary msg to tracing buffer
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002123 *
2124 */
Steven Rostedt40ce74f2009-03-19 14:03:53 -04002125int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002126{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04002127 struct trace_event_call *call = &event_bprint;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002128 struct ring_buffer_event *event;
Steven Rostedte77405a2009-09-02 14:17:06 -04002129 struct ring_buffer *buffer;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002130 struct trace_array *tr = &global_trace;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002131 struct bprint_entry *entry;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002132 unsigned long flags;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002133 char *tbuffer;
2134 int len = 0, size, pc;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002135
2136 if (unlikely(tracing_selftest_running || tracing_disabled))
2137 return 0;
2138
2139 /* Don't pollute graph traces with trace_vprintk internals */
2140 pause_graph_tracing();
2141
2142 pc = preempt_count();
Steven Rostedt5168ae52010-06-03 09:36:50 -04002143 preempt_disable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002144
Steven Rostedt07d777f2011-09-22 14:01:55 -04002145 tbuffer = get_trace_buf();
2146 if (!tbuffer) {
2147 len = 0;
2148 goto out;
2149 }
2150
2151 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2152
2153 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002154 goto out;
2155
Steven Rostedt07d777f2011-09-22 14:01:55 -04002156 local_save_flags(flags);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002157 size = sizeof(*entry) + sizeof(u32) * len;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002158 buffer = tr->trace_buffer.buffer;
Steven Rostedte77405a2009-09-02 14:17:06 -04002159 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2160 flags, pc);
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002161 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04002162 goto out;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002163 entry = ring_buffer_event_data(event);
2164 entry->ip = ip;
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002165 entry->fmt = fmt;
2166
Steven Rostedt07d777f2011-09-22 14:01:55 -04002167 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
Tom Zanussif306cc82013-10-24 08:34:17 -05002168 if (!call_filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04002169 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04002170 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
Steven Rostedtd9313692010-01-06 17:27:11 -05002171 }
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002172
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002173out:
Steven Rostedt5168ae52010-06-03 09:36:50 -04002174 preempt_enable_notrace();
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002175 unpause_graph_tracing();
2176
2177 return len;
2178}
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002179EXPORT_SYMBOL_GPL(trace_vbprintk);
2180
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002181static int
2182__trace_array_vprintk(struct ring_buffer *buffer,
2183 unsigned long ip, const char *fmt, va_list args)
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002184{
Steven Rostedt (Red Hat)2425bcb2015-05-05 11:45:27 -04002185 struct trace_event_call *call = &event_print;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002186 struct ring_buffer_event *event;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002187 int len = 0, size, pc;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002188 struct print_entry *entry;
Steven Rostedt07d777f2011-09-22 14:01:55 -04002189 unsigned long flags;
2190 char *tbuffer;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002191
2192 if (tracing_disabled || tracing_selftest_running)
2193 return 0;
2194
Steven Rostedt07d777f2011-09-22 14:01:55 -04002195 /* Don't pollute graph traces with trace_vprintk internals */
2196 pause_graph_tracing();
2197
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002198 pc = preempt_count();
2199 preempt_disable_notrace();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002200
Steven Rostedt07d777f2011-09-22 14:01:55 -04002201
2202 tbuffer = get_trace_buf();
2203 if (!tbuffer) {
2204 len = 0;
2205 goto out;
2206 }
2207
Dan Carpenter3558a5a2014-11-27 18:57:52 +03002208 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002209
Steven Rostedt07d777f2011-09-22 14:01:55 -04002210 local_save_flags(flags);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002211 size = sizeof(*entry) + len + 1;
Steven Rostedte77405a2009-09-02 14:17:06 -04002212 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
Steven Rostedt07d777f2011-09-22 14:01:55 -04002213 flags, pc);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002214 if (!event)
Steven Rostedt07d777f2011-09-22 14:01:55 -04002215 goto out;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002216 entry = ring_buffer_event_data(event);
Carsten Emdec13d2f72009-11-16 20:56:13 +01002217 entry->ip = ip;
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002218
Dan Carpenter3558a5a2014-11-27 18:57:52 +03002219 memcpy(&entry->buf, tbuffer, len + 1);
Tom Zanussif306cc82013-10-24 08:34:17 -05002220 if (!call_filter_check_discard(call, entry, buffer, event)) {
Steven Rostedt7ffbd482012-10-11 12:14:25 -04002221 __buffer_unlock_commit(buffer, event);
Steven Rostedt (Red Hat)2d34f482015-09-30 11:45:22 -04002222 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
Steven Rostedtd9313692010-01-06 17:27:11 -05002223 }
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002224 out:
2225 preempt_enable_notrace();
Steven Rostedt07d777f2011-09-22 14:01:55 -04002226 unpause_graph_tracing();
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002227
2228 return len;
2229}
Steven Rostedt659372d2009-09-03 19:11:07 -04002230
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002231int trace_array_vprintk(struct trace_array *tr,
2232 unsigned long ip, const char *fmt, va_list args)
2233{
2234 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2235}
2236
2237int trace_array_printk(struct trace_array *tr,
2238 unsigned long ip, const char *fmt, ...)
2239{
2240 int ret;
2241 va_list ap;
2242
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002243 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002244 return 0;
2245
2246 va_start(ap, fmt);
2247 ret = trace_array_vprintk(tr, ip, fmt, ap);
2248 va_end(ap);
2249 return ret;
2250}
2251
2252int trace_array_printk_buf(struct ring_buffer *buffer,
2253 unsigned long ip, const char *fmt, ...)
2254{
2255 int ret;
2256 va_list ap;
2257
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002258 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002259 return 0;
2260
2261 va_start(ap, fmt);
2262 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2263 va_end(ap);
2264 return ret;
2265}
2266
Steven Rostedt659372d2009-09-03 19:11:07 -04002267int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2268{
Steven Rostedta813a152009-10-09 01:41:35 -04002269 return trace_array_vprintk(&global_trace, ip, fmt, args);
Steven Rostedt659372d2009-09-03 19:11:07 -04002270}
Frederic Weisbecker769b0442009-03-06 17:21:49 +01002271EXPORT_SYMBOL_GPL(trace_vprintk);
2272
Robert Richtere2ac8ef2008-11-12 12:59:32 +01002273static void trace_iterator_increment(struct trace_iterator *iter)
Steven Rostedt5a90f572008-09-03 17:42:51 -04002274{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002275 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2276
Steven Rostedt5a90f572008-09-03 17:42:51 -04002277 iter->idx++;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002278 if (buf_iter)
2279 ring_buffer_read(buf_iter, NULL);
Steven Rostedt5a90f572008-09-03 17:42:51 -04002280}
2281
Ingo Molnare309b412008-05-12 21:20:51 +02002282static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04002283peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2284 unsigned long *lost_events)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002285{
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002286 struct ring_buffer_event *event;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002287 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002288
Steven Rostedtd7690412008-10-01 00:29:53 -04002289 if (buf_iter)
2290 event = ring_buffer_iter_peek(buf_iter, ts);
2291 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002292 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
Steven Rostedtbc21b472010-03-31 19:49:26 -04002293 lost_events);
Steven Rostedtd7690412008-10-01 00:29:53 -04002294
Steven Rostedt4a9bd3f2011-07-14 16:36:53 -04002295 if (event) {
2296 iter->ent_size = ring_buffer_event_length(event);
2297 return ring_buffer_event_data(event);
2298 }
2299 iter->ent_size = 0;
2300 return NULL;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002301}
Steven Rostedtd7690412008-10-01 00:29:53 -04002302
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002303static struct trace_entry *
Steven Rostedtbc21b472010-03-31 19:49:26 -04002304__find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2305 unsigned long *missing_events, u64 *ent_ts)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002306{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002307 struct ring_buffer *buffer = iter->trace_buffer->buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002308 struct trace_entry *ent, *next = NULL;
Lai Jiangshanaa274972010-04-05 17:11:05 +08002309 unsigned long lost_events = 0, next_lost = 0;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002310 int cpu_file = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002311 u64 next_ts = 0, ts;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002312 int next_cpu = -1;
Steven Rostedt12b5da32012-03-27 10:43:28 -04002313 int next_size = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002314 int cpu;
2315
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002316 /*
2317 * If we are in a per_cpu trace file, don't bother by iterating over
2318 * all cpu and peek directly.
2319 */
Steven Rostedtae3b5092013-01-23 15:22:59 -05002320 if (cpu_file > RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002321 if (ring_buffer_empty_cpu(buffer, cpu_file))
2322 return NULL;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002323 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002324 if (ent_cpu)
2325 *ent_cpu = cpu_file;
2326
2327 return ent;
2328 }
2329
Steven Rostedtab464282008-05-12 21:21:00 +02002330 for_each_tracing_cpu(cpu) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002331
2332 if (ring_buffer_empty_cpu(buffer, cpu))
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002333 continue;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002334
Steven Rostedtbc21b472010-03-31 19:49:26 -04002335 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002336
Ingo Molnarcdd31cd2008-05-12 21:20:46 +02002337 /*
2338 * Pick the entry with the smallest timestamp:
2339 */
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002340 if (ent && (!next || ts < next_ts)) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002341 next = ent;
2342 next_cpu = cpu;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002343 next_ts = ts;
Steven Rostedtbc21b472010-03-31 19:49:26 -04002344 next_lost = lost_events;
Steven Rostedt12b5da32012-03-27 10:43:28 -04002345 next_size = iter->ent_size;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002346 }
2347 }
2348
Steven Rostedt12b5da32012-03-27 10:43:28 -04002349 iter->ent_size = next_size;
2350
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002351 if (ent_cpu)
2352 *ent_cpu = next_cpu;
2353
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002354 if (ent_ts)
2355 *ent_ts = next_ts;
2356
Steven Rostedtbc21b472010-03-31 19:49:26 -04002357 if (missing_events)
2358 *missing_events = next_lost;
2359
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002360 return next;
2361}
2362
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002363/* Find the next real entry, without updating the iterator itself */
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002364struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2365 int *ent_cpu, u64 *ent_ts)
Steven Rostedtb3806b42008-05-12 21:20:46 +02002366{
Steven Rostedtbc21b472010-03-31 19:49:26 -04002367 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002368}
Ingo Molnar8c523a92008-05-12 21:20:46 +02002369
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002370/* Find the next real entry, and increment the iterator to the next entry */
Jason Wessel955b61e2010-08-05 09:22:23 -05002371void *trace_find_next_entry_inc(struct trace_iterator *iter)
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002372{
Steven Rostedtbc21b472010-03-31 19:49:26 -04002373 iter->ent = __find_next_entry(iter, &iter->cpu,
2374 &iter->lost_events, &iter->ts);
Steven Rostedtb3806b42008-05-12 21:20:46 +02002375
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002376 if (iter->ent)
Robert Richtere2ac8ef2008-11-12 12:59:32 +01002377 trace_iterator_increment(iter);
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002378
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002379 return iter->ent ? iter : NULL;
Steven Rostedtb3806b42008-05-12 21:20:46 +02002380}
2381
Ingo Molnare309b412008-05-12 21:20:51 +02002382static void trace_consume(struct trace_iterator *iter)
Steven Rostedtb3806b42008-05-12 21:20:46 +02002383{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002384 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
Steven Rostedtbc21b472010-03-31 19:49:26 -04002385 &iter->lost_events);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002386}
2387
Ingo Molnare309b412008-05-12 21:20:51 +02002388static void *s_next(struct seq_file *m, void *v, loff_t *pos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002389{
2390 struct trace_iterator *iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002391 int i = (int)*pos;
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002392 void *ent;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002393
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002394 WARN_ON_ONCE(iter->leftover);
2395
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002396 (*pos)++;
2397
2398 /* can't go backwards */
2399 if (iter->idx > i)
2400 return NULL;
2401
2402 if (iter->idx < 0)
Jason Wessel955b61e2010-08-05 09:22:23 -05002403 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002404 else
2405 ent = iter;
2406
2407 while (ent && iter->idx < i)
Jason Wessel955b61e2010-08-05 09:22:23 -05002408 ent = trace_find_next_entry_inc(iter);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002409
2410 iter->pos = *pos;
2411
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002412 return ent;
2413}
2414
Jason Wessel955b61e2010-08-05 09:22:23 -05002415void tracing_iter_reset(struct trace_iterator *iter, int cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002416{
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002417 struct ring_buffer_event *event;
2418 struct ring_buffer_iter *buf_iter;
2419 unsigned long entries = 0;
2420 u64 ts;
2421
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002422 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002423
Steven Rostedt6d158a82012-06-27 20:46:14 -04002424 buf_iter = trace_buffer_iter(iter, cpu);
2425 if (!buf_iter)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002426 return;
2427
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002428 ring_buffer_iter_reset(buf_iter);
2429
2430 /*
2431 * We could have the case with the max latency tracers
2432 * that a reset never took place on a cpu. This is evident
2433 * by the timestamp being before the start of the buffer.
2434 */
2435 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002436 if (ts >= iter->trace_buffer->time_start)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002437 break;
2438 entries++;
2439 ring_buffer_read(buf_iter, NULL);
2440 }
2441
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002442 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002443}
2444
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002445/*
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002446 * The current tracer is copied to avoid a global locking
2447 * all around.
2448 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002449static void *s_start(struct seq_file *m, loff_t *pos)
2450{
2451 struct trace_iterator *iter = m->private;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002452 struct trace_array *tr = iter->tr;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002453 int cpu_file = iter->cpu_file;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002454 void *p = NULL;
2455 loff_t l = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04002456 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002457
Hiraku Toyooka2fd196e2012-12-26 11:52:52 +09002458 /*
2459 * copy the tracer to avoid using a global lock all around.
2460 * iter->trace is a copy of current_trace, the pointer to the
2461 * name may be used instead of a strcmp(), as iter->trace->name
2462 * will point to the same string as current_trace->name.
2463 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002464 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002465 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2466 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01002467 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002468
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002469#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002470 if (iter->snapshot && iter->trace->use_max_tr)
2471 return ERR_PTR(-EBUSY);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002472#endif
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002473
2474 if (!iter->snapshot)
2475 atomic_inc(&trace_record_cmdline_disabled);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002476
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002477 if (*pos != iter->pos) {
2478 iter->ent = NULL;
2479 iter->cpu = 0;
2480 iter->idx = -1;
2481
Steven Rostedtae3b5092013-01-23 15:22:59 -05002482 if (cpu_file == RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002483 for_each_tracing_cpu(cpu)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002484 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01002485 } else
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002486 tracing_iter_reset(iter, cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002487
Lai Jiangshanac91d852010-03-02 17:54:50 +08002488 iter->leftover = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002489 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2490 ;
2491
2492 } else {
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002493 /*
2494 * If we overflowed the seq_file before, then we want
2495 * to just reuse the trace_seq buffer again.
2496 */
2497 if (iter->leftover)
2498 p = iter;
2499 else {
2500 l = *pos - 1;
2501 p = s_next(m, p, &l);
2502 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002503 }
2504
Lai Jiangshan4f535962009-05-18 19:35:34 +08002505 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002506 trace_access_lock(cpu_file);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002507 return p;
2508}
2509
2510static void s_stop(struct seq_file *m, void *p)
2511{
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002512 struct trace_iterator *iter = m->private;
2513
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002514#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002515 if (iter->snapshot && iter->trace->use_max_tr)
2516 return;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002517#endif
Hiraku Toyookadebdd572012-12-26 11:53:00 +09002518
2519 if (!iter->snapshot)
2520 atomic_dec(&trace_record_cmdline_disabled);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002521
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08002522 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08002523 trace_event_read_unlock();
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002524}
2525
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002526static void
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002527get_total_entries(struct trace_buffer *buf,
2528 unsigned long *total, unsigned long *entries)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002529{
2530 unsigned long count;
2531 int cpu;
2532
2533 *total = 0;
2534 *entries = 0;
2535
2536 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002537 count = ring_buffer_entries_cpu(buf->buffer, cpu);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002538 /*
2539 * If this buffer has skipped entries, then we hold all
2540 * entries for the trace and we need to ignore the
2541 * ones before the time stamp.
2542 */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002543 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2544 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002545 /* total is the same as the entries */
2546 *total += count;
2547 } else
2548 *total += count +
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002549 ring_buffer_overrun_cpu(buf->buffer, cpu);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002550 *entries += count;
2551 }
2552}
2553
Ingo Molnare309b412008-05-12 21:20:51 +02002554static void print_lat_help_header(struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002555{
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002556 seq_puts(m, "# _------=> CPU# \n"
2557 "# / _-----=> irqs-off \n"
2558 "# | / _----=> need-resched \n"
2559 "# || / _---=> hardirq/softirq \n"
2560 "# ||| / _--=> preempt-depth \n"
2561 "# |||| / delay \n"
2562 "# cmd pid ||||| time | caller \n"
2563 "# \\ / ||||| \\ | / \n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002564}
2565
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002566static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002567{
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002568 unsigned long total;
2569 unsigned long entries;
2570
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002571 get_total_entries(buf, &total, &entries);
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002572 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2573 entries, total, num_online_cpus());
2574 seq_puts(m, "#\n");
2575}
2576
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002577static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002578{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002579 print_event_info(buf, m);
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002580 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2581 "# | | | | |\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002582}
2583
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002584static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
Steven Rostedt77271ce2011-11-17 09:34:33 -05002585{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002586 print_event_info(buf, m);
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002587 seq_puts(m, "# _-----=> irqs-off\n"
2588 "# / _----=> need-resched\n"
2589 "# | / _---=> hardirq/softirq\n"
2590 "# || / _--=> preempt-depth\n"
2591 "# ||| / delay\n"
2592 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2593 "# | | | |||| | |\n");
Steven Rostedt77271ce2011-11-17 09:34:33 -05002594}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002595
Jiri Olsa62b915f2010-04-02 19:01:22 +02002596void
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002597print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2598{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002599 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002600 struct trace_buffer *buf = iter->trace_buffer;
2601 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04002602 struct tracer *type = iter->trace;
Steven Rostedt39eaf7e2011-11-17 10:35:16 -05002603 unsigned long entries;
2604 unsigned long total;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002605 const char *name = "preemption";
2606
Steven Rostedt (Red Hat)d840f712013-02-01 18:38:47 -05002607 name = type->name;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002608
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002609 get_total_entries(buf, &total, &entries);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002610
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002611 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002612 name, UTS_RELEASE);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002613 seq_puts(m, "# -----------------------------------"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002614 "---------------------------------\n");
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002615 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002616 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
Steven Rostedt57f50be2008-05-12 21:20:44 +02002617 nsecs_to_usecs(data->saved_latency),
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002618 entries,
Steven Rostedt4c11d7a2008-05-12 21:20:43 +02002619 total,
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002620 buf->cpu,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002621#if defined(CONFIG_PREEMPT_NONE)
2622 "server",
2623#elif defined(CONFIG_PREEMPT_VOLUNTARY)
2624 "desktop",
Steven Rostedtb5c21b42008-07-10 20:58:12 -04002625#elif defined(CONFIG_PREEMPT)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002626 "preempt",
2627#else
2628 "unknown",
2629#endif
2630 /* These are reserved for later use */
2631 0, 0, 0, 0);
2632#ifdef CONFIG_SMP
2633 seq_printf(m, " #P:%d)\n", num_online_cpus());
2634#else
2635 seq_puts(m, ")\n");
2636#endif
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002637 seq_puts(m, "# -----------------\n");
2638 seq_printf(m, "# | task: %.16s-%d "
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002639 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
Eric W. Biedermand20b92a2012-03-13 16:02:19 -07002640 data->comm, data->pid,
2641 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002642 data->policy, data->rt_priority);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002643 seq_puts(m, "# -----------------\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002644
2645 if (data->critical_start) {
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002646 seq_puts(m, "# => started at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002647 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2648 trace_print_seq(m, &iter->seq);
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002649 seq_puts(m, "\n# => ended at: ");
Steven Rostedt214023c2008-05-12 21:20:46 +02002650 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2651 trace_print_seq(m, &iter->seq);
Steven Rostedt8248ac02009-09-02 12:27:41 -04002652 seq_puts(m, "\n#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002653 }
2654
KOSAKI Motohiro888b55d2009-03-08 13:12:43 +09002655 seq_puts(m, "#\n");
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002656}
2657
Steven Rostedta3097202008-11-07 22:36:02 -05002658static void test_cpu_buff_start(struct trace_iterator *iter)
2659{
2660 struct trace_seq *s = &iter->seq;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002661 struct trace_array *tr = iter->tr;
Steven Rostedta3097202008-11-07 22:36:02 -05002662
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002663 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
Steven Rostedt12ef7d42008-11-12 17:52:38 -05002664 return;
2665
2666 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2667 return;
2668
Rusty Russell44623442009-01-01 10:12:23 +10302669 if (cpumask_test_cpu(iter->cpu, iter->started))
Steven Rostedta3097202008-11-07 22:36:02 -05002670 return;
2671
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002672 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04002673 return;
2674
Rusty Russell44623442009-01-01 10:12:23 +10302675 cpumask_set_cpu(iter->cpu, iter->started);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02002676
2677 /* Don't print started cpu buffer for the first entry of the trace */
2678 if (iter->idx > 1)
2679 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2680 iter->cpu);
Steven Rostedta3097202008-11-07 22:36:02 -05002681}
2682
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002683static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002684{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002685 struct trace_array *tr = iter->tr;
Steven Rostedt214023c2008-05-12 21:20:46 +02002686 struct trace_seq *s = &iter->seq;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002687 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002688 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002689 struct trace_event *event;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002690
Ingo Molnar4e3c3332008-05-12 21:20:45 +02002691 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002692
Steven Rostedta3097202008-11-07 22:36:02 -05002693 test_cpu_buff_start(iter);
2694
Steven Rostedtf633cef2008-12-23 23:24:13 -05002695 event = ftrace_find_event(entry->type);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002696
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002697 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002698 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2699 trace_print_lat_context(iter);
2700 else
2701 trace_print_context(iter);
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002702 }
2703
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002704 if (trace_seq_has_overflowed(s))
2705 return TRACE_TYPE_PARTIAL_LINE;
2706
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002707 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002708 return event->funcs->trace(iter, sym_flags, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002709
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002710 trace_seq_printf(s, "Unknown type %d\n", entry->type);
Steven Rostedt7104f302008-10-01 10:52:51 -04002711
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002712 return trace_handle_return(s);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002713}
2714
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002715static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002716{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002717 struct trace_array *tr = iter->tr;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002718 struct trace_seq *s = &iter->seq;
2719 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002720 struct trace_event *event;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002721
2722 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002723
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002724 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002725 trace_seq_printf(s, "%d %d %llu ",
2726 entry->pid, iter->cpu, iter->ts);
2727
2728 if (trace_seq_has_overflowed(s))
2729 return TRACE_TYPE_PARTIAL_LINE;
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002730
Steven Rostedtf633cef2008-12-23 23:24:13 -05002731 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002732 if (event)
Steven Rostedta9a57762010-04-22 18:46:14 -04002733 return event->funcs->raw(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002734
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002735 trace_seq_printf(s, "%d ?\n", entry->type);
Steven Rostedt7104f302008-10-01 10:52:51 -04002736
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002737 return trace_handle_return(s);
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002738}
2739
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002740static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002741{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002742 struct trace_array *tr = iter->tr;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002743 struct trace_seq *s = &iter->seq;
2744 unsigned char newline = '\n';
2745 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002746 struct trace_event *event;
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002747
2748 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002749
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002750 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002751 SEQ_PUT_HEX_FIELD(s, entry->pid);
2752 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2753 SEQ_PUT_HEX_FIELD(s, iter->ts);
2754 if (trace_seq_has_overflowed(s))
2755 return TRACE_TYPE_PARTIAL_LINE;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002756 }
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002757
Steven Rostedtf633cef2008-12-23 23:24:13 -05002758 event = ftrace_find_event(entry->type);
Arnaldo Carvalho de Melo268ccda2009-02-04 20:16:39 -02002759 if (event) {
Steven Rostedta9a57762010-04-22 18:46:14 -04002760 enum print_line_t ret = event->funcs->hex(iter, 0, event);
Arnaldo Carvalho de Melod9793bd2009-02-03 20:20:41 -02002761 if (ret != TRACE_TYPE_HANDLED)
2762 return ret;
2763 }
Steven Rostedt7104f302008-10-01 10:52:51 -04002764
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002765 SEQ_PUT_FIELD(s, newline);
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002766
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002767 return trace_handle_return(s);
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002768}
2769
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002770static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002771{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002772 struct trace_array *tr = iter->tr;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002773 struct trace_seq *s = &iter->seq;
2774 struct trace_entry *entry;
Steven Rostedtf633cef2008-12-23 23:24:13 -05002775 struct trace_event *event;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002776
2777 entry = iter->ent;
Steven Rostedtdd0e5452008-08-01 12:26:41 -04002778
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002779 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002780 SEQ_PUT_FIELD(s, entry->pid);
2781 SEQ_PUT_FIELD(s, iter->cpu);
2782 SEQ_PUT_FIELD(s, iter->ts);
2783 if (trace_seq_has_overflowed(s))
2784 return TRACE_TYPE_PARTIAL_LINE;
Frederic Weisbeckerc4a8e8b2009-02-02 20:29:21 -02002785 }
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002786
Steven Rostedtf633cef2008-12-23 23:24:13 -05002787 event = ftrace_find_event(entry->type);
Steven Rostedta9a57762010-04-22 18:46:14 -04002788 return event ? event->funcs->binary(iter, 0, event) :
2789 TRACE_TYPE_HANDLED;
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002790}
2791
Jiri Olsa62b915f2010-04-02 19:01:22 +02002792int trace_empty(struct trace_iterator *iter)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002793{
Steven Rostedt6d158a82012-06-27 20:46:14 -04002794 struct ring_buffer_iter *buf_iter;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002795 int cpu;
2796
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002797 /* If we are looking at one CPU buffer, only check that one */
Steven Rostedtae3b5092013-01-23 15:22:59 -05002798 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002799 cpu = iter->cpu_file;
Steven Rostedt6d158a82012-06-27 20:46:14 -04002800 buf_iter = trace_buffer_iter(iter, cpu);
2801 if (buf_iter) {
2802 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002803 return 0;
2804 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002805 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
Steven Rostedt9aba60f2009-03-11 19:52:30 -04002806 return 0;
2807 }
2808 return 1;
2809 }
2810
Steven Rostedtab464282008-05-12 21:21:00 +02002811 for_each_tracing_cpu(cpu) {
Steven Rostedt6d158a82012-06-27 20:46:14 -04002812 buf_iter = trace_buffer_iter(iter, cpu);
2813 if (buf_iter) {
2814 if (!ring_buffer_iter_empty(buf_iter))
Steven Rostedtd7690412008-10-01 00:29:53 -04002815 return 0;
2816 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002817 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
Steven Rostedtd7690412008-10-01 00:29:53 -04002818 return 0;
2819 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002820 }
Steven Rostedtd7690412008-10-01 00:29:53 -04002821
Frederic Weisbecker797d3712008-09-30 18:13:45 +02002822 return 1;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002823}
2824
Lai Jiangshan4f535962009-05-18 19:35:34 +08002825/* Called with trace_event_read_lock() held. */
Jason Wessel955b61e2010-08-05 09:22:23 -05002826enum print_line_t print_trace_line(struct trace_iterator *iter)
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002827{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002828 struct trace_array *tr = iter->tr;
2829 unsigned long trace_flags = tr->trace_flags;
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002830 enum print_line_t ret;
2831
Steven Rostedt (Red Hat)19a7fe22014-11-12 10:29:54 -05002832 if (iter->lost_events) {
2833 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2834 iter->cpu, iter->lost_events);
2835 if (trace_seq_has_overflowed(&iter->seq))
2836 return TRACE_TYPE_PARTIAL_LINE;
2837 }
Steven Rostedtbc21b472010-03-31 19:49:26 -04002838
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02002839 if (iter->trace && iter->trace->print_line) {
2840 ret = iter->trace->print_line(iter);
2841 if (ret != TRACE_TYPE_UNHANDLED)
2842 return ret;
2843 }
Thomas Gleixner72829bc2008-05-23 21:37:28 +02002844
Steven Rostedt (Red Hat)09ae7232013-03-08 21:02:34 -05002845 if (iter->ent->type == TRACE_BPUTS &&
2846 trace_flags & TRACE_ITER_PRINTK &&
2847 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2848 return trace_print_bputs_msg_only(iter);
2849
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002850 if (iter->ent->type == TRACE_BPRINT &&
2851 trace_flags & TRACE_ITER_PRINTK &&
2852 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002853 return trace_print_bprintk_msg_only(iter);
Frederic Weisbecker48ead022009-03-12 18:24:49 +01002854
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002855 if (iter->ent->type == TRACE_PRINT &&
2856 trace_flags & TRACE_ITER_PRINTK &&
2857 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
Steven Rostedt5ef841f2009-03-19 12:20:38 -04002858 return trace_print_printk_msg_only(iter);
Frederic Weisbecker66896a82008-12-13 20:18:13 +01002859
Ingo Molnarcb0f12a2008-05-12 21:20:47 +02002860 if (trace_flags & TRACE_ITER_BIN)
2861 return print_bin_fmt(iter);
2862
Ingo Molnar5e3ca0e2008-05-12 21:20:49 +02002863 if (trace_flags & TRACE_ITER_HEX)
2864 return print_hex_fmt(iter);
2865
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002866 if (trace_flags & TRACE_ITER_RAW)
2867 return print_raw_fmt(iter);
2868
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002869 return print_trace_fmt(iter);
2870}
2871
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002872void trace_latency_header(struct seq_file *m)
2873{
2874 struct trace_iterator *iter = m->private;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002875 struct trace_array *tr = iter->tr;
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002876
2877 /* print nothing if the buffers are empty */
2878 if (trace_empty(iter))
2879 return;
2880
2881 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2882 print_trace_header(m, iter);
2883
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002884 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
Jiri Olsa7e9a49e2011-11-07 16:08:49 +01002885 print_lat_help_header(m);
2886}
2887
Jiri Olsa62b915f2010-04-02 19:01:22 +02002888void trace_default_header(struct seq_file *m)
2889{
2890 struct trace_iterator *iter = m->private;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04002891 struct trace_array *tr = iter->tr;
2892 unsigned long trace_flags = tr->trace_flags;
Jiri Olsa62b915f2010-04-02 19:01:22 +02002893
Jiri Olsaf56e7f82011-06-03 16:58:49 +02002894 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2895 return;
2896
Jiri Olsa62b915f2010-04-02 19:01:22 +02002897 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2898 /* print nothing if the buffers are empty */
2899 if (trace_empty(iter))
2900 return;
2901 print_trace_header(m, iter);
2902 if (!(trace_flags & TRACE_ITER_VERBOSE))
2903 print_lat_help_header(m);
2904 } else {
Steven Rostedt77271ce2011-11-17 09:34:33 -05002905 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2906 if (trace_flags & TRACE_ITER_IRQ_INFO)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002907 print_func_help_header_irq(iter->trace_buffer, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002908 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05002909 print_func_help_header(iter->trace_buffer, m);
Steven Rostedt77271ce2011-11-17 09:34:33 -05002910 }
Jiri Olsa62b915f2010-04-02 19:01:22 +02002911 }
2912}
2913
Steven Rostedte0a413f2011-09-29 21:26:16 -04002914static void test_ftrace_alive(struct seq_file *m)
2915{
2916 if (!ftrace_is_dead())
2917 return;
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002918 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2919 "# MAY BE MISSING FUNCTION EVENTS\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002920}
2921
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002922#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002923static void show_snapshot_main_help(struct seq_file *m)
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002924{
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002925 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2926 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2927 "# Takes a snapshot of the main buffer.\n"
2928 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2929 "# (Doesn't have to be '2' works with any number that\n"
2930 "# is not a '0' or '1')\n");
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002931}
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002932
2933static void show_snapshot_percpu_help(struct seq_file *m)
2934{
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002935 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002936#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002937 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2938 "# Takes a snapshot of the main buffer for this cpu.\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002939#else
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002940 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2941 "# Must use main snapshot file to allocate.\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002942#endif
Rasmus Villemoesd79ac282014-11-08 21:42:11 +01002943 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2944 "# (Doesn't have to be '2' works with any number that\n"
2945 "# is not a '0' or '1')\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002946}
2947
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002948static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2949{
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05002950 if (iter->tr->allocated_snapshot)
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002951 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002952 else
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002953 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002954
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01002955 seq_puts(m, "# Snapshot commands:\n");
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05002956 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2957 show_snapshot_main_help(m);
2958 else
2959 show_snapshot_percpu_help(m);
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002960}
2961#else
2962/* Should never be called */
2963static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2964#endif
2965
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002966static int s_show(struct seq_file *m, void *v)
2967{
2968 struct trace_iterator *iter = v;
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002969 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002970
2971 if (iter->ent == NULL) {
2972 if (iter->tr) {
2973 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2974 seq_puts(m, "#\n");
Steven Rostedte0a413f2011-09-29 21:26:16 -04002975 test_ftrace_alive(m);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002976 }
Steven Rostedt (Red Hat)d8741e22013-03-05 10:25:16 -05002977 if (iter->snapshot && trace_empty(iter))
2978 print_snapshot_help(m, iter);
2979 else if (iter->trace && iter->trace->print_header)
Markus Metzger8bba1bf2008-11-25 09:12:31 +01002980 iter->trace->print_header(m);
Jiri Olsa62b915f2010-04-02 19:01:22 +02002981 else
2982 trace_default_header(m);
2983
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002984 } else if (iter->leftover) {
2985 /*
2986 * If we filled the seq_file buffer earlier, we
2987 * want to just show it now.
2988 */
2989 ret = trace_print_seq(m, &iter->seq);
2990
2991 /* ret should this time be zero, but you never know */
2992 iter->leftover = ret;
2993
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02002994 } else {
Ingo Molnarf9896bf2008-05-12 21:20:47 +02002995 print_trace_line(iter);
Steven Rostedta63ce5b2009-12-07 09:11:39 -05002996 ret = trace_print_seq(m, &iter->seq);
2997 /*
2998 * If we overflow the seq_file buffer, then it will
2999 * ask us for this data again at start up.
3000 * Use that instead.
3001 * ret is 0 if seq_file write succeeded.
3002 * -1 otherwise.
3003 */
3004 iter->leftover = ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003005 }
3006
3007 return 0;
3008}
3009
Oleg Nesterov649e9c72013-07-23 17:25:54 +02003010/*
3011 * Should be used after trace_array_get(), trace_types_lock
3012 * ensures that i_cdev was already initialized.
3013 */
3014static inline int tracing_get_cpu(struct inode *inode)
3015{
3016 if (inode->i_cdev) /* See trace_create_cpu_file() */
3017 return (long)inode->i_cdev - 1;
3018 return RING_BUFFER_ALL_CPUS;
3019}
3020
James Morris88e9d342009-09-22 16:43:43 -07003021static const struct seq_operations tracer_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003022 .start = s_start,
3023 .next = s_next,
3024 .stop = s_stop,
3025 .show = s_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003026};
3027
Ingo Molnare309b412008-05-12 21:20:51 +02003028static struct trace_iterator *
Oleg Nesterov6484c712013-07-23 17:26:10 +02003029__tracing_open(struct inode *inode, struct file *file, bool snapshot)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003030{
Oleg Nesterov6484c712013-07-23 17:26:10 +02003031 struct trace_array *tr = inode->i_private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003032 struct trace_iterator *iter;
Jiri Olsa50e18b92012-04-25 10:23:39 +02003033 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003034
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003035 if (tracing_disabled)
3036 return ERR_PTR(-ENODEV);
Steven Rostedt60a11772008-05-12 21:20:44 +02003037
Jiri Olsa50e18b92012-04-25 10:23:39 +02003038 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003039 if (!iter)
3040 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003041
Gil Fruchter72917232015-06-09 10:32:35 +03003042 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
Steven Rostedt6d158a82012-06-27 20:46:14 -04003043 GFP_KERNEL);
Dan Carpenter93574fc2012-07-11 09:35:08 +03003044 if (!iter->buffer_iter)
3045 goto release;
3046
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003047 /*
3048 * We make a copy of the current tracer to avoid concurrent
3049 * changes on it while we are reading.
3050 */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003051 mutex_lock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003052 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003053 if (!iter->trace)
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003054 goto fail;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003055
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003056 *iter->trace = *tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003057
Li Zefan79f55992009-06-15 14:58:26 +08003058 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02003059 goto fail;
3060
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003061 iter->tr = tr;
3062
3063#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003064 /* Currently only the top directory has a snapshot */
3065 if (tr->current_trace->print_max || snapshot)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003066 iter->trace_buffer = &tr->max_buffer;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003067 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003068#endif
3069 iter->trace_buffer = &tr->trace_buffer;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003070 iter->snapshot = snapshot;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003071 iter->pos = -1;
Oleg Nesterov6484c712013-07-23 17:26:10 +02003072 iter->cpu_file = tracing_get_cpu(inode);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003073 mutex_init(&iter->mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003074
Markus Metzger8bba1bf2008-11-25 09:12:31 +01003075 /* Notify the tracer early; before we stop tracing. */
3076 if (iter->trace && iter->trace->open)
Markus Metzgera93751c2008-12-11 13:53:26 +01003077 iter->trace->open(iter);
Markus Metzger8bba1bf2008-11-25 09:12:31 +01003078
Steven Rostedt12ef7d42008-11-12 17:52:38 -05003079 /* Annotate start of buffers if we had overruns */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003080 if (ring_buffer_overruns(iter->trace_buffer->buffer))
Steven Rostedt12ef7d42008-11-12 17:52:38 -05003081 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3082
David Sharp8be07092012-11-13 12:18:22 -08003083 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09003084 if (trace_clocks[tr->clock_id].in_ns)
David Sharp8be07092012-11-13 12:18:22 -08003085 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3086
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003087 /* stop the trace while dumping if we are not opening "snapshot" */
3088 if (!iter->snapshot)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003089 tracing_stop_tr(tr);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04003090
Steven Rostedtae3b5092013-01-23 15:22:59 -05003091 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003092 for_each_tracing_cpu(cpu) {
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003093 iter->buffer_iter[cpu] =
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003094 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
David Miller72c9ddf2010-04-20 15:47:11 -07003095 }
3096 ring_buffer_read_prepare_sync();
3097 for_each_tracing_cpu(cpu) {
3098 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04003099 tracing_iter_reset(iter, cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01003100 }
3101 } else {
3102 cpu = iter->cpu_file;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003103 iter->buffer_iter[cpu] =
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003104 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
David Miller72c9ddf2010-04-20 15:47:11 -07003105 ring_buffer_read_prepare_sync();
3106 ring_buffer_read_start(iter->buffer_iter[cpu]);
Steven Rostedt2f26ebd2009-09-01 11:06:29 -04003107 tracing_iter_reset(iter, cpu);
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003108 }
3109
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003110 mutex_unlock(&trace_types_lock);
3111
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003112 return iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003113
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003114 fail:
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003115 mutex_unlock(&trace_types_lock);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003116 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04003117 kfree(iter->buffer_iter);
Dan Carpenter93574fc2012-07-11 09:35:08 +03003118release:
Jiri Olsa50e18b92012-04-25 10:23:39 +02003119 seq_release_private(inode, file);
3120 return ERR_PTR(-ENOMEM);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003121}
3122
3123int tracing_open_generic(struct inode *inode, struct file *filp)
3124{
Steven Rostedt60a11772008-05-12 21:20:44 +02003125 if (tracing_disabled)
3126 return -ENODEV;
3127
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003128 filp->private_data = inode->i_private;
3129 return 0;
3130}
3131
Geyslan G. Bem2e864212013-10-18 21:15:54 -03003132bool tracing_is_disabled(void)
3133{
3134 return (tracing_disabled) ? true: false;
3135}
3136
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003137/*
3138 * Open and update trace_array ref count.
3139 * Must have the current trace_array passed to it.
3140 */
Steven Rostedt (Red Hat)dcc30222013-07-02 20:30:52 -04003141static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003142{
3143 struct trace_array *tr = inode->i_private;
3144
3145 if (tracing_disabled)
3146 return -ENODEV;
3147
3148 if (trace_array_get(tr) < 0)
3149 return -ENODEV;
3150
3151 filp->private_data = inode->i_private;
3152
3153 return 0;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003154}
3155
Hannes Eder4fd27352009-02-10 19:44:12 +01003156static int tracing_release(struct inode *inode, struct file *file)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003157{
Oleg Nesterov6484c712013-07-23 17:26:10 +02003158 struct trace_array *tr = inode->i_private;
matt mooney907f2782010-09-27 19:04:53 -07003159 struct seq_file *m = file->private_data;
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003160 struct trace_iterator *iter;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003161 int cpu;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003162
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003163 if (!(file->f_mode & FMODE_READ)) {
Oleg Nesterov6484c712013-07-23 17:26:10 +02003164 trace_array_put(tr);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003165 return 0;
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003166 }
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003167
Oleg Nesterov6484c712013-07-23 17:26:10 +02003168 /* Writes do not use seq_file */
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003169 iter = m->private;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003170 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05003171
Steven Rostedt3928a8a2008-09-29 23:02:41 -04003172 for_each_tracing_cpu(cpu) {
3173 if (iter->buffer_iter[cpu])
3174 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3175 }
3176
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003177 if (iter->trace && iter->trace->close)
3178 iter->trace->close(iter);
3179
Hiraku Toyookadebdd572012-12-26 11:53:00 +09003180 if (!iter->snapshot)
3181 /* reenable tracing if it was previously enabled */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003182 tracing_start_tr(tr);
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003183
3184 __trace_array_put(tr);
3185
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003186 mutex_unlock(&trace_types_lock);
3187
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003188 mutex_destroy(&iter->mutex);
Frederic Weisbeckerb0dfa972009-04-01 22:53:08 +02003189 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01003190 kfree(iter->trace);
Steven Rostedt6d158a82012-06-27 20:46:14 -04003191 kfree(iter->buffer_iter);
Jiri Olsa50e18b92012-04-25 10:23:39 +02003192 seq_release_private(inode, file);
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003193
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003194 return 0;
3195}
3196
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003197static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3198{
3199 struct trace_array *tr = inode->i_private;
3200
3201 trace_array_put(tr);
3202 return 0;
3203}
3204
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003205static int tracing_single_release_tr(struct inode *inode, struct file *file)
3206{
3207 struct trace_array *tr = inode->i_private;
3208
3209 trace_array_put(tr);
3210
3211 return single_release(inode, file);
3212}
3213
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003214static int tracing_open(struct inode *inode, struct file *file)
3215{
Oleg Nesterov6484c712013-07-23 17:26:10 +02003216 struct trace_array *tr = inode->i_private;
Steven Rostedt85a2f9b2009-02-27 00:12:38 -05003217 struct trace_iterator *iter;
3218 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003219
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003220 if (trace_array_get(tr) < 0)
3221 return -ENODEV;
3222
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003223 /* If this file was open for write, then erase contents */
Oleg Nesterov6484c712013-07-23 17:26:10 +02003224 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3225 int cpu = tracing_get_cpu(inode);
3226
3227 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003228 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003229 else
Oleg Nesterov6484c712013-07-23 17:26:10 +02003230 tracing_reset(&tr->trace_buffer, cpu);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003231 }
3232
3233 if (file->f_mode & FMODE_READ) {
Oleg Nesterov6484c712013-07-23 17:26:10 +02003234 iter = __tracing_open(inode, file, false);
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003235 if (IS_ERR(iter))
3236 ret = PTR_ERR(iter);
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003237 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003238 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3239 }
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04003240
3241 if (ret < 0)
3242 trace_array_put(tr);
3243
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003244 return ret;
3245}
3246
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003247/*
3248 * Some tracers are not suitable for instance buffers.
3249 * A tracer is always available for the global array (toplevel)
3250 * or if it explicitly states that it is.
3251 */
3252static bool
3253trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3254{
3255 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3256}
3257
3258/* Find the next tracer that this trace array may use */
3259static struct tracer *
3260get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3261{
3262 while (t && !trace_ok_for_array(t, tr))
3263 t = t->next;
3264
3265 return t;
3266}
3267
Ingo Molnare309b412008-05-12 21:20:51 +02003268static void *
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003269t_next(struct seq_file *m, void *v, loff_t *pos)
3270{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003271 struct trace_array *tr = m->private;
Li Zefanf129e962009-06-24 09:53:44 +08003272 struct tracer *t = v;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003273
3274 (*pos)++;
3275
3276 if (t)
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003277 t = get_tracer_for_array(tr, t->next);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003278
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003279 return t;
3280}
3281
3282static void *t_start(struct seq_file *m, loff_t *pos)
3283{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003284 struct trace_array *tr = m->private;
Li Zefanf129e962009-06-24 09:53:44 +08003285 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003286 loff_t l = 0;
3287
3288 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003289
3290 t = get_tracer_for_array(tr, trace_types);
3291 for (; t && l < *pos; t = t_next(m, t, &l))
3292 ;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003293
3294 return t;
3295}
3296
3297static void t_stop(struct seq_file *m, void *p)
3298{
3299 mutex_unlock(&trace_types_lock);
3300}
3301
3302static int t_show(struct seq_file *m, void *v)
3303{
3304 struct tracer *t = v;
3305
3306 if (!t)
3307 return 0;
3308
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01003309 seq_puts(m, t->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003310 if (t->next)
3311 seq_putc(m, ' ');
3312 else
3313 seq_putc(m, '\n');
3314
3315 return 0;
3316}
3317
James Morris88e9d342009-09-22 16:43:43 -07003318static const struct seq_operations show_traces_seq_ops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003319 .start = t_start,
3320 .next = t_next,
3321 .stop = t_stop,
3322 .show = t_show,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003323};
3324
3325static int show_traces_open(struct inode *inode, struct file *file)
3326{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003327 struct trace_array *tr = inode->i_private;
3328 struct seq_file *m;
3329 int ret;
3330
Steven Rostedt60a11772008-05-12 21:20:44 +02003331 if (tracing_disabled)
3332 return -ENODEV;
3333
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05003334 ret = seq_open(file, &show_traces_seq_ops);
3335 if (ret)
3336 return ret;
3337
3338 m = file->private_data;
3339 m->private = tr;
3340
3341 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003342}
3343
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003344static ssize_t
3345tracing_write_stub(struct file *filp, const char __user *ubuf,
3346 size_t count, loff_t *ppos)
3347{
3348 return count;
3349}
3350
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003351loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
Slava Pestov364829b2010-11-24 15:13:16 -08003352{
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003353 int ret;
3354
Slava Pestov364829b2010-11-24 15:13:16 -08003355 if (file->f_mode & FMODE_READ)
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003356 ret = seq_lseek(file, offset, whence);
Slava Pestov364829b2010-11-24 15:13:16 -08003357 else
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003358 file->f_pos = ret = 0;
3359
3360 return ret;
Slava Pestov364829b2010-11-24 15:13:16 -08003361}
3362
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003363static const struct file_operations tracing_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003364 .open = tracing_open,
3365 .read = seq_read,
Steven Rostedt4acd4d02009-03-18 10:40:24 -04003366 .write = tracing_write_stub,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05003367 .llseek = tracing_lseek,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02003368 .release = tracing_release,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003369};
3370
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003371static const struct file_operations show_traces_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003372 .open = show_traces_open,
3373 .read = seq_read,
3374 .release = seq_release,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003375 .llseek = seq_lseek,
Ingo Molnarc7078de2008-05-12 21:20:52 +02003376};
3377
Ingo Molnar36dfe922008-05-12 21:20:52 +02003378/*
Ingo Molnar36dfe922008-05-12 21:20:52 +02003379 * The tracer itself will not take this lock, but still we want
3380 * to provide a consistent cpumask to user-space:
3381 */
3382static DEFINE_MUTEX(tracing_cpumask_update_lock);
3383
3384/*
3385 * Temporary storage for the character representation of the
3386 * CPU bitmask (and one more byte for the newline):
3387 */
3388static char mask_str[NR_CPUS + 1];
3389
Ingo Molnarc7078de2008-05-12 21:20:52 +02003390static ssize_t
3391tracing_cpumask_read(struct file *filp, char __user *ubuf,
3392 size_t count, loff_t *ppos)
3393{
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003394 struct trace_array *tr = file_inode(filp)->i_private;
Ingo Molnar36dfe922008-05-12 21:20:52 +02003395 int len;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003396
3397 mutex_lock(&tracing_cpumask_update_lock);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003398
Tejun Heo1a402432015-02-13 14:37:39 -08003399 len = snprintf(mask_str, count, "%*pb\n",
3400 cpumask_pr_args(tr->tracing_cpumask));
3401 if (len >= count) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02003402 count = -EINVAL;
3403 goto out_err;
3404 }
Ingo Molnar36dfe922008-05-12 21:20:52 +02003405 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3406
3407out_err:
Ingo Molnarc7078de2008-05-12 21:20:52 +02003408 mutex_unlock(&tracing_cpumask_update_lock);
3409
3410 return count;
3411}
3412
3413static ssize_t
3414tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3415 size_t count, loff_t *ppos)
3416{
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003417 struct trace_array *tr = file_inode(filp)->i_private;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303418 cpumask_var_t tracing_cpumask_new;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003419 int err, cpu;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303420
3421 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3422 return -ENOMEM;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003423
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303424 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003425 if (err)
3426 goto err_unlock;
3427
Li Zefan215368e2009-06-15 10:56:42 +08003428 mutex_lock(&tracing_cpumask_update_lock);
3429
Steven Rostedta5e25882008-12-02 15:34:05 -05003430 local_irq_disable();
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05003431 arch_spin_lock(&tr->max_lock);
Steven Rostedtab464282008-05-12 21:21:00 +02003432 for_each_tracing_cpu(cpu) {
Ingo Molnar36dfe922008-05-12 21:20:52 +02003433 /*
3434 * Increase/decrease the disabled counter if we are
3435 * about to flip a bit in the cpumask:
3436 */
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003437 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303438 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003439 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3440 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003441 }
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003442 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303443 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003444 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3445 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003446 }
3447 }
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05003448 arch_spin_unlock(&tr->max_lock);
Steven Rostedta5e25882008-12-02 15:34:05 -05003449 local_irq_enable();
Ingo Molnar36dfe922008-05-12 21:20:52 +02003450
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003451 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003452
Ingo Molnarc7078de2008-05-12 21:20:52 +02003453 mutex_unlock(&tracing_cpumask_update_lock);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10303454 free_cpumask_var(tracing_cpumask_new);
Ingo Molnarc7078de2008-05-12 21:20:52 +02003455
Ingo Molnarc7078de2008-05-12 21:20:52 +02003456 return count;
Ingo Molnar36dfe922008-05-12 21:20:52 +02003457
3458err_unlock:
Li Zefan215368e2009-06-15 10:56:42 +08003459 free_cpumask_var(tracing_cpumask_new);
Ingo Molnar36dfe922008-05-12 21:20:52 +02003460
3461 return err;
Ingo Molnarc7078de2008-05-12 21:20:52 +02003462}
3463
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003464static const struct file_operations tracing_cpumask_fops = {
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003465 .open = tracing_open_generic_tr,
Ingo Molnarc7078de2008-05-12 21:20:52 +02003466 .read = tracing_cpumask_read,
3467 .write = tracing_cpumask_write,
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07003468 .release = tracing_release_generic_tr,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003469 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003470};
3471
Li Zefanfdb372e2009-12-08 11:15:59 +08003472static int tracing_trace_options_show(struct seq_file *m, void *v)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003473{
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003474 struct tracer_opt *trace_opts;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003475 struct trace_array *tr = m->private;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003476 u32 tracer_flags;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003477 int i;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003478
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003479 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003480 tracer_flags = tr->current_trace->flags->val;
3481 trace_opts = tr->current_trace->flags->opts;
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003482
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003483 for (i = 0; trace_options[i]; i++) {
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003484 if (tr->trace_flags & (1 << i))
Li Zefanfdb372e2009-12-08 11:15:59 +08003485 seq_printf(m, "%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003486 else
Li Zefanfdb372e2009-12-08 11:15:59 +08003487 seq_printf(m, "no%s\n", trace_options[i]);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003488 }
3489
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003490 for (i = 0; trace_opts[i].name; i++) {
3491 if (tracer_flags & trace_opts[i].bit)
Li Zefanfdb372e2009-12-08 11:15:59 +08003492 seq_printf(m, "%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003493 else
Li Zefanfdb372e2009-12-08 11:15:59 +08003494 seq_printf(m, "no%s\n", trace_opts[i].name);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003495 }
Steven Rostedtd8e83d22009-02-26 23:55:58 -05003496 mutex_unlock(&trace_types_lock);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003497
Li Zefanfdb372e2009-12-08 11:15:59 +08003498 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003499}
3500
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003501static int __set_tracer_option(struct trace_array *tr,
Li Zefan8d18eaa2009-12-08 11:17:06 +08003502 struct tracer_flags *tracer_flags,
3503 struct tracer_opt *opts, int neg)
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003504{
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003505 struct tracer *trace = tr->current_trace;
Li Zefan8d18eaa2009-12-08 11:17:06 +08003506 int ret;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003507
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003508 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003509 if (ret)
3510 return ret;
3511
3512 if (neg)
Zhaolei77708412009-08-07 18:53:21 +08003513 tracer_flags->val &= ~opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003514 else
Zhaolei77708412009-08-07 18:53:21 +08003515 tracer_flags->val |= opts->bit;
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003516 return 0;
3517}
3518
Li Zefan8d18eaa2009-12-08 11:17:06 +08003519/* Try to assign a tracer specific option */
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003520static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
Li Zefan8d18eaa2009-12-08 11:17:06 +08003521{
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003522 struct tracer *trace = tr->current_trace;
Li Zefan8d18eaa2009-12-08 11:17:06 +08003523 struct tracer_flags *tracer_flags = trace->flags;
3524 struct tracer_opt *opts = NULL;
3525 int i;
3526
3527 for (i = 0; tracer_flags->opts[i].name; i++) {
3528 opts = &tracer_flags->opts[i];
3529
3530 if (strcmp(cmp, opts->name) == 0)
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003531 return __set_tracer_option(tr, trace->flags, opts, neg);
Li Zefan8d18eaa2009-12-08 11:17:06 +08003532 }
3533
3534 return -EINVAL;
3535}
3536
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003537/* Some tracers require overwrite to stay enabled */
3538int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3539{
3540 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3541 return -1;
3542
3543 return 0;
3544}
3545
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003546int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003547{
3548 /* do nothing if flag is already set */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003549 if (!!(tr->trace_flags & mask) == !!enabled)
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003550 return 0;
3551
3552 /* Give the tracer a chance to approve the change */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003553 if (tr->current_trace->flag_changed)
Steven Rostedt (Red Hat)bf6065b2014-01-10 17:51:01 -05003554 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003555 return -EINVAL;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003556
3557 if (enabled)
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003558 tr->trace_flags |= mask;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003559 else
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04003560 tr->trace_flags &= ~mask;
Li Zefane870e9a2010-07-02 11:07:32 +08003561
3562 if (mask == TRACE_ITER_RECORD_CMD)
3563 trace_event_enable_cmd_record(enabled);
David Sharp750912f2010-12-08 13:46:47 -08003564
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003565 if (mask == TRACE_ITER_OVERWRITE) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003566 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003567#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05003568 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
Steven Rostedt (Red Hat)80902822013-03-14 14:20:54 -04003569#endif
3570 }
Steven Rostedt81698832012-10-11 10:15:05 -04003571
Steven Rostedt (Red Hat)b9f91082015-09-29 18:21:35 -04003572 if (mask == TRACE_ITER_PRINTK) {
Steven Rostedt81698832012-10-11 10:15:05 -04003573 trace_printk_start_stop_comm(enabled);
Steven Rostedt (Red Hat)b9f91082015-09-29 18:21:35 -04003574 trace_printk_control(enabled);
3575 }
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003576
3577 return 0;
Steven Rostedtaf4617b2009-03-17 18:09:55 -04003578}
3579
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003580static int trace_set_options(struct trace_array *tr, char *option)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003581{
Li Zefan8d18eaa2009-12-08 11:17:06 +08003582 char *cmp;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003583 int neg = 0;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003584 int ret = -ENODEV;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003585 int i;
3586
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003587 cmp = strstrip(option);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003588
Li Zefan8d18eaa2009-12-08 11:17:06 +08003589 if (strncmp(cmp, "no", 2) == 0) {
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003590 neg = 1;
3591 cmp += 2;
3592 }
3593
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003594 mutex_lock(&trace_types_lock);
3595
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003596 for (i = 0; trace_options[i]; i++) {
Li Zefan8d18eaa2009-12-08 11:17:06 +08003597 if (strcmp(cmp, trace_options[i]) == 0) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003598 ret = set_tracer_flag(tr, 1 << i, !neg);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003599 break;
3600 }
3601 }
Frederic Weisbeckeradf9f192008-11-17 19:23:42 +01003602
3603 /* If no option could be set, test the specific tracer options */
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003604 if (!trace_options[i])
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05003605 ret = set_tracer_option(tr, cmp, neg);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04003606
3607 mutex_unlock(&trace_types_lock);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003608
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003609 return ret;
3610}
3611
3612static ssize_t
3613tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3614 size_t cnt, loff_t *ppos)
3615{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003616 struct seq_file *m = filp->private_data;
3617 struct trace_array *tr = m->private;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003618 char buf[64];
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003619 int ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003620
3621 if (cnt >= sizeof(buf))
3622 return -EINVAL;
3623
3624 if (copy_from_user(&buf, ubuf, cnt))
3625 return -EFAULT;
3626
Steven Rostedta8dd2172013-01-09 20:54:17 -05003627 buf[cnt] = 0;
3628
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003629 ret = trace_set_options(tr, buf);
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04003630 if (ret < 0)
3631 return ret;
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04003632
Jiri Olsacf8517c2009-10-23 19:36:16 -04003633 *ppos += cnt;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003634
3635 return cnt;
3636}
3637
Li Zefanfdb372e2009-12-08 11:15:59 +08003638static int tracing_trace_options_open(struct inode *inode, struct file *file)
3639{
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003640 struct trace_array *tr = inode->i_private;
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003641 int ret;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003642
Li Zefanfdb372e2009-12-08 11:15:59 +08003643 if (tracing_disabled)
3644 return -ENODEV;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04003645
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003646 if (trace_array_get(tr) < 0)
3647 return -ENODEV;
3648
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07003649 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3650 if (ret < 0)
3651 trace_array_put(tr);
3652
3653 return ret;
Li Zefanfdb372e2009-12-08 11:15:59 +08003654}
3655
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003656static const struct file_operations tracing_iter_fops = {
Li Zefanfdb372e2009-12-08 11:15:59 +08003657 .open = tracing_trace_options_open,
3658 .read = seq_read,
3659 .llseek = seq_lseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04003660 .release = tracing_single_release_tr,
Steven Rostedtee6bce52008-11-12 17:52:37 -05003661 .write = tracing_trace_options_write,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02003662};
3663
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003664static const char readme_msg[] =
3665 "tracing mini-HOWTO:\n\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003666 "# echo 0 > tracing_on : quick way to disable tracing\n"
3667 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3668 " Important files:\n"
3669 " trace\t\t\t- The static contents of the buffer\n"
3670 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3671 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3672 " current_tracer\t- function and latency tracers\n"
3673 " available_tracers\t- list of configured tracers for current_tracer\n"
3674 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3675 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3676 " trace_clock\t\t-change the clock used to order events\n"
3677 " local: Per cpu clock but may not be synced across CPUs\n"
3678 " global: Synced across CPUs but slows tracing down.\n"
3679 " counter: Not a clock, but just an increment\n"
3680 " uptime: Jiffy counter from time of boot\n"
3681 " perf: Same clock that perf events use\n"
3682#ifdef CONFIG_X86_64
3683 " x86-tsc: TSC cycle counter\n"
3684#endif
3685 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3686 " tracing_cpumask\t- Limit which CPUs to trace\n"
3687 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3688 "\t\t\t Remove sub-buffer with rmdir\n"
3689 " trace_options\t\t- Set format or modify how tracing happens\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003690 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3691 "\t\t\t option name\n"
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003692 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003693#ifdef CONFIG_DYNAMIC_FTRACE
3694 "\n available_filter_functions - list of functions that can be filtered on\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003695 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3696 "\t\t\t functions\n"
3697 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3698 "\t modules: Can select a group via module\n"
3699 "\t Format: :mod:<module-name>\n"
3700 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3701 "\t triggers: a command to perform when function is hit\n"
3702 "\t Format: <function>:<trigger>[:count]\n"
3703 "\t trigger: traceon, traceoff\n"
3704 "\t\t enable_event:<system>:<event>\n"
3705 "\t\t disable_event:<system>:<event>\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003706#ifdef CONFIG_STACKTRACE
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003707 "\t\t stacktrace\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003708#endif
3709#ifdef CONFIG_TRACER_SNAPSHOT
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003710 "\t\t snapshot\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003711#endif
Steven Rostedt (Red Hat)17a280e2014-04-10 22:43:37 -04003712 "\t\t dump\n"
3713 "\t\t cpudump\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003714 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3715 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3716 "\t The first one will disable tracing every time do_fault is hit\n"
3717 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3718 "\t The first time do trap is hit and it disables tracing, the\n"
3719 "\t counter will decrement to 2. If tracing is already disabled,\n"
3720 "\t the counter will not decrement. It only decrements when the\n"
3721 "\t trigger did work\n"
3722 "\t To remove trigger without count:\n"
3723 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3724 "\t To remove trigger with a count:\n"
3725 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003726 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003727 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3728 "\t modules: Can select a group via module command :mod:\n"
3729 "\t Does not accept triggers\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003730#endif /* CONFIG_DYNAMIC_FTRACE */
3731#ifdef CONFIG_FUNCTION_TRACER
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003732 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3733 "\t\t (function)\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003734#endif
3735#ifdef CONFIG_FUNCTION_GRAPH_TRACER
3736 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
Namhyung Kimd048a8c72014-06-13 01:23:53 +09003737 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003738 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3739#endif
3740#ifdef CONFIG_TRACER_SNAPSHOT
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003741 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3742 "\t\t\t snapshot buffer. Read the contents for more\n"
3743 "\t\t\t information\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003744#endif
zhangwei(Jovi)991821c2013-07-15 16:32:34 +08003745#ifdef CONFIG_STACK_TRACER
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003746 " stack_trace\t\t- Shows the max stack trace when active\n"
3747 " stack_max_size\t- Shows current max stack size that was traced\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003748 "\t\t\t Write into this file to reset the max size (trigger a\n"
3749 "\t\t\t new trace)\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003750#ifdef CONFIG_DYNAMIC_FTRACE
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003751 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3752 "\t\t\t traces\n"
Steven Rostedt (Red Hat)22f45642013-03-15 17:23:20 -04003753#endif
zhangwei(Jovi)991821c2013-07-15 16:32:34 +08003754#endif /* CONFIG_STACK_TRACER */
Tom Zanussi26f25562014-01-17 15:11:44 -06003755 " events/\t\t- Directory containing all trace event subsystems:\n"
3756 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3757 " events/<system>/\t- Directory containing all trace events for <system>:\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003758 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3759 "\t\t\t events\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003760 " filter\t\t- If set, only events passing filter are traced\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003761 " events/<system>/<event>/\t- Directory containing control files for\n"
3762 "\t\t\t <event>:\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003763 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3764 " filter\t\t- If set, only events passing filter are traced\n"
3765 " trigger\t\t- If set, a command to perform when event is hit\n"
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003766 "\t Format: <trigger>[:count][if <filter>]\n"
3767 "\t trigger: traceon, traceoff\n"
3768 "\t enable_event:<system>:<event>\n"
3769 "\t disable_event:<system>:<event>\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003770#ifdef CONFIG_STACKTRACE
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003771 "\t\t stacktrace\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003772#endif
3773#ifdef CONFIG_TRACER_SNAPSHOT
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003774 "\t\t snapshot\n"
Tom Zanussi26f25562014-01-17 15:11:44 -06003775#endif
Steven Rostedt (Red Hat)71485c42014-01-23 00:10:04 -05003776 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3777 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3778 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3779 "\t events/block/block_unplug/trigger\n"
3780 "\t The first disables tracing every time block_unplug is hit.\n"
3781 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3782 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3783 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3784 "\t Like function triggers, the counter is only decremented if it\n"
3785 "\t enabled or disabled tracing.\n"
3786 "\t To remove a trigger without a count:\n"
3787 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3788 "\t To remove a trigger with a count:\n"
3789 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3790 "\t Filters can be ignored when removing a trigger.\n"
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003791;
3792
3793static ssize_t
3794tracing_readme_read(struct file *filp, char __user *ubuf,
3795 size_t cnt, loff_t *ppos)
3796{
3797 return simple_read_from_buffer(ubuf, cnt, ppos,
3798 readme_msg, strlen(readme_msg));
3799}
3800
Steven Rostedt5e2336a2009-03-05 21:44:55 -05003801static const struct file_operations tracing_readme_fops = {
Ingo Molnarc7078de2008-05-12 21:20:52 +02003802 .open = tracing_open_generic,
3803 .read = tracing_readme_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02003804 .llseek = generic_file_llseek,
Ingo Molnar7bd2f242008-05-12 21:20:45 +02003805};
3806
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003807static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
Avadh Patel69abe6a2009-04-10 16:04:48 -04003808{
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003809 unsigned int *ptr = v;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003810
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003811 if (*pos || m->count)
3812 ptr++;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003813
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003814 (*pos)++;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003815
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003816 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3817 ptr++) {
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003818 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
Avadh Patel69abe6a2009-04-10 16:04:48 -04003819 continue;
3820
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003821 return ptr;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003822 }
3823
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003824 return NULL;
3825}
Avadh Patel69abe6a2009-04-10 16:04:48 -04003826
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003827static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3828{
3829 void *v;
3830 loff_t l = 0;
Avadh Patel69abe6a2009-04-10 16:04:48 -04003831
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04003832 preempt_disable();
3833 arch_spin_lock(&trace_cmdline_lock);
3834
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003835 v = &savedcmd->map_cmdline_to_pid[0];
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003836 while (l <= *pos) {
3837 v = saved_cmdlines_next(m, v, &l);
3838 if (!v)
3839 return NULL;
3840 }
3841
3842 return v;
3843}
3844
3845static void saved_cmdlines_stop(struct seq_file *m, void *v)
3846{
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04003847 arch_spin_unlock(&trace_cmdline_lock);
3848 preempt_enable();
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003849}
3850
3851static int saved_cmdlines_show(struct seq_file *m, void *v)
3852{
3853 char buf[TASK_COMM_LEN];
3854 unsigned int *pid = v;
3855
Steven Rostedt (Red Hat)4c27e752014-05-30 10:49:46 -04003856 __trace_find_cmdline(*pid, buf);
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003857 seq_printf(m, "%d %s\n", *pid, buf);
3858 return 0;
3859}
3860
3861static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3862 .start = saved_cmdlines_start,
3863 .next = saved_cmdlines_next,
3864 .stop = saved_cmdlines_stop,
3865 .show = saved_cmdlines_show,
3866};
3867
3868static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3869{
3870 if (tracing_disabled)
3871 return -ENODEV;
3872
3873 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
Avadh Patel69abe6a2009-04-10 16:04:48 -04003874}
3875
3876static const struct file_operations tracing_saved_cmdlines_fops = {
Yoshihiro YUNOMAE42584c82014-02-20 17:44:31 +09003877 .open = tracing_saved_cmdlines_open,
3878 .read = seq_read,
3879 .llseek = seq_lseek,
3880 .release = seq_release,
Avadh Patel69abe6a2009-04-10 16:04:48 -04003881};
3882
3883static ssize_t
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003884tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3885 size_t cnt, loff_t *ppos)
3886{
3887 char buf[64];
3888 int r;
3889
3890 arch_spin_lock(&trace_cmdline_lock);
Namhyung Kima6af8fb2014-06-10 16:11:35 +09003891 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003892 arch_spin_unlock(&trace_cmdline_lock);
3893
3894 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3895}
3896
3897static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3898{
3899 kfree(s->saved_cmdlines);
3900 kfree(s->map_cmdline_to_pid);
3901 kfree(s);
3902}
3903
3904static int tracing_resize_saved_cmdlines(unsigned int val)
3905{
3906 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3907
Namhyung Kima6af8fb2014-06-10 16:11:35 +09003908 s = kmalloc(sizeof(*s), GFP_KERNEL);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09003909 if (!s)
3910 return -ENOMEM;
3911
3912 if (allocate_cmdlines_buffer(val, s) < 0) {
3913 kfree(s);
3914 return -ENOMEM;
3915 }
3916
3917 arch_spin_lock(&trace_cmdline_lock);
3918 savedcmd_temp = savedcmd;
3919 savedcmd = s;
3920 arch_spin_unlock(&trace_cmdline_lock);
3921 free_saved_cmdlines_buffer(savedcmd_temp);
3922
3923 return 0;
3924}
3925
3926static ssize_t
3927tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3928 size_t cnt, loff_t *ppos)
3929{
3930 unsigned long val;
3931 int ret;
3932
3933 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3934 if (ret)
3935 return ret;
3936
3937 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3938 if (!val || val > PID_MAX_DEFAULT)
3939 return -EINVAL;
3940
3941 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3942 if (ret < 0)
3943 return ret;
3944
3945 *ppos += cnt;
3946
3947 return cnt;
3948}
3949
3950static const struct file_operations tracing_saved_cmdlines_size_fops = {
3951 .open = tracing_open_generic,
3952 .read = tracing_saved_cmdlines_size_read,
3953 .write = tracing_saved_cmdlines_size_write,
3954};
3955
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04003956#ifdef CONFIG_TRACE_ENUM_MAP_FILE
3957static union trace_enum_map_item *
3958update_enum_map(union trace_enum_map_item *ptr)
3959{
3960 if (!ptr->map.enum_string) {
3961 if (ptr->tail.next) {
3962 ptr = ptr->tail.next;
3963 /* Set ptr to the next real item (skip head) */
3964 ptr++;
3965 } else
3966 return NULL;
3967 }
3968 return ptr;
3969}
3970
3971static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
3972{
3973 union trace_enum_map_item *ptr = v;
3974
3975 /*
3976 * Paranoid! If ptr points to end, we don't want to increment past it.
3977 * This really should never happen.
3978 */
3979 ptr = update_enum_map(ptr);
3980 if (WARN_ON_ONCE(!ptr))
3981 return NULL;
3982
3983 ptr++;
3984
3985 (*pos)++;
3986
3987 ptr = update_enum_map(ptr);
3988
3989 return ptr;
3990}
3991
3992static void *enum_map_start(struct seq_file *m, loff_t *pos)
3993{
3994 union trace_enum_map_item *v;
3995 loff_t l = 0;
3996
3997 mutex_lock(&trace_enum_mutex);
3998
3999 v = trace_enum_maps;
4000 if (v)
4001 v++;
4002
4003 while (v && l < *pos) {
4004 v = enum_map_next(m, v, &l);
4005 }
4006
4007 return v;
4008}
4009
4010static void enum_map_stop(struct seq_file *m, void *v)
4011{
4012 mutex_unlock(&trace_enum_mutex);
4013}
4014
4015static int enum_map_show(struct seq_file *m, void *v)
4016{
4017 union trace_enum_map_item *ptr = v;
4018
4019 seq_printf(m, "%s %ld (%s)\n",
4020 ptr->map.enum_string, ptr->map.enum_value,
4021 ptr->map.system);
4022
4023 return 0;
4024}
4025
4026static const struct seq_operations tracing_enum_map_seq_ops = {
4027 .start = enum_map_start,
4028 .next = enum_map_next,
4029 .stop = enum_map_stop,
4030 .show = enum_map_show,
4031};
4032
4033static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4034{
4035 if (tracing_disabled)
4036 return -ENODEV;
4037
4038 return seq_open(filp, &tracing_enum_map_seq_ops);
4039}
4040
4041static const struct file_operations tracing_enum_map_fops = {
4042 .open = tracing_enum_map_open,
4043 .read = seq_read,
4044 .llseek = seq_lseek,
4045 .release = seq_release,
4046};
4047
4048static inline union trace_enum_map_item *
4049trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4050{
4051 /* Return tail of array given the head */
4052 return ptr + ptr->head.length + 1;
4053}
4054
4055static void
4056trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4057 int len)
4058{
4059 struct trace_enum_map **stop;
4060 struct trace_enum_map **map;
4061 union trace_enum_map_item *map_array;
4062 union trace_enum_map_item *ptr;
4063
4064 stop = start + len;
4065
4066 /*
4067 * The trace_enum_maps contains the map plus a head and tail item,
4068 * where the head holds the module and length of array, and the
4069 * tail holds a pointer to the next list.
4070 */
4071 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4072 if (!map_array) {
4073 pr_warning("Unable to allocate trace enum mapping\n");
4074 return;
4075 }
4076
4077 mutex_lock(&trace_enum_mutex);
4078
4079 if (!trace_enum_maps)
4080 trace_enum_maps = map_array;
4081 else {
4082 ptr = trace_enum_maps;
4083 for (;;) {
4084 ptr = trace_enum_jmp_to_tail(ptr);
4085 if (!ptr->tail.next)
4086 break;
4087 ptr = ptr->tail.next;
4088
4089 }
4090 ptr->tail.next = map_array;
4091 }
4092 map_array->head.mod = mod;
4093 map_array->head.length = len;
4094 map_array++;
4095
4096 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4097 map_array->map = **map;
4098 map_array++;
4099 }
4100 memset(map_array, 0, sizeof(*map_array));
4101
4102 mutex_unlock(&trace_enum_mutex);
4103}
4104
4105static void trace_create_enum_file(struct dentry *d_tracer)
4106{
4107 trace_create_file("enum_map", 0444, d_tracer,
4108 NULL, &tracing_enum_map_fops);
4109}
4110
4111#else /* CONFIG_TRACE_ENUM_MAP_FILE */
4112static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4113static inline void trace_insert_enum_map_file(struct module *mod,
4114 struct trace_enum_map **start, int len) { }
4115#endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4116
4117static void trace_insert_enum_map(struct module *mod,
4118 struct trace_enum_map **start, int len)
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04004119{
4120 struct trace_enum_map **map;
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04004121
4122 if (len <= 0)
4123 return;
4124
4125 map = start;
4126
4127 trace_event_enum_update(map, len);
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04004128
4129 trace_insert_enum_map_file(mod, start, len);
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04004130}
4131
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09004132static ssize_t
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004133tracing_set_trace_read(struct file *filp, char __user *ubuf,
4134 size_t cnt, loff_t *ppos)
4135{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004136 struct trace_array *tr = filp->private_data;
Li Zefanee6c2c12009-09-18 14:06:47 +08004137 char buf[MAX_TRACER_SIZE+2];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004138 int r;
4139
4140 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004141 r = sprintf(buf, "%s\n", tr->current_trace->name);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004142 mutex_unlock(&trace_types_lock);
4143
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004144 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004145}
4146
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02004147int tracer_init(struct tracer *t, struct trace_array *tr)
4148{
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004149 tracing_reset_online_cpus(&tr->trace_buffer);
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02004150 return t->init(tr);
4151}
4152
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004153static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004154{
4155 int cpu;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05004156
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004157 for_each_tracing_cpu(cpu)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004158 per_cpu_ptr(buf->data, cpu)->entries = val;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004159}
4160
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004161#ifdef CONFIG_TRACER_MAX_TRACE
Hiraku Toyookad60da502012-10-17 11:56:16 +09004162/* resize @tr's buffer to the size of @size_tr's entries */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004163static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4164 struct trace_buffer *size_buf, int cpu_id)
Hiraku Toyookad60da502012-10-17 11:56:16 +09004165{
4166 int cpu, ret = 0;
4167
4168 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4169 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004170 ret = ring_buffer_resize(trace_buf->buffer,
4171 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
Hiraku Toyookad60da502012-10-17 11:56:16 +09004172 if (ret < 0)
4173 break;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004174 per_cpu_ptr(trace_buf->data, cpu)->entries =
4175 per_cpu_ptr(size_buf->data, cpu)->entries;
Hiraku Toyookad60da502012-10-17 11:56:16 +09004176 }
4177 } else {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004178 ret = ring_buffer_resize(trace_buf->buffer,
4179 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
Hiraku Toyookad60da502012-10-17 11:56:16 +09004180 if (ret == 0)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004181 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4182 per_cpu_ptr(size_buf->data, cpu_id)->entries;
Hiraku Toyookad60da502012-10-17 11:56:16 +09004183 }
4184
4185 return ret;
4186}
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004187#endif /* CONFIG_TRACER_MAX_TRACE */
Hiraku Toyookad60da502012-10-17 11:56:16 +09004188
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004189static int __tracing_resize_ring_buffer(struct trace_array *tr,
4190 unsigned long size, int cpu)
Steven Rostedt73c51622009-03-11 13:42:01 -04004191{
4192 int ret;
4193
4194 /*
4195 * If kernel or user changes the size of the ring buffer
Steven Rostedta123c522009-03-12 11:21:08 -04004196 * we use the size that was given, and we can forget about
4197 * expanding it later.
Steven Rostedt73c51622009-03-11 13:42:01 -04004198 */
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05004199 ring_buffer_expanded = true;
Steven Rostedt73c51622009-03-11 13:42:01 -04004200
Steven Rostedtb382ede62012-10-10 21:44:34 -04004201 /* May be called before buffers are initialized */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004202 if (!tr->trace_buffer.buffer)
Steven Rostedtb382ede62012-10-10 21:44:34 -04004203 return 0;
4204
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004205 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04004206 if (ret < 0)
4207 return ret;
4208
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004209#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004210 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4211 !tr->current_trace->use_max_tr)
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004212 goto out;
4213
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004214 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04004215 if (ret < 0) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004216 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4217 &tr->trace_buffer, cpu);
Steven Rostedt73c51622009-03-11 13:42:01 -04004218 if (r < 0) {
Steven Rostedta123c522009-03-12 11:21:08 -04004219 /*
4220 * AARGH! We are left with different
4221 * size max buffer!!!!
4222 * The max buffer is our "snapshot" buffer.
4223 * When a tracer needs a snapshot (one of the
4224 * latency tracers), it swaps the max buffer
4225 * with the saved snap shot. We succeeded to
4226 * update the size of the main buffer, but failed to
4227 * update the size of the max buffer. But when we tried
4228 * to reset the main buffer to the original size, we
4229 * failed there too. This is very unlikely to
4230 * happen, but if it does, warn and kill all
4231 * tracing.
4232 */
Steven Rostedt73c51622009-03-11 13:42:01 -04004233 WARN_ON(1);
4234 tracing_disabled = 1;
4235 }
4236 return ret;
4237 }
4238
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004239 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004240 set_buffer_entries(&tr->max_buffer, size);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004241 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004242 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004243
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004244 out:
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004245#endif /* CONFIG_TRACER_MAX_TRACE */
4246
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004247 if (cpu == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004248 set_buffer_entries(&tr->trace_buffer, size);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004249 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004250 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
Steven Rostedt73c51622009-03-11 13:42:01 -04004251
4252 return ret;
4253}
4254
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004255static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4256 unsigned long size, int cpu_id)
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004257{
Vaibhav Nagarnaik83f40312012-05-03 18:59:50 -07004258 int ret = size;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004259
4260 mutex_lock(&trace_types_lock);
4261
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004262 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4263 /* make sure, this cpu is enabled in the mask */
4264 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4265 ret = -EINVAL;
4266 goto out;
4267 }
4268 }
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004269
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004270 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004271 if (ret < 0)
4272 ret = -ENOMEM;
4273
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004274out:
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004275 mutex_unlock(&trace_types_lock);
4276
4277 return ret;
4278}
4279
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004280
Steven Rostedt1852fcc2009-03-11 14:33:00 -04004281/**
4282 * tracing_update_buffers - used by tracing facility to expand ring buffers
4283 *
4284 * To save on memory when the tracing is never used on a system with it
4285 * configured in. The ring buffers are set to a minimum size. But once
4286 * a user starts to use the tracing facility, then they need to grow
4287 * to their default size.
4288 *
4289 * This function is to be called when a tracer is about to be used.
4290 */
4291int tracing_update_buffers(void)
4292{
4293 int ret = 0;
4294
Steven Rostedt1027fcb2009-03-12 11:33:20 -04004295 mutex_lock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04004296 if (!ring_buffer_expanded)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004297 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004298 RING_BUFFER_ALL_CPUS);
Steven Rostedt1027fcb2009-03-12 11:33:20 -04004299 mutex_unlock(&trace_types_lock);
Steven Rostedt1852fcc2009-03-11 14:33:00 -04004300
4301 return ret;
4302}
4303
Steven Rostedt577b7852009-02-26 23:43:05 -05004304struct trace_option_dentry;
4305
4306static struct trace_option_dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004307create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
Steven Rostedt577b7852009-02-26 23:43:05 -05004308
Steven Rostedt (Red Hat)6b450d22014-01-14 08:43:01 -05004309/*
4310 * Used to clear out the tracer before deletion of an instance.
4311 * Must have trace_types_lock held.
4312 */
4313static void tracing_set_nop(struct trace_array *tr)
4314{
4315 if (tr->current_trace == &nop_trace)
4316 return;
4317
Steven Rostedt (Red Hat)50512ab2014-01-14 08:52:35 -05004318 tr->current_trace->enabled--;
Steven Rostedt (Red Hat)6b450d22014-01-14 08:43:01 -05004319
4320 if (tr->current_trace->reset)
4321 tr->current_trace->reset(tr);
4322
4323 tr->current_trace = &nop_trace;
4324}
4325
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04004326static void add_tracer_options(struct trace_array *tr, struct tracer *t)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004327{
Steven Rostedt (Red Hat)09d23a12015-02-03 12:45:53 -05004328 /* Only enable if the directory has been created already. */
4329 if (!tr->dir)
4330 return;
4331
4332 /* Currently, only the top instance has options */
4333 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL))
4334 return;
4335
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04004336 /* Ignore if they were already created */
4337 if (t->topts)
4338 return;
4339
4340 t->topts = create_trace_option_files(tr, t);
Steven Rostedt (Red Hat)09d23a12015-02-03 12:45:53 -05004341}
4342
4343static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4344{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004345 struct tracer *t;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004346#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt34600f02013-01-22 13:35:11 -05004347 bool had_max_tr;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004348#endif
Peter Zijlstrad9e54072008-11-01 19:57:37 +01004349 int ret = 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004350
Steven Rostedt1027fcb2009-03-12 11:33:20 -04004351 mutex_lock(&trace_types_lock);
4352
Steven Rostedt73c51622009-03-11 13:42:01 -04004353 if (!ring_buffer_expanded) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004354 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004355 RING_BUFFER_ALL_CPUS);
Steven Rostedt73c51622009-03-11 13:42:01 -04004356 if (ret < 0)
Frederic Weisbecker59f586d2009-03-15 22:10:39 +01004357 goto out;
Steven Rostedt73c51622009-03-11 13:42:01 -04004358 ret = 0;
4359 }
4360
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004361 for (t = trace_types; t; t = t->next) {
4362 if (strcmp(t->name, buf) == 0)
4363 break;
4364 }
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02004365 if (!t) {
4366 ret = -EINVAL;
4367 goto out;
4368 }
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004369 if (t == tr->current_trace)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004370 goto out;
4371
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05004372 /* Some tracers are only allowed for the top level buffer */
4373 if (!trace_ok_for_array(t, tr)) {
4374 ret = -EINVAL;
4375 goto out;
4376 }
4377
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05004378 /* If trace pipe files are being read, we can't change the tracer */
4379 if (tr->current_trace->ref) {
4380 ret = -EBUSY;
4381 goto out;
4382 }
4383
Steven Rostedt9f029e82008-11-12 15:24:24 -05004384 trace_branch_disable();
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04004385
Steven Rostedt (Red Hat)50512ab2014-01-14 08:52:35 -05004386 tr->current_trace->enabled--;
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04004387
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004388 if (tr->current_trace->reset)
4389 tr->current_trace->reset(tr);
Steven Rostedt34600f02013-01-22 13:35:11 -05004390
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004391 /* Current trace needs to be nop_trace before synchronize_sched */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004392 tr->current_trace = &nop_trace;
Steven Rostedt34600f02013-01-22 13:35:11 -05004393
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05004394#ifdef CONFIG_TRACER_MAX_TRACE
4395 had_max_tr = tr->allocated_snapshot;
Steven Rostedt34600f02013-01-22 13:35:11 -05004396
4397 if (had_max_tr && !t->use_max_tr) {
4398 /*
4399 * We need to make sure that the update_max_tr sees that
4400 * current_trace changed to nop_trace to keep it from
4401 * swapping the buffers after we resize it.
4402 * The update_max_tr is called from interrupts disabled
4403 * so a synchronized_sched() is sufficient.
4404 */
4405 synchronize_sched();
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04004406 free_snapshot(tr);
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004407 }
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004408#endif
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004409
4410#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt34600f02013-01-22 13:35:11 -05004411 if (t->use_max_tr && !had_max_tr) {
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04004412 ret = alloc_snapshot(tr);
Hiraku Toyookad60da502012-10-17 11:56:16 +09004413 if (ret < 0)
4414 goto out;
KOSAKI Motohiroef710e12010-07-01 14:34:35 +09004415 }
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004416#endif
Steven Rostedt577b7852009-02-26 23:43:05 -05004417
Frederic Weisbecker1c800252008-11-16 05:57:26 +01004418 if (t->init) {
Arnaldo Carvalho de Melob6f11df2009-02-05 18:02:00 -02004419 ret = tracer_init(t, tr);
Frederic Weisbecker1c800252008-11-16 05:57:26 +01004420 if (ret)
4421 goto out;
4422 }
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004423
Steven Rostedt2b6080f2012-05-11 13:29:49 -04004424 tr->current_trace = t;
Steven Rostedt (Red Hat)50512ab2014-01-14 08:52:35 -05004425 tr->current_trace->enabled++;
Steven Rostedt9f029e82008-11-12 15:24:24 -05004426 trace_branch_enable(tr);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004427 out:
4428 mutex_unlock(&trace_types_lock);
4429
Peter Zijlstrad9e54072008-11-01 19:57:37 +01004430 return ret;
4431}
4432
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004433static ssize_t
4434tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4435 size_t cnt, loff_t *ppos)
4436{
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05004437 struct trace_array *tr = filp->private_data;
Li Zefanee6c2c12009-09-18 14:06:47 +08004438 char buf[MAX_TRACER_SIZE+1];
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004439 int i;
4440 size_t ret;
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01004441 int err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004442
Steven Rostedt60063a62008-10-28 10:44:24 -04004443 ret = cnt;
4444
Li Zefanee6c2c12009-09-18 14:06:47 +08004445 if (cnt > MAX_TRACER_SIZE)
4446 cnt = MAX_TRACER_SIZE;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004447
4448 if (copy_from_user(&buf, ubuf, cnt))
4449 return -EFAULT;
4450
4451 buf[cnt] = 0;
4452
4453 /* strip ending whitespace. */
4454 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4455 buf[i] = 0;
4456
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05004457 err = tracing_set_tracer(tr, buf);
Frederic Weisbeckere6e7a652008-11-16 05:53:19 +01004458 if (err)
4459 return err;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004460
Jiri Olsacf8517c2009-10-23 19:36:16 -04004461 *ppos += ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004462
Frederic Weisbeckerc2931e02008-10-04 22:04:44 +02004463 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004464}
4465
4466static ssize_t
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04004467tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4468 size_t cnt, loff_t *ppos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004469{
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004470 char buf[64];
4471 int r;
4472
Steven Rostedtcffae432008-05-12 21:21:00 +02004473 r = snprintf(buf, sizeof(buf), "%ld\n",
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004474 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
Steven Rostedtcffae432008-05-12 21:21:00 +02004475 if (r > sizeof(buf))
4476 r = sizeof(buf);
Ingo Molnar4bf39a92008-05-12 21:20:46 +02004477 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004478}
4479
4480static ssize_t
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04004481tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4482 size_t cnt, loff_t *ppos)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004483{
Hannes Eder5e398412009-02-10 19:44:34 +01004484 unsigned long val;
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02004485 int ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004486
Peter Huewe22fe9b52011-06-07 21:58:27 +02004487 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4488 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02004489 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02004490
4491 *ptr = val * 1000;
4492
4493 return cnt;
4494}
4495
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04004496static ssize_t
4497tracing_thresh_read(struct file *filp, char __user *ubuf,
4498 size_t cnt, loff_t *ppos)
4499{
4500 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4501}
4502
4503static ssize_t
4504tracing_thresh_write(struct file *filp, const char __user *ubuf,
4505 size_t cnt, loff_t *ppos)
4506{
4507 struct trace_array *tr = filp->private_data;
4508 int ret;
4509
4510 mutex_lock(&trace_types_lock);
4511 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4512 if (ret < 0)
4513 goto out;
4514
4515 if (tr->current_trace->update_thresh) {
4516 ret = tr->current_trace->update_thresh(tr);
4517 if (ret < 0)
4518 goto out;
4519 }
4520
4521 ret = cnt;
4522out:
4523 mutex_unlock(&trace_types_lock);
4524
4525 return ret;
4526}
4527
4528static ssize_t
4529tracing_max_lat_read(struct file *filp, char __user *ubuf,
4530 size_t cnt, loff_t *ppos)
4531{
4532 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4533}
4534
4535static ssize_t
4536tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4537 size_t cnt, loff_t *ppos)
4538{
4539 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4540}
4541
Steven Rostedtb3806b42008-05-12 21:20:46 +02004542static int tracing_open_pipe(struct inode *inode, struct file *filp)
4543{
Oleg Nesterov15544202013-07-23 17:25:57 +02004544 struct trace_array *tr = inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004545 struct trace_iterator *iter;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004546 int ret = 0;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004547
4548 if (tracing_disabled)
4549 return -ENODEV;
4550
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004551 if (trace_array_get(tr) < 0)
4552 return -ENODEV;
4553
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004554 mutex_lock(&trace_types_lock);
4555
Steven Rostedtb3806b42008-05-12 21:20:46 +02004556 /* create a buffer to store the information to pass to userspace */
4557 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004558 if (!iter) {
4559 ret = -ENOMEM;
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07004560 __trace_array_put(tr);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004561 goto out;
4562 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02004563
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04004564 trace_seq_init(&iter->seq);
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05004565 iter->trace = tr->current_trace;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004566
4567 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4568 ret = -ENOMEM;
4569 goto fail;
Rusty Russell44623442009-01-01 10:12:23 +10304570 }
4571
Steven Rostedta3097202008-11-07 22:36:02 -05004572 /* trace pipe does not show start of buffer */
Rusty Russell44623442009-01-01 10:12:23 +10304573 cpumask_setall(iter->started);
Steven Rostedta3097202008-11-07 22:36:02 -05004574
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04004575 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
Steven Rostedt112f38a72009-06-01 15:16:05 -04004576 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4577
David Sharp8be07092012-11-13 12:18:22 -08004578 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09004579 if (trace_clocks[tr->clock_id].in_ns)
David Sharp8be07092012-11-13 12:18:22 -08004580 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4581
Oleg Nesterov15544202013-07-23 17:25:57 +02004582 iter->tr = tr;
4583 iter->trace_buffer = &tr->trace_buffer;
4584 iter->cpu_file = tracing_get_cpu(inode);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004585 mutex_init(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004586 filp->private_data = iter;
4587
Steven Rostedt107bad82008-05-12 21:21:01 +02004588 if (iter->trace->pipe_open)
4589 iter->trace->pipe_open(iter);
Steven Rostedt107bad82008-05-12 21:21:01 +02004590
Arnd Bergmannb4447862010-07-07 23:40:11 +02004591 nonseekable_open(inode, filp);
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05004592
4593 tr->current_trace->ref++;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004594out:
4595 mutex_unlock(&trace_types_lock);
4596 return ret;
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004597
4598fail:
4599 kfree(iter->trace);
4600 kfree(iter);
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004601 __trace_array_put(tr);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004602 mutex_unlock(&trace_types_lock);
4603 return ret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004604}
4605
4606static int tracing_release_pipe(struct inode *inode, struct file *file)
4607{
4608 struct trace_iterator *iter = file->private_data;
Oleg Nesterov15544202013-07-23 17:25:57 +02004609 struct trace_array *tr = inode->i_private;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004610
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004611 mutex_lock(&trace_types_lock);
4612
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05004613 tr->current_trace->ref--;
4614
Steven Rostedt29bf4a52009-12-09 12:37:43 -05004615 if (iter->trace->pipe_close)
Steven Rostedtc521efd2009-12-07 09:06:24 -05004616 iter->trace->pipe_close(iter);
4617
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01004618 mutex_unlock(&trace_types_lock);
4619
Rusty Russell44623442009-01-01 10:12:23 +10304620 free_cpumask_var(iter->started);
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004621 mutex_destroy(&iter->mutex);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004622 kfree(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004623
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04004624 trace_array_put(tr);
4625
Steven Rostedtb3806b42008-05-12 21:20:46 +02004626 return 0;
4627}
4628
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004629static unsigned int
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004630trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004631{
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04004632 struct trace_array *tr = iter->tr;
4633
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004634 /* Iterators are static, they should be filled or empty */
4635 if (trace_buffer_iter(iter, iter->cpu_file))
4636 return POLLIN | POLLRDNORM;
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004637
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04004638 if (tr->trace_flags & TRACE_ITER_BLOCK)
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004639 /*
4640 * Always select as readable when in blocking mode
4641 */
4642 return POLLIN | POLLRDNORM;
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004643 else
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004644 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
Steven Rostedt (Red Hat)15693452013-02-28 19:59:17 -05004645 filp, poll_table);
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004646}
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004647
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05004648static unsigned int
4649tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4650{
4651 struct trace_iterator *iter = filp->private_data;
4652
4653 return trace_poll(iter, filp, poll_table);
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02004654}
4655
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05004656/* Must be called with iter->mutex held. */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004657static int tracing_wait_pipe(struct file *filp)
4658{
4659 struct trace_iterator *iter = filp->private_data;
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04004660 int ret;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004661
4662 while (trace_empty(iter)) {
4663
4664 if ((filp->f_flags & O_NONBLOCK)) {
4665 return -EAGAIN;
4666 }
4667
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004668 /*
Liu Bo250bfd32013-01-14 10:54:11 +08004669 * We block until we read something and tracing is disabled.
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004670 * We still block if tracing is disabled, but we have never
4671 * read anything. This allows a user to cat this file, and
4672 * then enable tracing. But after we have read something,
4673 * we give an EOF when tracing is again disabled.
4674 *
4675 * iter->pos will be 0 if we haven't read anything.
4676 */
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04004677 if (!tracing_is_on() && iter->pos)
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004678 break;
Steven Rostedt (Red Hat)f4874262014-04-29 16:07:28 -04004679
4680 mutex_unlock(&iter->mutex);
4681
Rabin Vincente30f53a2014-11-10 19:46:34 +01004682 ret = wait_on_pipe(iter, false);
Steven Rostedt (Red Hat)f4874262014-04-29 16:07:28 -04004683
4684 mutex_lock(&iter->mutex);
4685
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04004686 if (ret)
4687 return ret;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004688 }
4689
4690 return 1;
4691}
4692
Steven Rostedtb3806b42008-05-12 21:20:46 +02004693/*
4694 * Consumer reader.
4695 */
4696static ssize_t
4697tracing_read_pipe(struct file *filp, char __user *ubuf,
4698 size_t cnt, loff_t *ppos)
4699{
4700 struct trace_iterator *iter = filp->private_data;
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004701 ssize_t sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004702
4703 /* return any leftover data */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004704 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4705 if (sret != -EBUSY)
4706 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004707
Steven Rostedtf9520752009-03-02 14:04:40 -05004708 trace_seq_init(&iter->seq);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004709
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004710 /*
4711 * Avoid more than one consumer on a single file descriptor
4712 * This is just a matter of traces coherency, the ring buffer itself
4713 * is protected.
4714 */
4715 mutex_lock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02004716 if (iter->trace->read) {
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004717 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4718 if (sret)
Steven Rostedt107bad82008-05-12 21:21:01 +02004719 goto out;
Steven Rostedt107bad82008-05-12 21:21:01 +02004720 }
4721
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004722waitagain:
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004723 sret = tracing_wait_pipe(filp);
4724 if (sret <= 0)
4725 goto out;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004726
4727 /* stop when tracing is finished */
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004728 if (trace_empty(iter)) {
4729 sret = 0;
Steven Rostedt107bad82008-05-12 21:21:01 +02004730 goto out;
Eduard - Gabriel Munteanuff987812009-02-09 08:15:55 +02004731 }
Steven Rostedtb3806b42008-05-12 21:20:46 +02004732
4733 if (cnt >= PAGE_SIZE)
4734 cnt = PAGE_SIZE - 1;
4735
Steven Rostedt53d0aa72008-05-12 21:21:01 +02004736 /* reset all but tr, trace, and overruns */
Steven Rostedt53d0aa72008-05-12 21:21:01 +02004737 memset(&iter->seq, 0,
4738 sizeof(struct trace_iterator) -
4739 offsetof(struct trace_iterator, seq));
Andrew Vagined5467d2013-08-02 21:16:43 +04004740 cpumask_clear(iter->started);
Steven Rostedt4823ed72008-05-12 21:21:01 +02004741 iter->pos = -1;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004742
Lai Jiangshan4f535962009-05-18 19:35:34 +08004743 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004744 trace_access_lock(iter->cpu_file);
Jason Wessel955b61e2010-08-05 09:22:23 -05004745 while (trace_find_next_entry_inc(iter) != NULL) {
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02004746 enum print_line_t ret;
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004747 int save_len = iter->seq.seq.len;
Steven Rostedt088b1e422008-05-12 21:20:48 +02004748
Ingo Molnarf9896bf2008-05-12 21:20:47 +02004749 ret = print_trace_line(iter);
Frederic Weisbecker2c4f0352008-09-29 20:18:34 +02004750 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt088b1e422008-05-12 21:20:48 +02004751 /* don't print partial lines */
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004752 iter->seq.seq.len = save_len;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004753 break;
Steven Rostedt088b1e422008-05-12 21:20:48 +02004754 }
Frederic Weisbeckerb91facc2009-02-06 18:30:44 +01004755 if (ret != TRACE_TYPE_NO_CONSUME)
4756 trace_consume(iter);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004757
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004758 if (trace_seq_used(&iter->seq) >= cnt)
Steven Rostedtb3806b42008-05-12 21:20:46 +02004759 break;
Jiri Olsaee5e51f2011-03-25 12:05:18 +01004760
4761 /*
4762 * Setting the full flag means we reached the trace_seq buffer
4763 * size and we should leave by partial output condition above.
4764 * One of the trace_seq_* functions is not used properly.
4765 */
4766 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4767 iter->ent->type);
Steven Rostedtb3806b42008-05-12 21:20:46 +02004768 }
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004769 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004770 trace_event_read_unlock();
Steven Rostedtb3806b42008-05-12 21:20:46 +02004771
Steven Rostedtb3806b42008-05-12 21:20:46 +02004772 /* Now copy what we have to the user */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004773 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004774 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
Steven Rostedtf9520752009-03-02 14:04:40 -05004775 trace_seq_init(&iter->seq);
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004776
4777 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004778 * If there was nothing to send to user, in spite of consuming trace
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004779 * entries, go back to wait for more entries.
4780 */
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004781 if (sret == -EBUSY)
Pekka Paalanen9ff4b972008-09-29 20:23:48 +02004782 goto waitagain;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004783
Steven Rostedt107bad82008-05-12 21:21:01 +02004784out:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004785 mutex_unlock(&iter->mutex);
Steven Rostedt107bad82008-05-12 21:21:01 +02004786
Pekka Paalanen6c6c2792008-05-12 21:21:02 +02004787 return sret;
Steven Rostedtb3806b42008-05-12 21:20:46 +02004788}
4789
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004790static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4791 unsigned int idx)
4792{
4793 __free_page(spd->pages[idx]);
4794}
4795
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08004796static const struct pipe_buf_operations tracing_pipe_buf_ops = {
Steven Rostedt34cd4992009-02-09 12:06:29 -05004797 .can_merge = 0,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004798 .confirm = generic_pipe_buf_confirm,
Al Viro92fdd982014-01-17 07:53:39 -05004799 .release = generic_pipe_buf_release,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004800 .steal = generic_pipe_buf_steal,
4801 .get = generic_pipe_buf_get,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004802};
4803
Steven Rostedt34cd4992009-02-09 12:06:29 -05004804static size_t
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01004805tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004806{
4807 size_t count;
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004808 int save_len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004809 int ret;
4810
4811 /* Seq buffer is page-sized, exactly what we need. */
4812 for (;;) {
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004813 save_len = iter->seq.seq.len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004814 ret = print_trace_line(iter);
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004815
4816 if (trace_seq_has_overflowed(&iter->seq)) {
4817 iter->seq.seq.len = save_len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004818 break;
4819 }
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004820
4821 /*
4822 * This should not be hit, because it should only
4823 * be set if the iter->seq overflowed. But check it
4824 * anyway to be safe.
4825 */
Steven Rostedt34cd4992009-02-09 12:06:29 -05004826 if (ret == TRACE_TYPE_PARTIAL_LINE) {
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004827 iter->seq.seq.len = save_len;
4828 break;
4829 }
4830
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004831 count = trace_seq_used(&iter->seq) - save_len;
Steven Rostedt (Red Hat)74f06bb2014-11-17 13:12:22 -05004832 if (rem < count) {
4833 rem = 0;
4834 iter->seq.seq.len = save_len;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004835 break;
4836 }
4837
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08004838 if (ret != TRACE_TYPE_NO_CONSUME)
4839 trace_consume(iter);
Steven Rostedt34cd4992009-02-09 12:06:29 -05004840 rem -= count;
Jason Wessel955b61e2010-08-05 09:22:23 -05004841 if (!trace_find_next_entry_inc(iter)) {
Steven Rostedt34cd4992009-02-09 12:06:29 -05004842 rem = 0;
4843 iter->ent = NULL;
4844 break;
4845 }
4846 }
4847
4848 return rem;
4849}
4850
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004851static ssize_t tracing_splice_read_pipe(struct file *filp,
4852 loff_t *ppos,
4853 struct pipe_inode_info *pipe,
4854 size_t len,
4855 unsigned int flags)
4856{
Jens Axboe35f3d142010-05-20 10:43:18 +02004857 struct page *pages_def[PIPE_DEF_BUFFERS];
4858 struct partial_page partial_def[PIPE_DEF_BUFFERS];
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004859 struct trace_iterator *iter = filp->private_data;
4860 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02004861 .pages = pages_def,
4862 .partial = partial_def,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004863 .nr_pages = 0, /* This gets updated below. */
Eric Dumazet047fe362012-06-12 15:24:40 +02004864 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt34cd4992009-02-09 12:06:29 -05004865 .flags = flags,
4866 .ops = &tracing_pipe_buf_ops,
4867 .spd_release = tracing_spd_release_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004868 };
4869 ssize_t ret;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004870 size_t rem;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004871 unsigned int i;
4872
Jens Axboe35f3d142010-05-20 10:43:18 +02004873 if (splice_grow_spd(pipe, &spd))
4874 return -ENOMEM;
4875
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004876 mutex_lock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004877
4878 if (iter->trace->splice_read) {
4879 ret = iter->trace->splice_read(iter, filp,
4880 ppos, pipe, len, flags);
4881 if (ret)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004882 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004883 }
4884
4885 ret = tracing_wait_pipe(filp);
4886 if (ret <= 0)
Steven Rostedt34cd4992009-02-09 12:06:29 -05004887 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004888
Jason Wessel955b61e2010-08-05 09:22:23 -05004889 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004890 ret = -EFAULT;
Steven Rostedt34cd4992009-02-09 12:06:29 -05004891 goto out_err;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004892 }
4893
Lai Jiangshan4f535962009-05-18 19:35:34 +08004894 trace_event_read_lock();
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004895 trace_access_lock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004896
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004897 /* Fill as many pages as possible. */
Al Viroa786c062014-04-11 12:01:03 -04004898 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004899 spd.pages[i] = alloc_page(GFP_KERNEL);
4900 if (!spd.pages[i])
Steven Rostedt34cd4992009-02-09 12:06:29 -05004901 break;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004902
Frederic Weisbeckerfa7c7f62009-02-11 02:51:30 +01004903 rem = tracing_fill_pipe_page(rem, iter);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004904
4905 /* Copy the data into the page, so we can start over. */
4906 ret = trace_seq_to_buffer(&iter->seq,
Jens Axboe35f3d142010-05-20 10:43:18 +02004907 page_address(spd.pages[i]),
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004908 trace_seq_used(&iter->seq));
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004909 if (ret < 0) {
Jens Axboe35f3d142010-05-20 10:43:18 +02004910 __free_page(spd.pages[i]);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004911 break;
4912 }
Jens Axboe35f3d142010-05-20 10:43:18 +02004913 spd.partial[i].offset = 0;
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05004914 spd.partial[i].len = trace_seq_used(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004915
Steven Rostedtf9520752009-03-02 14:04:40 -05004916 trace_seq_init(&iter->seq);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004917 }
4918
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08004919 trace_access_unlock(iter->cpu_file);
Lai Jiangshan4f535962009-05-18 19:35:34 +08004920 trace_event_read_unlock();
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004921 mutex_unlock(&iter->mutex);
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004922
4923 spd.nr_pages = i;
4924
Jens Axboe35f3d142010-05-20 10:43:18 +02004925 ret = splice_to_pipe(pipe, &spd);
4926out:
Eric Dumazet047fe362012-06-12 15:24:40 +02004927 splice_shrink_spd(&spd);
Jens Axboe35f3d142010-05-20 10:43:18 +02004928 return ret;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004929
Steven Rostedt34cd4992009-02-09 12:06:29 -05004930out_err:
Frederic Weisbeckerd7350c3f2009-02-25 06:13:16 +01004931 mutex_unlock(&iter->mutex);
Jens Axboe35f3d142010-05-20 10:43:18 +02004932 goto out;
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02004933}
4934
Steven Rostedta98a3c32008-05-12 21:20:59 +02004935static ssize_t
4936tracing_entries_read(struct file *filp, char __user *ubuf,
4937 size_t cnt, loff_t *ppos)
4938{
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02004939 struct inode *inode = file_inode(filp);
4940 struct trace_array *tr = inode->i_private;
4941 int cpu = tracing_get_cpu(inode);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004942 char buf[64];
4943 int r = 0;
4944 ssize_t ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004945
Steven Rostedtdb526ca2009-03-12 13:53:25 -04004946 mutex_lock(&trace_types_lock);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004947
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02004948 if (cpu == RING_BUFFER_ALL_CPUS) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004949 int cpu, buf_size_same;
4950 unsigned long size;
4951
4952 size = 0;
4953 buf_size_same = 1;
4954 /* check if all cpu sizes are same */
4955 for_each_tracing_cpu(cpu) {
4956 /* fill in the size from first enabled cpu */
4957 if (size == 0)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05004958 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4959 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004960 buf_size_same = 0;
4961 break;
4962 }
4963 }
4964
4965 if (buf_size_same) {
4966 if (!ring_buffer_expanded)
4967 r = sprintf(buf, "%lu (expanded: %lu)\n",
4968 size >> 10,
4969 trace_buf_size >> 10);
4970 else
4971 r = sprintf(buf, "%lu\n", size >> 10);
4972 } else
4973 r = sprintf(buf, "X\n");
4974 } else
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02004975 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004976
Steven Rostedtdb526ca2009-03-12 13:53:25 -04004977 mutex_unlock(&trace_types_lock);
4978
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08004979 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4980 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004981}
4982
4983static ssize_t
4984tracing_entries_write(struct file *filp, const char __user *ubuf,
4985 size_t cnt, loff_t *ppos)
4986{
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02004987 struct inode *inode = file_inode(filp);
4988 struct trace_array *tr = inode->i_private;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004989 unsigned long val;
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07004990 int ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004991
Peter Huewe22fe9b52011-06-07 21:58:27 +02004992 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4993 if (ret)
Steven Rostedtc6caeeb2008-05-12 21:21:00 +02004994 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02004995
4996 /* must have at least 1 entry */
4997 if (!val)
4998 return -EINVAL;
4999
Steven Rostedt1696b2b2008-11-13 00:09:35 -05005000 /* value is in KB */
5001 val <<= 10;
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005002 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005003 if (ret < 0)
5004 return ret;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005005
Jiri Olsacf8517c2009-10-23 19:36:16 -04005006 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005007
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005008 return cnt;
5009}
Steven Rostedtbf5e6512008-11-10 21:46:00 -05005010
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005011static ssize_t
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005012tracing_total_entries_read(struct file *filp, char __user *ubuf,
5013 size_t cnt, loff_t *ppos)
5014{
5015 struct trace_array *tr = filp->private_data;
5016 char buf[64];
5017 int r, cpu;
5018 unsigned long size = 0, expanded_size = 0;
5019
5020 mutex_lock(&trace_types_lock);
5021 for_each_tracing_cpu(cpu) {
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005022 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005023 if (!ring_buffer_expanded)
5024 expanded_size += trace_buf_size >> 10;
5025 }
5026 if (ring_buffer_expanded)
5027 r = sprintf(buf, "%lu\n", size);
5028 else
5029 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5030 mutex_unlock(&trace_types_lock);
5031
5032 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5033}
5034
5035static ssize_t
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005036tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5037 size_t cnt, loff_t *ppos)
5038{
5039 /*
5040 * There is no need to read what the user has written, this function
5041 * is just to make sure that there is no error when "echo" is used
5042 */
5043
5044 *ppos += cnt;
Steven Rostedta98a3c32008-05-12 21:20:59 +02005045
5046 return cnt;
5047}
5048
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005049static int
5050tracing_free_buffer_release(struct inode *inode, struct file *filp)
5051{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005052 struct trace_array *tr = inode->i_private;
5053
Steven Rostedtcf30cf62011-06-14 22:44:07 -04005054 /* disable tracing ? */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04005055 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
Alexander Z Lam711e1242013-08-02 18:36:15 -07005056 tracer_tracing_off(tr);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005057 /* resize the ring buffer to 0 */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005058 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005059
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005060 trace_array_put(tr);
5061
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005062 return 0;
5063}
5064
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005065static ssize_t
5066tracing_mark_write(struct file *filp, const char __user *ubuf,
5067 size_t cnt, loff_t *fpos)
5068{
Steven Rostedtd696b582011-09-22 11:50:27 -04005069 unsigned long addr = (unsigned long)ubuf;
Alexander Z Lam2d716192013-07-01 15:31:24 -07005070 struct trace_array *tr = filp->private_data;
Steven Rostedtd696b582011-09-22 11:50:27 -04005071 struct ring_buffer_event *event;
5072 struct ring_buffer *buffer;
5073 struct print_entry *entry;
5074 unsigned long irq_flags;
5075 struct page *pages[2];
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005076 void *map_page[2];
Steven Rostedtd696b582011-09-22 11:50:27 -04005077 int nr_pages = 1;
5078 ssize_t written;
Steven Rostedtd696b582011-09-22 11:50:27 -04005079 int offset;
5080 int size;
5081 int len;
5082 int ret;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005083 int i;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005084
Steven Rostedtc76f0692008-11-07 22:36:02 -05005085 if (tracing_disabled)
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005086 return -EINVAL;
5087
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04005088 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
Mandeep Singh Baines5224c3a2012-09-07 18:12:19 -07005089 return -EINVAL;
5090
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005091 if (cnt > TRACE_BUF_SIZE)
5092 cnt = TRACE_BUF_SIZE;
5093
Steven Rostedtd696b582011-09-22 11:50:27 -04005094 /*
5095 * Userspace is injecting traces into the kernel trace buffer.
5096 * We want to be as non intrusive as possible.
5097 * To do so, we do not want to allocate any special buffers
5098 * or take any locks, but instead write the userspace data
5099 * straight into the ring buffer.
5100 *
5101 * First we need to pin the userspace buffer into memory,
5102 * which, most likely it is, because it just referenced it.
5103 * But there's no guarantee that it is. By using get_user_pages_fast()
5104 * and kmap_atomic/kunmap_atomic() we can get access to the
5105 * pages directly. We then write the data directly into the
5106 * ring buffer.
5107 */
5108 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005109
Steven Rostedtd696b582011-09-22 11:50:27 -04005110 /* check if we cross pages */
5111 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5112 nr_pages = 2;
5113
5114 offset = addr & (PAGE_SIZE - 1);
5115 addr &= PAGE_MASK;
5116
5117 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5118 if (ret < nr_pages) {
5119 while (--ret >= 0)
5120 put_page(pages[ret]);
5121 written = -EFAULT;
5122 goto out;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005123 }
5124
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005125 for (i = 0; i < nr_pages; i++)
5126 map_page[i] = kmap_atomic(pages[i]);
Steven Rostedtd696b582011-09-22 11:50:27 -04005127
5128 local_save_flags(irq_flags);
5129 size = sizeof(*entry) + cnt + 2; /* possible \n added */
Alexander Z Lam2d716192013-07-01 15:31:24 -07005130 buffer = tr->trace_buffer.buffer;
Steven Rostedtd696b582011-09-22 11:50:27 -04005131 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5132 irq_flags, preempt_count());
5133 if (!event) {
5134 /* Ring buffer disabled, return as if not open for write */
5135 written = -EBADF;
5136 goto out_unlock;
5137 }
5138
5139 entry = ring_buffer_event_data(event);
5140 entry->ip = _THIS_IP_;
5141
5142 if (nr_pages == 2) {
5143 len = PAGE_SIZE - offset;
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005144 memcpy(&entry->buf, map_page[0] + offset, len);
5145 memcpy(&entry->buf[len], map_page[1], cnt - len);
Steven Rostedtd696b582011-09-22 11:50:27 -04005146 } else
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005147 memcpy(&entry->buf, map_page[0] + offset, cnt);
Steven Rostedtd696b582011-09-22 11:50:27 -04005148
5149 if (entry->buf[cnt - 1] != '\n') {
5150 entry->buf[cnt] = '\n';
5151 entry->buf[cnt + 1] = '\0';
5152 } else
5153 entry->buf[cnt] = '\0';
5154
Steven Rostedt7ffbd482012-10-11 12:14:25 -04005155 __buffer_unlock_commit(buffer, event);
Steven Rostedtd696b582011-09-22 11:50:27 -04005156
5157 written = cnt;
5158
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02005159 *fpos += written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005160
Steven Rostedtd696b582011-09-22 11:50:27 -04005161 out_unlock:
Vikram Mulukutla72158532014-12-17 18:50:56 -08005162 for (i = nr_pages - 1; i >= 0; i--) {
Steven Rostedt6edb2a82012-05-11 23:28:49 -04005163 kunmap_atomic(map_page[i]);
5164 put_page(pages[i]);
5165 }
Steven Rostedtd696b582011-09-22 11:50:27 -04005166 out:
Marcin Slusarz1aa54bc2010-07-28 01:18:01 +02005167 return written;
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005168}
5169
Li Zefan13f16d22009-12-08 11:16:11 +08005170static int tracing_clock_show(struct seq_file *m, void *v)
Zhaolei5079f322009-08-25 16:12:56 +08005171{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005172 struct trace_array *tr = m->private;
Zhaolei5079f322009-08-25 16:12:56 +08005173 int i;
5174
5175 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
Li Zefan13f16d22009-12-08 11:16:11 +08005176 seq_printf(m,
Zhaolei5079f322009-08-25 16:12:56 +08005177 "%s%s%s%s", i ? " " : "",
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005178 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5179 i == tr->clock_id ? "]" : "");
Li Zefan13f16d22009-12-08 11:16:11 +08005180 seq_putc(m, '\n');
Zhaolei5079f322009-08-25 16:12:56 +08005181
Li Zefan13f16d22009-12-08 11:16:11 +08005182 return 0;
Zhaolei5079f322009-08-25 16:12:56 +08005183}
5184
Steven Rostedte1e232c2014-02-10 23:38:46 -05005185static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
Zhaolei5079f322009-08-25 16:12:56 +08005186{
Zhaolei5079f322009-08-25 16:12:56 +08005187 int i;
5188
Zhaolei5079f322009-08-25 16:12:56 +08005189 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5190 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5191 break;
5192 }
5193 if (i == ARRAY_SIZE(trace_clocks))
5194 return -EINVAL;
5195
Zhaolei5079f322009-08-25 16:12:56 +08005196 mutex_lock(&trace_types_lock);
5197
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005198 tr->clock_id = i;
5199
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005200 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
Zhaolei5079f322009-08-25 16:12:56 +08005201
David Sharp60303ed2012-10-11 16:27:52 -07005202 /*
5203 * New clock may not be consistent with the previous clock.
5204 * Reset the buffer so that it doesn't have incomparable timestamps.
5205 */
Alexander Z Lam94571582013-08-02 18:36:16 -07005206 tracing_reset_online_cpus(&tr->trace_buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005207
5208#ifdef CONFIG_TRACER_MAX_TRACE
5209 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
5210 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
Alexander Z Lam94571582013-08-02 18:36:16 -07005211 tracing_reset_online_cpus(&tr->max_buffer);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005212#endif
David Sharp60303ed2012-10-11 16:27:52 -07005213
Zhaolei5079f322009-08-25 16:12:56 +08005214 mutex_unlock(&trace_types_lock);
5215
Steven Rostedte1e232c2014-02-10 23:38:46 -05005216 return 0;
5217}
5218
5219static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5220 size_t cnt, loff_t *fpos)
5221{
5222 struct seq_file *m = filp->private_data;
5223 struct trace_array *tr = m->private;
5224 char buf[64];
5225 const char *clockstr;
5226 int ret;
5227
5228 if (cnt >= sizeof(buf))
5229 return -EINVAL;
5230
5231 if (copy_from_user(&buf, ubuf, cnt))
5232 return -EFAULT;
5233
5234 buf[cnt] = 0;
5235
5236 clockstr = strstrip(buf);
5237
5238 ret = tracing_set_clock(tr, clockstr);
5239 if (ret)
5240 return ret;
5241
Zhaolei5079f322009-08-25 16:12:56 +08005242 *fpos += cnt;
5243
5244 return cnt;
5245}
5246
Li Zefan13f16d22009-12-08 11:16:11 +08005247static int tracing_clock_open(struct inode *inode, struct file *file)
5248{
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005249 struct trace_array *tr = inode->i_private;
5250 int ret;
5251
Li Zefan13f16d22009-12-08 11:16:11 +08005252 if (tracing_disabled)
5253 return -ENODEV;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005254
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005255 if (trace_array_get(tr))
5256 return -ENODEV;
5257
5258 ret = single_open(file, tracing_clock_show, inode->i_private);
5259 if (ret < 0)
5260 trace_array_put(tr);
5261
5262 return ret;
Li Zefan13f16d22009-12-08 11:16:11 +08005263}
5264
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005265struct ftrace_buffer_info {
5266 struct trace_iterator iter;
5267 void *spare;
5268 unsigned int read;
5269};
5270
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005271#ifdef CONFIG_TRACER_SNAPSHOT
5272static int tracing_snapshot_open(struct inode *inode, struct file *file)
5273{
Oleg Nesterov6484c712013-07-23 17:26:10 +02005274 struct trace_array *tr = inode->i_private;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005275 struct trace_iterator *iter;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005276 struct seq_file *m;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005277 int ret = 0;
5278
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005279 if (trace_array_get(tr) < 0)
5280 return -ENODEV;
5281
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005282 if (file->f_mode & FMODE_READ) {
Oleg Nesterov6484c712013-07-23 17:26:10 +02005283 iter = __tracing_open(inode, file, true);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005284 if (IS_ERR(iter))
5285 ret = PTR_ERR(iter);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005286 } else {
5287 /* Writes still need the seq_file to hold the private data */
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005288 ret = -ENOMEM;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005289 m = kzalloc(sizeof(*m), GFP_KERNEL);
5290 if (!m)
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005291 goto out;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005292 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5293 if (!iter) {
5294 kfree(m);
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005295 goto out;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005296 }
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005297 ret = 0;
5298
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005299 iter->tr = tr;
Oleg Nesterov6484c712013-07-23 17:26:10 +02005300 iter->trace_buffer = &tr->max_buffer;
5301 iter->cpu_file = tracing_get_cpu(inode);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005302 m->private = iter;
5303 file->private_data = m;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005304 }
Alexander Z Lamf77d09a2013-07-18 11:18:44 -07005305out:
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005306 if (ret < 0)
5307 trace_array_put(tr);
5308
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005309 return ret;
5310}
5311
5312static ssize_t
5313tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5314 loff_t *ppos)
5315{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005316 struct seq_file *m = filp->private_data;
5317 struct trace_iterator *iter = m->private;
5318 struct trace_array *tr = iter->tr;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005319 unsigned long val;
5320 int ret;
5321
5322 ret = tracing_update_buffers();
5323 if (ret < 0)
5324 return ret;
5325
5326 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5327 if (ret)
5328 return ret;
5329
5330 mutex_lock(&trace_types_lock);
5331
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005332 if (tr->current_trace->use_max_tr) {
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005333 ret = -EBUSY;
5334 goto out;
5335 }
5336
5337 switch (val) {
5338 case 0:
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005339 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5340 ret = -EINVAL;
5341 break;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005342 }
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04005343 if (tr->allocated_snapshot)
5344 free_snapshot(tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005345 break;
5346 case 1:
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005347/* Only allow per-cpu swap if the ring buffer supports it */
5348#ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5349 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5350 ret = -EINVAL;
5351 break;
5352 }
5353#endif
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05005354 if (!tr->allocated_snapshot) {
Steven Rostedt (Red Hat)3209cff2013-03-12 11:17:54 -04005355 ret = alloc_snapshot(tr);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005356 if (ret < 0)
5357 break;
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005358 }
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005359 local_irq_disable();
5360 /* Now, we're going to swap */
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005361 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05005362 update_max_tr(tr, current, smp_processor_id());
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005363 else
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05005364 update_max_tr_single(tr, current, iter->cpu_file);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005365 local_irq_enable();
5366 break;
5367 default:
Steven Rostedt (Red Hat)45ad21c2013-03-05 18:25:02 -05005368 if (tr->allocated_snapshot) {
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05005369 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5370 tracing_reset_online_cpus(&tr->max_buffer);
5371 else
5372 tracing_reset(&tr->max_buffer, iter->cpu_file);
5373 }
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005374 break;
5375 }
5376
5377 if (ret >= 0) {
5378 *ppos += cnt;
5379 ret = cnt;
5380 }
5381out:
5382 mutex_unlock(&trace_types_lock);
5383 return ret;
5384}
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005385
5386static int tracing_snapshot_release(struct inode *inode, struct file *file)
5387{
5388 struct seq_file *m = file->private_data;
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005389 int ret;
5390
5391 ret = tracing_release(inode, file);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005392
5393 if (file->f_mode & FMODE_READ)
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005394 return ret;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005395
5396 /* If write only, the seq_file is just a stub */
5397 if (m)
5398 kfree(m->private);
5399 kfree(m);
5400
5401 return 0;
5402}
5403
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005404static int tracing_buffers_open(struct inode *inode, struct file *filp);
5405static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5406 size_t count, loff_t *ppos);
5407static int tracing_buffers_release(struct inode *inode, struct file *file);
5408static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5409 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5410
5411static int snapshot_raw_open(struct inode *inode, struct file *filp)
5412{
5413 struct ftrace_buffer_info *info;
5414 int ret;
5415
5416 ret = tracing_buffers_open(inode, filp);
5417 if (ret < 0)
5418 return ret;
5419
5420 info = filp->private_data;
5421
5422 if (info->iter.trace->use_max_tr) {
5423 tracing_buffers_release(inode, filp);
5424 return -EBUSY;
5425 }
5426
5427 info->iter.snapshot = true;
5428 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5429
5430 return ret;
5431}
5432
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005433#endif /* CONFIG_TRACER_SNAPSHOT */
5434
5435
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04005436static const struct file_operations tracing_thresh_fops = {
5437 .open = tracing_open_generic,
5438 .read = tracing_thresh_read,
5439 .write = tracing_thresh_write,
5440 .llseek = generic_file_llseek,
5441};
5442
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005443static const struct file_operations tracing_max_lat_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005444 .open = tracing_open_generic,
5445 .read = tracing_max_lat_read,
5446 .write = tracing_max_lat_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005447 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005448};
5449
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005450static const struct file_operations set_tracer_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005451 .open = tracing_open_generic,
5452 .read = tracing_set_trace_read,
5453 .write = tracing_set_trace_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005454 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005455};
5456
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005457static const struct file_operations tracing_pipe_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005458 .open = tracing_open_pipe,
Soeren Sandmann Pedersen2a2cc8f2008-05-12 21:20:49 +02005459 .poll = tracing_poll_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005460 .read = tracing_read_pipe,
Eduard - Gabriel Munteanu3c568192009-02-09 08:15:56 +02005461 .splice_read = tracing_splice_read_pipe,
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005462 .release = tracing_release_pipe,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005463 .llseek = no_llseek,
Steven Rostedtb3806b42008-05-12 21:20:46 +02005464};
5465
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005466static const struct file_operations tracing_entries_fops = {
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005467 .open = tracing_open_generic_tr,
Steven Rostedta98a3c32008-05-12 21:20:59 +02005468 .read = tracing_entries_read,
5469 .write = tracing_entries_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005470 .llseek = generic_file_llseek,
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02005471 .release = tracing_release_generic_tr,
Steven Rostedta98a3c32008-05-12 21:20:59 +02005472};
5473
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005474static const struct file_operations tracing_total_entries_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005475 .open = tracing_open_generic_tr,
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005476 .read = tracing_total_entries_read,
5477 .llseek = generic_file_llseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005478 .release = tracing_release_generic_tr,
Vaibhav Nagarnaikf81ab072011-08-16 14:46:15 -07005479};
5480
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005481static const struct file_operations tracing_free_buffer_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005482 .open = tracing_open_generic_tr,
Vaibhav Nagarnaik4f271a22011-06-13 17:51:57 -07005483 .write = tracing_free_buffer_write,
5484 .release = tracing_free_buffer_release,
5485};
5486
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005487static const struct file_operations tracing_mark_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005488 .open = tracing_open_generic_tr,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005489 .write = tracing_mark_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005490 .llseek = generic_file_llseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005491 .release = tracing_release_generic_tr,
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03005492};
5493
Zhaolei5079f322009-08-25 16:12:56 +08005494static const struct file_operations trace_clock_fops = {
Li Zefan13f16d22009-12-08 11:16:11 +08005495 .open = tracing_clock_open,
5496 .read = seq_read,
5497 .llseek = seq_lseek,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005498 .release = tracing_single_release_tr,
Zhaolei5079f322009-08-25 16:12:56 +08005499 .write = tracing_clock_write,
5500};
5501
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005502#ifdef CONFIG_TRACER_SNAPSHOT
5503static const struct file_operations snapshot_fops = {
5504 .open = tracing_snapshot_open,
5505 .read = seq_read,
5506 .write = tracing_snapshot_write,
Steven Rostedt (Red Hat)098c8792013-12-21 17:39:40 -05005507 .llseek = tracing_lseek,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04005508 .release = tracing_snapshot_release,
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005509};
Hiraku Toyookadebdd572012-12-26 11:53:00 +09005510
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005511static const struct file_operations snapshot_raw_fops = {
5512 .open = snapshot_raw_open,
5513 .read = tracing_buffers_read,
5514 .release = tracing_buffers_release,
5515 .splice_read = tracing_buffers_splice_read,
5516 .llseek = no_llseek,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005517};
5518
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005519#endif /* CONFIG_TRACER_SNAPSHOT */
5520
Steven Rostedt2cadf912008-12-01 22:20:19 -05005521static int tracing_buffers_open(struct inode *inode, struct file *filp)
5522{
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02005523 struct trace_array *tr = inode->i_private;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005524 struct ftrace_buffer_info *info;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005525 int ret;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005526
5527 if (tracing_disabled)
5528 return -ENODEV;
5529
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005530 if (trace_array_get(tr) < 0)
5531 return -ENODEV;
5532
Steven Rostedt2cadf912008-12-01 22:20:19 -05005533 info = kzalloc(sizeof(*info), GFP_KERNEL);
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005534 if (!info) {
5535 trace_array_put(tr);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005536 return -ENOMEM;
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005537 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05005538
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005539 mutex_lock(&trace_types_lock);
5540
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005541 info->iter.tr = tr;
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02005542 info->iter.cpu_file = tracing_get_cpu(inode);
Steven Rostedtb6273442013-02-28 13:44:11 -05005543 info->iter.trace = tr->current_trace;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005544 info->iter.trace_buffer = &tr->trace_buffer;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005545 info->spare = NULL;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005546 /* Force reading ring buffer for first read */
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005547 info->read = (unsigned int)-1;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005548
5549 filp->private_data = info;
5550
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05005551 tr->current_trace->ref++;
5552
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005553 mutex_unlock(&trace_types_lock);
5554
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04005555 ret = nonseekable_open(inode, filp);
5556 if (ret < 0)
5557 trace_array_put(tr);
5558
5559 return ret;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005560}
5561
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005562static unsigned int
5563tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5564{
5565 struct ftrace_buffer_info *info = filp->private_data;
5566 struct trace_iterator *iter = &info->iter;
5567
5568 return trace_poll(iter, filp, poll_table);
5569}
5570
Steven Rostedt2cadf912008-12-01 22:20:19 -05005571static ssize_t
5572tracing_buffers_read(struct file *filp, char __user *ubuf,
5573 size_t count, loff_t *ppos)
5574{
5575 struct ftrace_buffer_info *info = filp->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005576 struct trace_iterator *iter = &info->iter;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005577 ssize_t ret;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005578 ssize_t size;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005579
Steven Rostedt2dc5d122009-03-04 19:10:05 -05005580 if (!count)
5581 return 0;
5582
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005583#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005584 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5585 return -EBUSY;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005586#endif
5587
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005588 if (!info->spare)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005589 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5590 iter->cpu_file);
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005591 if (!info->spare)
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005592 return -ENOMEM;
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005593
Steven Rostedt2cadf912008-12-01 22:20:19 -05005594 /* Do we have previous read data to read? */
5595 if (info->read < PAGE_SIZE)
5596 goto read;
5597
Steven Rostedtb6273442013-02-28 13:44:11 -05005598 again:
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005599 trace_access_lock(iter->cpu_file);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005600 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005601 &info->spare,
5602 count,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005603 iter->cpu_file, 0);
5604 trace_access_unlock(iter->cpu_file);
Steven Rostedtb6273442013-02-28 13:44:11 -05005605
5606 if (ret < 0) {
5607 if (trace_empty(iter)) {
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005608 if ((filp->f_flags & O_NONBLOCK))
5609 return -EAGAIN;
5610
Rabin Vincente30f53a2014-11-10 19:46:34 +01005611 ret = wait_on_pipe(iter, false);
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005612 if (ret)
5613 return ret;
5614
Steven Rostedtb6273442013-02-28 13:44:11 -05005615 goto again;
5616 }
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005617 return 0;
Steven Rostedtb6273442013-02-28 13:44:11 -05005618 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05005619
Steven Rostedt436fc282011-10-14 10:44:25 -04005620 info->read = 0;
Steven Rostedtb6273442013-02-28 13:44:11 -05005621 read:
Steven Rostedt2cadf912008-12-01 22:20:19 -05005622 size = PAGE_SIZE - info->read;
5623 if (size > count)
5624 size = count;
5625
5626 ret = copy_to_user(ubuf, info->spare + info->read, size);
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005627 if (ret == size)
5628 return -EFAULT;
5629
Steven Rostedt2dc5d122009-03-04 19:10:05 -05005630 size -= ret;
5631
Steven Rostedt2cadf912008-12-01 22:20:19 -05005632 *ppos += size;
5633 info->read += size;
5634
5635 return size;
5636}
5637
5638static int tracing_buffers_release(struct inode *inode, struct file *file)
5639{
5640 struct ftrace_buffer_info *info = file->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005641 struct trace_iterator *iter = &info->iter;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005642
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005643 mutex_lock(&trace_types_lock);
5644
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05005645 iter->tr->current_trace->ref--;
5646
Steven Rostedt (Red Hat)ff451962013-07-01 22:50:29 -04005647 __trace_array_put(iter->tr);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005648
Lai Jiangshanddd538f2009-04-02 15:16:59 +08005649 if (info->spare)
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005650 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005651 kfree(info);
5652
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05005653 mutex_unlock(&trace_types_lock);
5654
Steven Rostedt2cadf912008-12-01 22:20:19 -05005655 return 0;
5656}
5657
5658struct buffer_ref {
5659 struct ring_buffer *buffer;
5660 void *page;
5661 int ref;
5662};
5663
5664static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5665 struct pipe_buffer *buf)
5666{
5667 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5668
5669 if (--ref->ref)
5670 return;
5671
5672 ring_buffer_free_read_page(ref->buffer, ref->page);
5673 kfree(ref);
5674 buf->private = 0;
5675}
5676
Steven Rostedt2cadf912008-12-01 22:20:19 -05005677static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5678 struct pipe_buffer *buf)
5679{
5680 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5681
5682 ref->ref++;
5683}
5684
5685/* Pipe buffer operations for a buffer. */
Alexey Dobriyan28dfef82009-12-15 16:46:48 -08005686static const struct pipe_buf_operations buffer_pipe_buf_ops = {
Steven Rostedt2cadf912008-12-01 22:20:19 -05005687 .can_merge = 0,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005688 .confirm = generic_pipe_buf_confirm,
5689 .release = buffer_pipe_buf_release,
Masami Hiramatsud55cb6c2012-08-09 21:31:10 +09005690 .steal = generic_pipe_buf_steal,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005691 .get = buffer_pipe_buf_get,
5692};
5693
5694/*
5695 * Callback from splice_to_pipe(), if we need to release some pages
5696 * at the end of the spd in case we error'ed out in filling the pipe.
5697 */
5698static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5699{
5700 struct buffer_ref *ref =
5701 (struct buffer_ref *)spd->partial[i].private;
5702
5703 if (--ref->ref)
5704 return;
5705
5706 ring_buffer_free_read_page(ref->buffer, ref->page);
5707 kfree(ref);
5708 spd->partial[i].private = 0;
5709}
5710
5711static ssize_t
5712tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5713 struct pipe_inode_info *pipe, size_t len,
5714 unsigned int flags)
5715{
5716 struct ftrace_buffer_info *info = file->private_data;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005717 struct trace_iterator *iter = &info->iter;
Jens Axboe35f3d142010-05-20 10:43:18 +02005718 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5719 struct page *pages_def[PIPE_DEF_BUFFERS];
Steven Rostedt2cadf912008-12-01 22:20:19 -05005720 struct splice_pipe_desc spd = {
Jens Axboe35f3d142010-05-20 10:43:18 +02005721 .pages = pages_def,
5722 .partial = partial_def,
Eric Dumazet047fe362012-06-12 15:24:40 +02005723 .nr_pages_max = PIPE_DEF_BUFFERS,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005724 .flags = flags,
5725 .ops = &buffer_pipe_buf_ops,
5726 .spd_release = buffer_spd_release,
5727 };
5728 struct buffer_ref *ref;
Steven Rostedt93459c62009-04-29 00:23:13 -04005729 int entries, size, i;
Rabin Vincent07906da2014-11-06 22:26:07 +01005730 ssize_t ret = 0;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005731
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005732#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005733 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5734 return -EBUSY;
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005735#endif
5736
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005737 if (splice_grow_spd(pipe, &spd))
5738 return -ENOMEM;
Jens Axboe35f3d142010-05-20 10:43:18 +02005739
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005740 if (*ppos & (PAGE_SIZE - 1))
5741 return -EINVAL;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005742
5743 if (len & (PAGE_SIZE - 1)) {
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005744 if (len < PAGE_SIZE)
5745 return -EINVAL;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005746 len &= PAGE_MASK;
5747 }
5748
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005749 again:
5750 trace_access_lock(iter->cpu_file);
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005751 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt93459c62009-04-29 00:23:13 -04005752
Al Viroa786c062014-04-11 12:01:03 -04005753 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
Steven Rostedt2cadf912008-12-01 22:20:19 -05005754 struct page *page;
5755 int r;
5756
5757 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
Rabin Vincent07906da2014-11-06 22:26:07 +01005758 if (!ref) {
5759 ret = -ENOMEM;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005760 break;
Rabin Vincent07906da2014-11-06 22:26:07 +01005761 }
Steven Rostedt2cadf912008-12-01 22:20:19 -05005762
Steven Rostedt7267fa62009-04-29 00:16:21 -04005763 ref->ref = 1;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005764 ref->buffer = iter->trace_buffer->buffer;
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005765 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005766 if (!ref->page) {
Rabin Vincent07906da2014-11-06 22:26:07 +01005767 ret = -ENOMEM;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005768 kfree(ref);
5769 break;
5770 }
5771
5772 r = ring_buffer_read_page(ref->buffer, &ref->page,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005773 len, iter->cpu_file, 1);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005774 if (r < 0) {
Vaibhav Nagarnaik7ea59062011-05-03 17:56:42 -07005775 ring_buffer_free_read_page(ref->buffer, ref->page);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005776 kfree(ref);
5777 break;
5778 }
5779
5780 /*
5781 * zero out any left over data, this is going to
5782 * user land.
5783 */
5784 size = ring_buffer_page_len(ref->page);
5785 if (size < PAGE_SIZE)
5786 memset(ref->page + size, 0, PAGE_SIZE - size);
5787
5788 page = virt_to_page(ref->page);
5789
5790 spd.pages[i] = page;
5791 spd.partial[i].len = PAGE_SIZE;
5792 spd.partial[i].offset = 0;
5793 spd.partial[i].private = (unsigned long)ref;
5794 spd.nr_pages++;
Lai Jiangshan93cfb3c2009-04-02 15:17:08 +08005795 *ppos += PAGE_SIZE;
Steven Rostedt93459c62009-04-29 00:23:13 -04005796
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005797 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005798 }
5799
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005800 trace_access_unlock(iter->cpu_file);
Steven Rostedt2cadf912008-12-01 22:20:19 -05005801 spd.nr_pages = i;
5802
5803 /* did we read anything? */
5804 if (!spd.nr_pages) {
Rabin Vincent07906da2014-11-06 22:26:07 +01005805 if (ret)
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005806 return ret;
Rabin Vincent07906da2014-11-06 22:26:07 +01005807
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005808 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
5809 return -EAGAIN;
5810
Rabin Vincente30f53a2014-11-10 19:46:34 +01005811 ret = wait_on_pipe(iter, true);
Steven Rostedt (Red Hat)8b8b3682014-06-10 09:46:00 -04005812 if (ret)
Steven Rostedt (Red Hat)d716ff72014-12-15 22:31:07 -05005813 return ret;
Rabin Vincente30f53a2014-11-10 19:46:34 +01005814
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005815 goto again;
Steven Rostedt2cadf912008-12-01 22:20:19 -05005816 }
5817
5818 ret = splice_to_pipe(pipe, &spd);
Eric Dumazet047fe362012-06-12 15:24:40 +02005819 splice_shrink_spd(&spd);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05005820
Steven Rostedt2cadf912008-12-01 22:20:19 -05005821 return ret;
5822}
5823
5824static const struct file_operations tracing_buffers_fops = {
5825 .open = tracing_buffers_open,
5826 .read = tracing_buffers_read,
Steven Rostedtcc60cdc2013-02-28 09:17:16 -05005827 .poll = tracing_buffers_poll,
Steven Rostedt2cadf912008-12-01 22:20:19 -05005828 .release = tracing_buffers_release,
5829 .splice_read = tracing_buffers_splice_read,
5830 .llseek = no_llseek,
5831};
5832
Steven Rostedtc8d77182009-04-29 18:03:45 -04005833static ssize_t
5834tracing_stats_read(struct file *filp, char __user *ubuf,
5835 size_t count, loff_t *ppos)
5836{
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005837 struct inode *inode = file_inode(filp);
5838 struct trace_array *tr = inode->i_private;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005839 struct trace_buffer *trace_buf = &tr->trace_buffer;
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005840 int cpu = tracing_get_cpu(inode);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005841 struct trace_seq *s;
5842 unsigned long cnt;
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005843 unsigned long long t;
5844 unsigned long usec_rem;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005845
Li Zefane4f2d102009-06-15 10:57:28 +08005846 s = kmalloc(sizeof(*s), GFP_KERNEL);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005847 if (!s)
Roel Kluina6463652009-11-11 22:26:35 +01005848 return -ENOMEM;
Steven Rostedtc8d77182009-04-29 18:03:45 -04005849
5850 trace_seq_init(s);
5851
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005852 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005853 trace_seq_printf(s, "entries: %ld\n", cnt);
5854
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005855 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005856 trace_seq_printf(s, "overrun: %ld\n", cnt);
5857
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005858 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
Steven Rostedtc8d77182009-04-29 18:03:45 -04005859 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5860
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005861 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005862 trace_seq_printf(s, "bytes: %ld\n", cnt);
5863
Yoshihiro YUNOMAE58e8eed2013-04-23 10:32:39 +09005864 if (trace_clocks[tr->clock_id].in_ns) {
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005865 /* local or global for trace_clock */
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005866 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005867 usec_rem = do_div(t, USEC_PER_SEC);
5868 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5869 t, usec_rem);
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005870
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005871 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005872 usec_rem = do_div(t, USEC_PER_SEC);
5873 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5874 } else {
5875 /* counter or tsc mode for trace_clock */
5876 trace_seq_printf(s, "oldest event ts: %llu\n",
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005877 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005878
5879 trace_seq_printf(s, "now ts: %llu\n",
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005880 ring_buffer_time_stamp(trace_buf->buffer, cpu));
Yoshihiro YUNOMAE11043d82012-11-13 12:18:23 -08005881 }
Vaibhav Nagarnaikc64e1482011-08-16 14:46:16 -07005882
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005883 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
Slava Pestov884bfe82011-07-15 14:23:58 -07005884 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5885
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05005886 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
Steven Rostedt (Red Hat)ad964702013-01-29 17:45:49 -05005887 trace_seq_printf(s, "read events: %ld\n", cnt);
5888
Steven Rostedt (Red Hat)5ac483782014-11-14 15:49:41 -05005889 count = simple_read_from_buffer(ubuf, count, ppos,
5890 s->buffer, trace_seq_used(s));
Steven Rostedtc8d77182009-04-29 18:03:45 -04005891
5892 kfree(s);
5893
5894 return count;
5895}
5896
5897static const struct file_operations tracing_stats_fops = {
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005898 .open = tracing_open_generic_tr,
Steven Rostedtc8d77182009-04-29 18:03:45 -04005899 .read = tracing_stats_read,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005900 .llseek = generic_file_llseek,
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02005901 .release = tracing_release_generic_tr,
Steven Rostedtc8d77182009-04-29 18:03:45 -04005902};
5903
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005904#ifdef CONFIG_DYNAMIC_FTRACE
5905
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005906int __weak ftrace_arch_read_dyn_info(char *buf, int size)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005907{
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005908 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005909}
5910
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005911static ssize_t
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005912tracing_read_dyn_info(struct file *filp, char __user *ubuf,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005913 size_t cnt, loff_t *ppos)
5914{
Steven Rostedta26a2a22008-10-31 00:03:22 -04005915 static char ftrace_dyn_info_buffer[1024];
5916 static DEFINE_MUTEX(dyn_info_mutex);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005917 unsigned long *p = filp->private_data;
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005918 char *buf = ftrace_dyn_info_buffer;
Steven Rostedta26a2a22008-10-31 00:03:22 -04005919 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005920 int r;
5921
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005922 mutex_lock(&dyn_info_mutex);
5923 r = sprintf(buf, "%ld ", *p);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005924
Steven Rostedta26a2a22008-10-31 00:03:22 -04005925 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005926 buf[r++] = '\n';
5927
5928 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5929
5930 mutex_unlock(&dyn_info_mutex);
5931
5932 return r;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005933}
5934
Steven Rostedt5e2336a2009-03-05 21:44:55 -05005935static const struct file_operations tracing_dyn_info_fops = {
Ingo Molnar4bf39a92008-05-12 21:20:46 +02005936 .open = tracing_open_generic,
Steven Rostedtb807c3d2008-10-30 16:08:33 -04005937 .read = tracing_read_dyn_info,
Arnd Bergmannb4447862010-07-07 23:40:11 +02005938 .llseek = generic_file_llseek,
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005939};
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005940#endif /* CONFIG_DYNAMIC_FTRACE */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005941
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005942#if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5943static void
5944ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005945{
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005946 tracing_snapshot();
5947}
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02005948
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005949static void
5950ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5951{
5952 unsigned long *count = (long *)data;
5953
5954 if (!*count)
5955 return;
5956
5957 if (*count != -1)
5958 (*count)--;
5959
5960 tracing_snapshot();
5961}
5962
5963static int
5964ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5965 struct ftrace_probe_ops *ops, void *data)
5966{
5967 long count = (long)data;
5968
5969 seq_printf(m, "%ps:", (void *)ip);
5970
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01005971 seq_puts(m, "snapshot");
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005972
5973 if (count == -1)
Rasmus Villemoesfa6f0cc2014-11-08 21:42:10 +01005974 seq_puts(m, ":unlimited\n");
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04005975 else
5976 seq_printf(m, ":count=%ld\n", count);
5977
5978 return 0;
5979}
5980
5981static struct ftrace_probe_ops snapshot_probe_ops = {
5982 .func = ftrace_snapshot,
5983 .print = ftrace_snapshot_print,
5984};
5985
5986static struct ftrace_probe_ops snapshot_count_probe_ops = {
5987 .func = ftrace_count_snapshot,
5988 .print = ftrace_snapshot_print,
5989};
5990
5991static int
5992ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5993 char *glob, char *cmd, char *param, int enable)
5994{
5995 struct ftrace_probe_ops *ops;
5996 void *count = (void *)-1;
5997 char *number;
5998 int ret;
5999
6000 /* hash funcs only work with set_ftrace_filter */
6001 if (!enable)
6002 return -EINVAL;
6003
6004 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6005
6006 if (glob[0] == '!') {
6007 unregister_ftrace_function_probe_func(glob+1, ops);
6008 return 0;
6009 }
6010
6011 if (!param)
6012 goto out_reg;
6013
6014 number = strsep(&param, ":");
6015
6016 if (!strlen(number))
6017 goto out_reg;
6018
6019 /*
6020 * We use the callback data field (which is a pointer)
6021 * as our counter.
6022 */
6023 ret = kstrtoul(number, 0, (unsigned long *)&count);
6024 if (ret)
6025 return ret;
6026
6027 out_reg:
6028 ret = register_ftrace_function_probe(glob, ops, count);
6029
6030 if (ret >= 0)
6031 alloc_snapshot(&global_trace);
6032
6033 return ret < 0 ? ret : 0;
6034}
6035
6036static struct ftrace_func_command ftrace_snapshot_cmd = {
6037 .name = "snapshot",
6038 .func = ftrace_trace_snapshot_callback,
6039};
6040
Tom Zanussi38de93a2013-10-24 08:34:18 -05006041static __init int register_snapshot_cmd(void)
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04006042{
6043 return register_ftrace_command(&ftrace_snapshot_cmd);
6044}
6045#else
Tom Zanussi38de93a2013-10-24 08:34:18 -05006046static inline __init int register_snapshot_cmd(void) { return 0; }
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04006047#endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006048
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006049static struct dentry *tracing_get_dentry(struct trace_array *tr)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006050{
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006051 if (WARN_ON(!tr->dir))
6052 return ERR_PTR(-ENODEV);
6053
6054 /* Top directory uses NULL as the parent */
6055 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6056 return NULL;
6057
6058 /* All sub buffers have a descriptor */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006059 return tr->dir;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006060}
6061
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006062static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6063{
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006064 struct dentry *d_tracer;
6065
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006066 if (tr->percpu_dir)
6067 return tr->percpu_dir;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006068
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006069 d_tracer = tracing_get_dentry(tr);
Steven Rostedt (Red Hat)14a5ae42015-01-20 11:14:16 -05006070 if (IS_ERR(d_tracer))
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006071 return NULL;
6072
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006073 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006074
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006075 WARN_ONCE(!tr->percpu_dir,
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006076 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006077
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006078 return tr->percpu_dir;
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006079}
6080
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006081static struct dentry *
6082trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6083 void *data, long cpu, const struct file_operations *fops)
6084{
6085 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6086
6087 if (ret) /* See tracing_get_cpu() */
David Howells7682c912015-03-17 22:26:16 +00006088 d_inode(ret)->i_cdev = (void *)(cpu + 1);
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006089 return ret;
6090}
6091
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006092static void
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006093tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006094{
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006095 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006096 struct dentry *d_cpu;
Steven Rostedtdd49a382010-10-20 21:51:26 -04006097 char cpu_dir[30]; /* 30 characters should be more than enough */
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006098
Namhyung Kim0a3d7ce2012-04-23 10:11:57 +09006099 if (!d_percpu)
6100 return;
6101
Steven Rostedtdd49a382010-10-20 21:51:26 -04006102 snprintf(cpu_dir, 30, "cpu%ld", cpu);
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006103 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01006104 if (!d_cpu) {
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006105 pr_warning("Could not create tracefs '%s' entry\n", cpu_dir);
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01006106 return;
6107 }
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006108
Frederic Weisbecker8656e7a2009-02-26 00:41:38 +01006109 /* per cpu trace_pipe */
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006110 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
Oleg Nesterov15544202013-07-23 17:25:57 +02006111 tr, cpu, &tracing_pipe_fops);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006112
6113 /* per cpu trace */
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006114 trace_create_cpu_file("trace", 0644, d_cpu,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006115 tr, cpu, &tracing_fops);
Steven Rostedt7f96f932009-03-13 00:37:42 -04006116
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006117 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02006118 tr, cpu, &tracing_buffers_fops);
Steven Rostedtc8d77182009-04-29 18:03:45 -04006119
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006120 trace_create_cpu_file("stats", 0444, d_cpu,
Oleg Nesterov4d3435b2013-07-23 17:26:03 +02006121 tr, cpu, &tracing_stats_fops);
Vaibhav Nagarnaik438ced12012-02-02 12:00:41 -08006122
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006123 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02006124 tr, cpu, &tracing_entries_fops);
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05006125
6126#ifdef CONFIG_TRACER_SNAPSHOT
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006127 trace_create_cpu_file("snapshot", 0644, d_cpu,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006128 tr, cpu, &snapshot_fops);
Steven Rostedt (Red Hat)6de58e62013-03-05 16:18:16 -05006129
Oleg Nesterov649e9c72013-07-23 17:25:54 +02006130 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
Oleg Nesterov46ef2be2013-07-23 17:26:00 +02006131 tr, cpu, &snapshot_raw_fops);
Steven Rostedt (Red Hat)f1affca2013-03-05 14:35:11 -05006132#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006133}
6134
Steven Rostedt60a11772008-05-12 21:20:44 +02006135#ifdef CONFIG_FTRACE_SELFTEST
6136/* Let selftest have access to static functions in this file */
6137#include "trace_selftest.c"
6138#endif
6139
Steven Rostedt577b7852009-02-26 23:43:05 -05006140static ssize_t
6141trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6142 loff_t *ppos)
6143{
6144 struct trace_option_dentry *topt = filp->private_data;
6145 char *buf;
6146
6147 if (topt->flags->val & topt->opt->bit)
6148 buf = "1\n";
6149 else
6150 buf = "0\n";
6151
6152 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6153}
6154
6155static ssize_t
6156trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6157 loff_t *ppos)
6158{
6159 struct trace_option_dentry *topt = filp->private_data;
6160 unsigned long val;
Steven Rostedt577b7852009-02-26 23:43:05 -05006161 int ret;
6162
Peter Huewe22fe9b52011-06-07 21:58:27 +02006163 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6164 if (ret)
Steven Rostedt577b7852009-02-26 23:43:05 -05006165 return ret;
6166
Li Zefan8d18eaa2009-12-08 11:17:06 +08006167 if (val != 0 && val != 1)
Steven Rostedt577b7852009-02-26 23:43:05 -05006168 return -EINVAL;
Li Zefan8d18eaa2009-12-08 11:17:06 +08006169
6170 if (!!(topt->flags->val & topt->opt->bit) != val) {
6171 mutex_lock(&trace_types_lock);
Steven Rostedt (Red Hat)8c1a49a2014-01-10 11:13:54 -05006172 ret = __set_tracer_option(topt->tr, topt->flags,
Steven Rostedtc757bea2009-12-21 22:35:16 -05006173 topt->opt, !val);
Li Zefan8d18eaa2009-12-08 11:17:06 +08006174 mutex_unlock(&trace_types_lock);
6175 if (ret)
6176 return ret;
Steven Rostedt577b7852009-02-26 23:43:05 -05006177 }
6178
6179 *ppos += cnt;
6180
6181 return cnt;
6182}
6183
6184
6185static const struct file_operations trace_options_fops = {
6186 .open = tracing_open_generic,
6187 .read = trace_options_read,
6188 .write = trace_options_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02006189 .llseek = generic_file_llseek,
Steven Rostedt577b7852009-02-26 23:43:05 -05006190};
6191
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006192/*
6193 * In order to pass in both the trace_array descriptor as well as the index
6194 * to the flag that the trace option file represents, the trace_array
6195 * has a character array of trace_flags_index[], which holds the index
6196 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6197 * The address of this character array is passed to the flag option file
6198 * read/write callbacks.
6199 *
6200 * In order to extract both the index and the trace_array descriptor,
6201 * get_tr_index() uses the following algorithm.
6202 *
6203 * idx = *ptr;
6204 *
6205 * As the pointer itself contains the address of the index (remember
6206 * index[1] == 1).
6207 *
6208 * Then to get the trace_array descriptor, by subtracting that index
6209 * from the ptr, we get to the start of the index itself.
6210 *
6211 * ptr - idx == &index[0]
6212 *
6213 * Then a simple container_of() from that pointer gets us to the
6214 * trace_array descriptor.
6215 */
6216static void get_tr_index(void *data, struct trace_array **ptr,
6217 unsigned int *pindex)
6218{
6219 *pindex = *(unsigned char *)data;
6220
6221 *ptr = container_of(data - *pindex, struct trace_array,
6222 trace_flags_index);
6223}
6224
Steven Rostedta8259072009-02-26 22:19:12 -05006225static ssize_t
6226trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6227 loff_t *ppos)
6228{
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006229 void *tr_index = filp->private_data;
6230 struct trace_array *tr;
6231 unsigned int index;
Steven Rostedta8259072009-02-26 22:19:12 -05006232 char *buf;
6233
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006234 get_tr_index(tr_index, &tr, &index);
6235
6236 if (tr->trace_flags & (1 << index))
Steven Rostedta8259072009-02-26 22:19:12 -05006237 buf = "1\n";
6238 else
6239 buf = "0\n";
6240
6241 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6242}
6243
6244static ssize_t
6245trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6246 loff_t *ppos)
6247{
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006248 void *tr_index = filp->private_data;
6249 struct trace_array *tr;
6250 unsigned int index;
Steven Rostedta8259072009-02-26 22:19:12 -05006251 unsigned long val;
6252 int ret;
6253
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006254 get_tr_index(tr_index, &tr, &index);
6255
Peter Huewe22fe9b52011-06-07 21:58:27 +02006256 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6257 if (ret)
Steven Rostedta8259072009-02-26 22:19:12 -05006258 return ret;
6259
Zhaoleif2d84b62009-08-07 18:55:48 +08006260 if (val != 0 && val != 1)
Steven Rostedta8259072009-02-26 22:19:12 -05006261 return -EINVAL;
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04006262
6263 mutex_lock(&trace_types_lock);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006264 ret = set_tracer_flag(tr, 1 << index, val);
Steven Rostedt (Red Hat)69d34da2013-03-14 13:50:56 -04006265 mutex_unlock(&trace_types_lock);
Steven Rostedta8259072009-02-26 22:19:12 -05006266
Steven Rostedt (Red Hat)613f04a2013-03-14 15:03:53 -04006267 if (ret < 0)
6268 return ret;
6269
Steven Rostedta8259072009-02-26 22:19:12 -05006270 *ppos += cnt;
6271
6272 return cnt;
6273}
6274
Steven Rostedta8259072009-02-26 22:19:12 -05006275static const struct file_operations trace_options_core_fops = {
6276 .open = tracing_open_generic,
6277 .read = trace_options_core_read,
6278 .write = trace_options_core_write,
Arnd Bergmannb4447862010-07-07 23:40:11 +02006279 .llseek = generic_file_llseek,
Steven Rostedta8259072009-02-26 22:19:12 -05006280};
6281
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006282struct dentry *trace_create_file(const char *name,
Al Virof4ae40a2011-07-24 04:33:43 -04006283 umode_t mode,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006284 struct dentry *parent,
6285 void *data,
6286 const struct file_operations *fops)
6287{
6288 struct dentry *ret;
6289
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006290 ret = tracefs_create_file(name, mode, parent, data, fops);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006291 if (!ret)
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006292 pr_warning("Could not create tracefs '%s' entry\n", name);
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006293
6294 return ret;
6295}
6296
6297
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006298static struct dentry *trace_options_init_dentry(struct trace_array *tr)
Steven Rostedta8259072009-02-26 22:19:12 -05006299{
6300 struct dentry *d_tracer;
Steven Rostedta8259072009-02-26 22:19:12 -05006301
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006302 if (tr->options)
6303 return tr->options;
Steven Rostedta8259072009-02-26 22:19:12 -05006304
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006305 d_tracer = tracing_get_dentry(tr);
Steven Rostedt (Red Hat)14a5ae42015-01-20 11:14:16 -05006306 if (IS_ERR(d_tracer))
Steven Rostedta8259072009-02-26 22:19:12 -05006307 return NULL;
6308
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006309 tr->options = tracefs_create_dir("options", d_tracer);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006310 if (!tr->options) {
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006311 pr_warning("Could not create tracefs directory 'options'\n");
Steven Rostedta8259072009-02-26 22:19:12 -05006312 return NULL;
6313 }
6314
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006315 return tr->options;
Steven Rostedta8259072009-02-26 22:19:12 -05006316}
6317
Steven Rostedt577b7852009-02-26 23:43:05 -05006318static void
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006319create_trace_option_file(struct trace_array *tr,
6320 struct trace_option_dentry *topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05006321 struct tracer_flags *flags,
6322 struct tracer_opt *opt)
6323{
6324 struct dentry *t_options;
Steven Rostedt577b7852009-02-26 23:43:05 -05006325
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006326 t_options = trace_options_init_dentry(tr);
Steven Rostedt577b7852009-02-26 23:43:05 -05006327 if (!t_options)
6328 return;
6329
6330 topt->flags = flags;
6331 topt->opt = opt;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006332 topt->tr = tr;
Steven Rostedt577b7852009-02-26 23:43:05 -05006333
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006334 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
Steven Rostedt577b7852009-02-26 23:43:05 -05006335 &trace_options_fops);
6336
Steven Rostedt577b7852009-02-26 23:43:05 -05006337}
6338
6339static struct trace_option_dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006340create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
Steven Rostedt577b7852009-02-26 23:43:05 -05006341{
6342 struct trace_option_dentry *topts;
6343 struct tracer_flags *flags;
6344 struct tracer_opt *opts;
6345 int cnt;
6346
6347 if (!tracer)
6348 return NULL;
6349
6350 flags = tracer->flags;
6351
6352 if (!flags || !flags->opts)
6353 return NULL;
6354
6355 opts = flags->opts;
6356
6357 for (cnt = 0; opts[cnt].name; cnt++)
6358 ;
6359
Steven Rostedt0cfe8242009-02-27 10:51:10 -05006360 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
Steven Rostedt577b7852009-02-26 23:43:05 -05006361 if (!topts)
6362 return NULL;
6363
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04006364 for (cnt = 0; opts[cnt].name; cnt++) {
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006365 create_trace_option_file(tr, &topts[cnt], flags,
Steven Rostedt577b7852009-02-26 23:43:05 -05006366 &opts[cnt]);
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04006367 WARN_ONCE(topts[cnt].entry == NULL,
6368 "Failed to create trace option: %s",
6369 opts[cnt].name);
6370 }
Steven Rostedt577b7852009-02-26 23:43:05 -05006371
6372 return topts;
6373}
6374
Steven Rostedta8259072009-02-26 22:19:12 -05006375static struct dentry *
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006376create_trace_option_core_file(struct trace_array *tr,
6377 const char *option, long index)
Steven Rostedta8259072009-02-26 22:19:12 -05006378{
6379 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05006380
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006381 t_options = trace_options_init_dentry(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05006382 if (!t_options)
6383 return NULL;
6384
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006385 return trace_create_file(option, 0644, t_options,
6386 (void *)&tr->trace_flags_index[index],
6387 &trace_options_core_fops);
Steven Rostedta8259072009-02-26 22:19:12 -05006388}
6389
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006390static __init void create_trace_options_dir(struct trace_array *tr)
Steven Rostedta8259072009-02-26 22:19:12 -05006391{
6392 struct dentry *t_options;
Steven Rostedta8259072009-02-26 22:19:12 -05006393 int i;
6394
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006395 t_options = trace_options_init_dentry(tr);
Steven Rostedta8259072009-02-26 22:19:12 -05006396 if (!t_options)
6397 return;
6398
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006399 for (i = 0; trace_options[i]; i++)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006400 create_trace_option_core_file(tr, trace_options[i], i);
Steven Rostedta8259072009-02-26 22:19:12 -05006401}
6402
Steven Rostedt499e5472012-02-22 15:50:28 -05006403static ssize_t
6404rb_simple_read(struct file *filp, char __user *ubuf,
6405 size_t cnt, loff_t *ppos)
6406{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04006407 struct trace_array *tr = filp->private_data;
Steven Rostedt499e5472012-02-22 15:50:28 -05006408 char buf[64];
6409 int r;
6410
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04006411 r = tracer_tracing_is_on(tr);
Steven Rostedt499e5472012-02-22 15:50:28 -05006412 r = sprintf(buf, "%d\n", r);
6413
6414 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6415}
6416
6417static ssize_t
6418rb_simple_write(struct file *filp, const char __user *ubuf,
6419 size_t cnt, loff_t *ppos)
6420{
Steven Rostedt348f0fc2012-04-16 15:41:28 -04006421 struct trace_array *tr = filp->private_data;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006422 struct ring_buffer *buffer = tr->trace_buffer.buffer;
Steven Rostedt499e5472012-02-22 15:50:28 -05006423 unsigned long val;
6424 int ret;
6425
6426 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6427 if (ret)
6428 return ret;
6429
6430 if (buffer) {
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05006431 mutex_lock(&trace_types_lock);
6432 if (val) {
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04006433 tracer_tracing_on(tr);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006434 if (tr->current_trace->start)
6435 tr->current_trace->start(tr);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05006436 } else {
Steven Rostedt (Red Hat)10246fa2013-07-01 15:58:24 -04006437 tracer_tracing_off(tr);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006438 if (tr->current_trace->stop)
6439 tr->current_trace->stop(tr);
Steven Rostedt2df8f8a2013-01-11 16:14:10 -05006440 }
6441 mutex_unlock(&trace_types_lock);
Steven Rostedt499e5472012-02-22 15:50:28 -05006442 }
6443
6444 (*ppos)++;
6445
6446 return cnt;
6447}
6448
6449static const struct file_operations rb_simple_fops = {
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04006450 .open = tracing_open_generic_tr,
Steven Rostedt499e5472012-02-22 15:50:28 -05006451 .read = rb_simple_read,
6452 .write = rb_simple_write,
Steven Rostedt (Red Hat)7b85af62013-07-01 23:34:22 -04006453 .release = tracing_release_generic_tr,
Steven Rostedt499e5472012-02-22 15:50:28 -05006454 .llseek = default_llseek,
6455};
6456
Steven Rostedt277ba042012-08-03 16:10:49 -04006457struct dentry *trace_instance_dir;
6458
6459static void
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006460init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
Steven Rostedt277ba042012-08-03 16:10:49 -04006461
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006462static int
6463allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
Steven Rostedt277ba042012-08-03 16:10:49 -04006464{
6465 enum ring_buffer_flags rb_flags;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006466
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04006467 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006468
Steven Rostedt (Red Hat)dced3412014-01-14 10:19:46 -05006469 buf->tr = tr;
6470
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006471 buf->buffer = ring_buffer_alloc(size, rb_flags);
6472 if (!buf->buffer)
6473 return -ENOMEM;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006474
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006475 buf->data = alloc_percpu(struct trace_array_cpu);
6476 if (!buf->data) {
6477 ring_buffer_free(buf->buffer);
6478 return -ENOMEM;
6479 }
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006480
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006481 /* Allocate the first page for all buffers */
6482 set_buffer_entries(&tr->trace_buffer,
6483 ring_buffer_size(tr->trace_buffer.buffer, 0));
6484
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006485 return 0;
6486}
6487
6488static int allocate_trace_buffers(struct trace_array *tr, int size)
6489{
6490 int ret;
6491
6492 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6493 if (ret)
6494 return ret;
6495
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006496#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006497 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6498 allocate_snapshot ? size : 1);
6499 if (WARN_ON(ret)) {
6500 ring_buffer_free(tr->trace_buffer.buffer);
6501 free_percpu(tr->trace_buffer.data);
6502 return -ENOMEM;
6503 }
6504 tr->allocated_snapshot = allocate_snapshot;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006505
Steven Rostedt (Red Hat)55034cd2013-03-07 22:48:09 -05006506 /*
6507 * Only the top level trace array gets its snapshot allocated
6508 * from the kernel command line.
6509 */
6510 allocate_snapshot = false;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006511#endif
6512 return 0;
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006513}
6514
Steven Rostedt (Red Hat)f0b70cc2014-06-10 12:06:30 -04006515static void free_trace_buffer(struct trace_buffer *buf)
6516{
6517 if (buf->buffer) {
6518 ring_buffer_free(buf->buffer);
6519 buf->buffer = NULL;
6520 free_percpu(buf->data);
6521 buf->data = NULL;
6522 }
6523}
6524
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006525static void free_trace_buffers(struct trace_array *tr)
6526{
6527 if (!tr)
6528 return;
6529
Steven Rostedt (Red Hat)f0b70cc2014-06-10 12:06:30 -04006530 free_trace_buffer(&tr->trace_buffer);
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006531
6532#ifdef CONFIG_TRACER_MAX_TRACE
Steven Rostedt (Red Hat)f0b70cc2014-06-10 12:06:30 -04006533 free_trace_buffer(&tr->max_buffer);
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006534#endif
6535}
6536
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006537static void init_trace_flags_index(struct trace_array *tr)
6538{
6539 int i;
6540
6541 /* Used by the trace options files */
6542 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
6543 tr->trace_flags_index[i] = i;
6544}
6545
Steven Rostedt (Red Hat)eae47352015-01-21 10:01:39 -05006546static int instance_mkdir(const char *name)
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006547{
Steven Rostedt277ba042012-08-03 16:10:49 -04006548 struct trace_array *tr;
6549 int ret;
Steven Rostedt277ba042012-08-03 16:10:49 -04006550
6551 mutex_lock(&trace_types_lock);
6552
6553 ret = -EEXIST;
6554 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6555 if (tr->name && strcmp(tr->name, name) == 0)
6556 goto out_unlock;
6557 }
6558
6559 ret = -ENOMEM;
6560 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6561 if (!tr)
6562 goto out_unlock;
6563
6564 tr->name = kstrdup(name, GFP_KERNEL);
6565 if (!tr->name)
6566 goto out_free_tr;
6567
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006568 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6569 goto out_free_tr;
6570
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04006571 tr->trace_flags = global_trace.trace_flags;
6572
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006573 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6574
Steven Rostedt277ba042012-08-03 16:10:49 -04006575 raw_spin_lock_init(&tr->start_lock);
6576
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05006577 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6578
Steven Rostedt277ba042012-08-03 16:10:49 -04006579 tr->current_trace = &nop_trace;
6580
6581 INIT_LIST_HEAD(&tr->systems);
6582 INIT_LIST_HEAD(&tr->events);
6583
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05006584 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
Steven Rostedt277ba042012-08-03 16:10:49 -04006585 goto out_free_tr;
6586
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006587 tr->dir = tracefs_create_dir(name, trace_instance_dir);
Steven Rostedt277ba042012-08-03 16:10:49 -04006588 if (!tr->dir)
6589 goto out_free_tr;
6590
6591 ret = event_trace_add_tracer(tr->dir, tr);
Alexander Z Lam609e85a2013-07-10 17:34:34 -07006592 if (ret) {
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006593 tracefs_remove_recursive(tr->dir);
Steven Rostedt277ba042012-08-03 16:10:49 -04006594 goto out_free_tr;
Alexander Z Lam609e85a2013-07-10 17:34:34 -07006595 }
Steven Rostedt277ba042012-08-03 16:10:49 -04006596
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006597 init_tracer_tracefs(tr, tr->dir);
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04006598 init_trace_flags_index(tr);
Steven Rostedt277ba042012-08-03 16:10:49 -04006599
6600 list_add(&tr->list, &ftrace_trace_arrays);
6601
6602 mutex_unlock(&trace_types_lock);
6603
6604 return 0;
6605
6606 out_free_tr:
Steven Rostedt (Red Hat)23aaa3c2014-06-06 00:01:46 -04006607 free_trace_buffers(tr);
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006608 free_cpumask_var(tr->tracing_cpumask);
Steven Rostedt277ba042012-08-03 16:10:49 -04006609 kfree(tr->name);
6610 kfree(tr);
6611
6612 out_unlock:
6613 mutex_unlock(&trace_types_lock);
6614
6615 return ret;
6616
6617}
6618
Steven Rostedt (Red Hat)eae47352015-01-21 10:01:39 -05006619static int instance_rmdir(const char *name)
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006620{
6621 struct trace_array *tr;
6622 int found = 0;
6623 int ret;
6624
6625 mutex_lock(&trace_types_lock);
6626
6627 ret = -ENODEV;
6628 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6629 if (tr->name && strcmp(tr->name, name) == 0) {
6630 found = 1;
6631 break;
6632 }
6633 }
6634 if (!found)
6635 goto out_unlock;
6636
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05006637 ret = -EBUSY;
Steven Rostedt (Red Hat)cf6ab6d2014-12-15 20:13:31 -05006638 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
Steven Rostedt (Red Hat)a695cb52013-03-06 15:27:24 -05006639 goto out_unlock;
6640
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006641 list_del(&tr->list);
6642
Steven Rostedt (Red Hat)6b450d22014-01-14 08:43:01 -05006643 tracing_set_nop(tr);
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006644 event_trace_del_tracer(tr);
Steven Rostedt (Red Hat)591dffd2014-01-10 16:17:45 -05006645 ftrace_destroy_function_files(tr);
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006646 debugfs_remove_recursive(tr->dir);
Steven Rostedt (Red Hat)a9fcaaa2014-06-06 23:17:28 -04006647 free_trace_buffers(tr);
Steven Rostedt0c8916c2012-08-07 16:14:16 -04006648
6649 kfree(tr->name);
6650 kfree(tr);
6651
6652 ret = 0;
6653
6654 out_unlock:
6655 mutex_unlock(&trace_types_lock);
6656
6657 return ret;
6658}
6659
Steven Rostedt277ba042012-08-03 16:10:49 -04006660static __init void create_trace_instances(struct dentry *d_tracer)
6661{
Steven Rostedt (Red Hat)eae47352015-01-21 10:01:39 -05006662 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
6663 instance_mkdir,
6664 instance_rmdir);
Steven Rostedt277ba042012-08-03 16:10:49 -04006665 if (WARN_ON(!trace_instance_dir))
6666 return;
Steven Rostedt277ba042012-08-03 16:10:49 -04006667}
6668
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006669static void
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006670init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006671{
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006672 int cpu;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006673
Steven Rostedt (Red Hat)607e2ea2013-11-06 22:42:48 -05006674 trace_create_file("available_tracers", 0444, d_tracer,
6675 tr, &show_traces_fops);
6676
6677 trace_create_file("current_tracer", 0644, d_tracer,
6678 tr, &set_tracer_fops);
6679
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07006680 trace_create_file("tracing_cpumask", 0644, d_tracer,
6681 tr, &tracing_cpumask_fops);
6682
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006683 trace_create_file("trace_options", 0644, d_tracer,
6684 tr, &tracing_iter_fops);
6685
6686 trace_create_file("trace", 0644, d_tracer,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006687 tr, &tracing_fops);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006688
6689 trace_create_file("trace_pipe", 0444, d_tracer,
Oleg Nesterov15544202013-07-23 17:25:57 +02006690 tr, &tracing_pipe_fops);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006691
6692 trace_create_file("buffer_size_kb", 0644, d_tracer,
Oleg Nesterov0bc392e2013-07-23 17:26:06 +02006693 tr, &tracing_entries_fops);
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006694
6695 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6696 tr, &tracing_total_entries_fops);
6697
Wang YanQing238ae932013-05-26 16:52:01 +08006698 trace_create_file("free_buffer", 0200, d_tracer,
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006699 tr, &tracing_free_buffer_fops);
6700
6701 trace_create_file("trace_marker", 0220, d_tracer,
6702 tr, &tracing_mark_fops);
6703
6704 trace_create_file("trace_clock", 0644, d_tracer, tr,
6705 &trace_clock_fops);
6706
6707 trace_create_file("tracing_on", 0644, d_tracer,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006708 tr, &rb_simple_fops);
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05006709
Steven Rostedt (Red Hat)6d9b3fa2014-01-14 11:28:38 -05006710#ifdef CONFIG_TRACER_MAX_TRACE
6711 trace_create_file("tracing_max_latency", 0644, d_tracer,
6712 &tr->max_latency, &tracing_max_lat_fops);
6713#endif
6714
Steven Rostedt (Red Hat)591dffd2014-01-10 16:17:45 -05006715 if (ftrace_create_function_files(tr, d_tracer))
6716 WARN(1, "Could not allocate function filter files");
6717
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05006718#ifdef CONFIG_TRACER_SNAPSHOT
6719 trace_create_file("snapshot", 0644, d_tracer,
Oleg Nesterov6484c712013-07-23 17:26:10 +02006720 tr, &snapshot_fops);
Steven Rostedt (Red Hat)ce9bae52013-03-05 21:23:55 -05006721#endif
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006722
6723 for_each_tracing_cpu(cpu)
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006724 tracing_init_tracefs_percpu(tr, cpu);
Steven Rostedt (Red Hat)121aaee2013-03-05 21:52:25 -05006725
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006726}
6727
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006728static struct vfsmount *trace_automount(void *ingore)
6729{
6730 struct vfsmount *mnt;
6731 struct file_system_type *type;
6732
6733 /*
6734 * To maintain backward compatibility for tools that mount
6735 * debugfs to get to the tracing facility, tracefs is automatically
6736 * mounted to the debugfs/tracing directory.
6737 */
6738 type = get_fs_type("tracefs");
6739 if (!type)
6740 return NULL;
6741 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
6742 put_filesystem(type);
6743 if (IS_ERR(mnt))
6744 return NULL;
6745 mntget(mnt);
6746
6747 return mnt;
6748}
6749
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006750/**
6751 * tracing_init_dentry - initialize top level trace array
6752 *
6753 * This is called when creating files or directories in the tracing
6754 * directory. It is called via fs_initcall() by any of the boot up code
6755 * and expects to return the dentry of the top level tracing directory.
6756 */
6757struct dentry *tracing_init_dentry(void)
6758{
6759 struct trace_array *tr = &global_trace;
6760
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006761 /* The top level trace array uses NULL as parent */
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006762 if (tr->dir)
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006763 return NULL;
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006764
6765 if (WARN_ON(!debugfs_initialized()))
6766 return ERR_PTR(-ENODEV);
6767
Steven Rostedt (Red Hat)f76180b2015-01-20 15:48:46 -05006768 /*
6769 * As there may still be users that expect the tracing
6770 * files to exist in debugfs/tracing, we must automount
6771 * the tracefs file system there, so older tools still
6772 * work with the newer kerenl.
6773 */
6774 tr->dir = debugfs_create_automount("tracing", NULL,
6775 trace_automount, NULL);
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006776 if (!tr->dir) {
6777 pr_warn_once("Could not create debugfs directory 'tracing'\n");
6778 return ERR_PTR(-ENOMEM);
6779 }
6780
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006781 return NULL;
Steven Rostedt (Red Hat)7eeafbc2015-01-26 21:00:48 -05006782}
6783
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04006784extern struct trace_enum_map *__start_ftrace_enum_maps[];
6785extern struct trace_enum_map *__stop_ftrace_enum_maps[];
6786
6787static void __init trace_enum_init(void)
6788{
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006789 int len;
6790
6791 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006792 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04006793}
6794
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006795#ifdef CONFIG_MODULES
6796static void trace_module_add_enums(struct module *mod)
6797{
6798 if (!mod->num_trace_enums)
6799 return;
6800
6801 /*
6802 * Modules with bad taint do not have events created, do
6803 * not bother with enums either.
6804 */
6805 if (trace_module_has_bad_taint(mod))
6806 return;
6807
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006808 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006809}
6810
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006811#ifdef CONFIG_TRACE_ENUM_MAP_FILE
6812static void trace_module_remove_enums(struct module *mod)
6813{
6814 union trace_enum_map_item *map;
6815 union trace_enum_map_item **last = &trace_enum_maps;
6816
6817 if (!mod->num_trace_enums)
6818 return;
6819
6820 mutex_lock(&trace_enum_mutex);
6821
6822 map = trace_enum_maps;
6823
6824 while (map) {
6825 if (map->head.mod == mod)
6826 break;
6827 map = trace_enum_jmp_to_tail(map);
6828 last = &map->tail.next;
6829 map = map->tail.next;
6830 }
6831 if (!map)
6832 goto out;
6833
6834 *last = trace_enum_jmp_to_tail(map)->tail.next;
6835 kfree(map);
6836 out:
6837 mutex_unlock(&trace_enum_mutex);
6838}
6839#else
6840static inline void trace_module_remove_enums(struct module *mod) { }
6841#endif /* CONFIG_TRACE_ENUM_MAP_FILE */
6842
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006843static int trace_module_notify(struct notifier_block *self,
6844 unsigned long val, void *data)
6845{
6846 struct module *mod = data;
6847
6848 switch (val) {
6849 case MODULE_STATE_COMING:
6850 trace_module_add_enums(mod);
6851 break;
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006852 case MODULE_STATE_GOING:
6853 trace_module_remove_enums(mod);
6854 break;
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006855 }
6856
6857 return 0;
6858}
6859
6860static struct notifier_block trace_module_nb = {
6861 .notifier_call = trace_module_notify,
6862 .priority = 0,
6863};
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006864#endif /* CONFIG_MODULES */
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006865
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006866static __init int tracer_init_tracefs(void)
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006867{
6868 struct dentry *d_tracer;
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04006869 struct tracer *t;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006870
Lai Jiangshan7e53bd42010-01-06 20:08:50 +08006871 trace_access_lock_init();
6872
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006873 d_tracer = tracing_init_dentry();
Steven Rostedt (Red Hat)14a5ae42015-01-20 11:14:16 -05006874 if (IS_ERR(d_tracer))
Namhyung Kimed6f1c92013-04-10 09:18:12 +09006875 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006876
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05006877 init_tracer_tracefs(&global_trace, d_tracer);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006878
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006879 trace_create_file("tracing_thresh", 0644, d_tracer,
Stanislav Fomichev6508fa72014-07-18 15:17:27 +04006880 &global_trace, &tracing_thresh_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006881
Li Zefan339ae5d2009-04-17 10:34:30 +08006882 trace_create_file("README", 0444, d_tracer,
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006883 NULL, &tracing_readme_fops);
Ingo Molnar7bd2f242008-05-12 21:20:45 +02006884
Avadh Patel69abe6a2009-04-10 16:04:48 -04006885 trace_create_file("saved_cmdlines", 0444, d_tracer,
6886 NULL, &tracing_saved_cmdlines_fops);
Pekka Paalanen5bf9a1e2008-09-16 22:06:42 +03006887
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09006888 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6889 NULL, &tracing_saved_cmdlines_size_fops);
6890
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04006891 trace_enum_init();
6892
Steven Rostedt (Red Hat)98284132015-03-31 17:23:45 -04006893 trace_create_enum_file(d_tracer);
6894
Steven Rostedt (Red Hat)3673b8e2015-03-25 15:44:21 -04006895#ifdef CONFIG_MODULES
6896 register_module_notifier(&trace_module_nb);
6897#endif
6898
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006899#ifdef CONFIG_DYNAMIC_FTRACE
Frederic Weisbecker5452af62009-03-27 00:25:38 +01006900 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6901 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006902#endif
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006903
Steven Rostedt277ba042012-08-03 16:10:49 -04006904 create_trace_instances(d_tracer);
Hiraku Toyookadebdd572012-12-26 11:53:00 +09006905
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006906 create_trace_options_dir(&global_trace);
Frederic Weisbeckerb04cc6b2009-02-25 03:22:28 +01006907
Steven Rostedt (Red Hat)41d9c0b2015-09-29 17:31:55 -04006908 mutex_lock(&trace_types_lock);
6909 for (t = trace_types; t; t = t->next)
6910 add_tracer_options(&global_trace, t);
6911 mutex_unlock(&trace_types_lock);
Steven Rostedt (Red Hat)09d23a12015-02-03 12:45:53 -05006912
Frédéric Weisbeckerb5ad3842008-09-23 11:34:32 +01006913 return 0;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02006914}
6915
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006916static int trace_panic_handler(struct notifier_block *this,
6917 unsigned long event, void *unused)
6918{
Steven Rostedt944ac422008-10-23 19:26:08 -04006919 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006920 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006921 return NOTIFY_OK;
6922}
6923
6924static struct notifier_block trace_panic_notifier = {
6925 .notifier_call = trace_panic_handler,
6926 .next = NULL,
6927 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6928};
6929
6930static int trace_die_handler(struct notifier_block *self,
6931 unsigned long val,
6932 void *data)
6933{
6934 switch (val) {
6935 case DIE_OOPS:
Steven Rostedt944ac422008-10-23 19:26:08 -04006936 if (ftrace_dump_on_oops)
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02006937 ftrace_dump(ftrace_dump_on_oops);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006938 break;
6939 default:
6940 break;
6941 }
6942 return NOTIFY_OK;
6943}
6944
6945static struct notifier_block trace_die_notifier = {
6946 .notifier_call = trace_die_handler,
6947 .priority = 200
6948};
6949
6950/*
6951 * printk is set to max of 1024, we really don't need it that big.
6952 * Nothing should be printing 1000 characters anyway.
6953 */
6954#define TRACE_MAX_PRINT 1000
6955
6956/*
6957 * Define here KERN_TRACE so that we have one place to modify
6958 * it if we decide to change what log level the ftrace dump
6959 * should be at.
6960 */
Steven Rostedt428aee12009-01-14 12:24:42 -05006961#define KERN_TRACE KERN_EMERG
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006962
Jason Wessel955b61e2010-08-05 09:22:23 -05006963void
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006964trace_printk_seq(struct trace_seq *s)
6965{
6966 /* Probably should print a warning here. */
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04006967 if (s->seq.len >= TRACE_MAX_PRINT)
6968 s->seq.len = TRACE_MAX_PRINT;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006969
Steven Rostedt (Red Hat)820b75f2014-11-19 10:56:41 -05006970 /*
6971 * More paranoid code. Although the buffer size is set to
6972 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
6973 * an extra layer of protection.
6974 */
6975 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
6976 s->seq.len = s->seq.size - 1;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006977
6978 /* should be zero ended, but we are paranoid. */
Steven Rostedt (Red Hat)3a161d92014-06-25 15:54:42 -04006979 s->buffer[s->seq.len] = 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006980
6981 printk(KERN_TRACE "%s", s->buffer);
6982
Steven Rostedtf9520752009-03-02 14:04:40 -05006983 trace_seq_init(s);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04006984}
6985
Jason Wessel955b61e2010-08-05 09:22:23 -05006986void trace_init_global_iter(struct trace_iterator *iter)
6987{
6988 iter->tr = &global_trace;
Steven Rostedt2b6080f2012-05-11 13:29:49 -04006989 iter->trace = iter->tr->current_trace;
Steven Rostedtae3b5092013-01-23 15:22:59 -05006990 iter->cpu_file = RING_BUFFER_ALL_CPUS;
Steven Rostedt (Red Hat)12883ef2013-03-05 09:24:35 -05006991 iter->trace_buffer = &global_trace.trace_buffer;
Cody P Schaferb2f974d2013-10-23 11:49:57 -07006992
6993 if (iter->trace && iter->trace->open)
6994 iter->trace->open(iter);
6995
6996 /* Annotate start of buffers if we had overruns */
6997 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6998 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6999
7000 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7001 if (trace_clocks[iter->tr->clock_id].in_ns)
7002 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
Jason Wessel955b61e2010-08-05 09:22:23 -05007003}
7004
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007005void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007006{
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007007 /* use static because iter can be a bit big for the stack */
7008 static struct trace_iterator iter;
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007009 static atomic_t dump_running;
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007010 struct trace_array *tr = &global_trace;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007011 unsigned int old_userobj;
Steven Rostedtd7690412008-10-01 00:29:53 -04007012 unsigned long flags;
7013 int cnt = 0, cpu;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007014
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007015 /* Only allow one dump user at a time. */
7016 if (atomic_inc_return(&dump_running) != 1) {
7017 atomic_dec(&dump_running);
7018 return;
Steven Rostedte0a413f2011-09-29 21:26:16 -04007019 }
7020
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007021 /*
7022 * Always turn off tracing when we dump.
7023 * We don't need to show trace output of what happens
7024 * between multiple crashes.
7025 *
7026 * If the user does a sysrq-z, then they can re-enable
7027 * tracing with echo 1 > tracing_on.
7028 */
7029 tracing_off();
7030
7031 local_irq_save(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007032
Jovi Zhang38dbe0b2013-01-25 18:03:07 +08007033 /* Simulate the iterator */
Jason Wessel955b61e2010-08-05 09:22:23 -05007034 trace_init_global_iter(&iter);
7035
Steven Rostedtd7690412008-10-01 00:29:53 -04007036 for_each_tracing_cpu(cpu) {
Umesh Tiwari5e2d5ef2015-06-22 16:55:06 +05307037 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
Steven Rostedtd7690412008-10-01 00:29:53 -04007038 }
7039
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007040 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007041
Török Edwinb54d3de2008-11-22 13:28:48 +02007042 /* don't look at user memory in panic mode */
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007043 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
Török Edwinb54d3de2008-11-22 13:28:48 +02007044
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007045 switch (oops_dump_mode) {
7046 case DUMP_ALL:
Steven Rostedtae3b5092013-01-23 15:22:59 -05007047 iter.cpu_file = RING_BUFFER_ALL_CPUS;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007048 break;
7049 case DUMP_ORIG:
7050 iter.cpu_file = raw_smp_processor_id();
7051 break;
7052 case DUMP_NONE:
7053 goto out_enable;
7054 default:
7055 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
Steven Rostedtae3b5092013-01-23 15:22:59 -05007056 iter.cpu_file = RING_BUFFER_ALL_CPUS;
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007057 }
7058
7059 printk(KERN_TRACE "Dumping ftrace buffer:\n");
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007060
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007061 /* Did function tracer already get disabled? */
7062 if (ftrace_is_dead()) {
7063 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7064 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7065 }
7066
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007067 /*
7068 * We need to stop all tracing on all CPUS to read the
7069 * the next buffer. This is a bit expensive, but is
7070 * not done often. We fill all what we can read,
7071 * and then release the locks again.
7072 */
7073
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007074 while (!trace_empty(&iter)) {
7075
7076 if (!cnt)
7077 printk(KERN_TRACE "---------------------------------\n");
7078
7079 cnt++;
7080
7081 /* reset all but tr, trace, and overruns */
7082 memset(&iter.seq, 0,
7083 sizeof(struct trace_iterator) -
7084 offsetof(struct trace_iterator, seq));
7085 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7086 iter.pos = -1;
7087
Jason Wessel955b61e2010-08-05 09:22:23 -05007088 if (trace_find_next_entry_inc(&iter) != NULL) {
Lai Jiangshan74e7ff82009-07-28 20:17:22 +08007089 int ret;
7090
7091 ret = print_trace_line(&iter);
7092 if (ret != TRACE_TYPE_NO_CONSUME)
7093 trace_consume(&iter);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007094 }
Steven Rostedtb892e5c2012-03-01 22:06:48 -05007095 touch_nmi_watchdog();
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007096
7097 trace_printk_seq(&iter.seq);
7098 }
7099
7100 if (!cnt)
7101 printk(KERN_TRACE " (ftrace buffer empty)\n");
7102 else
7103 printk(KERN_TRACE "---------------------------------\n");
7104
Frederic Weisbeckercecbca92010-04-18 19:08:41 +02007105 out_enable:
Steven Rostedt (Red Hat)983f9382015-09-30 09:42:05 -04007106 tr->trace_flags |= old_userobj;
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007107
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007108 for_each_tracing_cpu(cpu) {
7109 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007110 }
Steven Rostedt (Red Hat)7fe70b52013-03-15 13:10:35 -04007111 atomic_dec(&dump_running);
Steven Rostedtcd891ae2009-04-28 11:39:34 -04007112 local_irq_restore(flags);
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007113}
Paul E. McKenneya8eecf22011-10-02 11:01:15 -07007114EXPORT_SYMBOL_GPL(ftrace_dump);
Frederic Weisbeckercf586b62009-03-22 05:04:35 +01007115
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02007116__init static int tracer_alloc_buffers(void)
7117{
Steven Rostedt73c51622009-03-11 13:42:01 -04007118 int ring_buf_size;
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307119 int ret = -ENOMEM;
7120
Steven Rostedt (Red Hat)b5e87c02015-09-29 18:13:33 -04007121 /*
7122 * Make sure we don't accidently add more trace options
7123 * than we have bits for.
7124 */
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04007125 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
Steven Rostedt (Red Hat)b5e87c02015-09-29 18:13:33 -04007126
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307127 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7128 goto out;
7129
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07007130 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307131 goto out_free_buffer_mask;
7132
Steven Rostedt07d777f2011-09-22 14:01:55 -04007133 /* Only allocate trace_printk buffers if a trace_printk exists */
7134 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
Steven Rostedt81698832012-10-11 10:15:05 -04007135 /* Must be called before global_trace.buffer is allocated */
Steven Rostedt07d777f2011-09-22 14:01:55 -04007136 trace_printk_init_buffers();
7137
Steven Rostedt73c51622009-03-11 13:42:01 -04007138 /* To save memory, keep the ring buffer size to its minimum */
7139 if (ring_buffer_expanded)
7140 ring_buf_size = trace_buf_size;
7141 else
7142 ring_buf_size = 1;
7143
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307144 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07007145 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02007146
Steven Rostedt2b6080f2012-05-11 13:29:49 -04007147 raw_spin_lock_init(&global_trace.start_lock);
7148
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04007149 /* Used for event triggers */
7150 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7151 if (!temp_buffer)
7152 goto out_free_cpumask;
7153
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09007154 if (trace_create_savedcmd() < 0)
7155 goto out_free_temp_buffer;
7156
Steven Rostedtab464282008-05-12 21:21:00 +02007157 /* TODO: make the number of buffers hot pluggable with CPUS */
Steven Rostedt (Red Hat)737223f2013-03-05 21:13:47 -05007158 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007159 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7160 WARN_ON(1);
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09007161 goto out_free_savedcmd;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007162 }
Steven Rostedta7603ff2012-08-06 16:24:11 -04007163
Steven Rostedt499e5472012-02-22 15:50:28 -05007164 if (global_trace.buffer_disabled)
7165 tracing_off();
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007166
Steven Rostedte1e232c2014-02-10 23:38:46 -05007167 if (trace_boot_clock) {
7168 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7169 if (ret < 0)
7170 pr_warning("Trace clock %s not defined, going back to default\n",
7171 trace_boot_clock);
7172 }
7173
Steven Rostedt (Red Hat)ca164312013-05-23 11:51:10 -04007174 /*
7175 * register_tracer() might reference current_trace, so it
7176 * needs to be set before we register anything. This is
7177 * just a bootstrap of current_trace anyway.
7178 */
Steven Rostedt2b6080f2012-05-11 13:29:49 -04007179 global_trace.current_trace = &nop_trace;
7180
Steven Rostedt (Red Hat)0b9b12c2014-01-14 10:04:59 -05007181 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7182
Steven Rostedt (Red Hat)4104d322014-01-10 17:01:58 -05007183 ftrace_init_global_array_ops(&global_trace);
7184
Steven Rostedt (Red Hat)9a38a882015-09-30 11:11:15 -04007185 init_trace_flags_index(&global_trace);
7186
Steven Rostedt (Red Hat)ca164312013-05-23 11:51:10 -04007187 register_tracer(&nop_trace);
7188
Steven Rostedt60a11772008-05-12 21:20:44 +02007189 /* All seems OK, enable tracing */
7190 tracing_disabled = 0;
Steven Rostedt3928a8a2008-09-29 23:02:41 -04007191
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007192 atomic_notifier_chain_register(&panic_notifier_list,
7193 &trace_panic_notifier);
7194
7195 register_die_notifier(&trace_die_notifier);
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01007196
Steven Rostedtae63b312012-05-03 23:09:03 -04007197 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7198
7199 INIT_LIST_HEAD(&global_trace.systems);
7200 INIT_LIST_HEAD(&global_trace.events);
7201 list_add(&global_trace.list, &ftrace_trace_arrays);
7202
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04007203 while (trace_boot_options) {
7204 char *option;
7205
7206 option = strsep(&trace_boot_options, ",");
Steven Rostedt2b6080f2012-05-11 13:29:49 -04007207 trace_set_options(&global_trace, option);
Steven Rostedt7bcfaf52012-11-01 22:56:07 -04007208 }
7209
Steven Rostedt (Red Hat)77fd5c12013-03-12 11:49:18 -04007210 register_snapshot_cmd();
7211
Frederic Weisbecker2fc1dfb2009-03-16 01:45:03 +01007212 return 0;
Steven Rostedt3f5a54e2008-07-30 22:36:46 -04007213
Yoshihiro YUNOMAE939c7a42014-06-05 10:24:27 +09007214out_free_savedcmd:
7215 free_saved_cmdlines_buffer(savedcmd);
Steven Rostedt (Red Hat)2c4a33a2014-03-25 23:39:41 -04007216out_free_temp_buffer:
7217 ring_buffer_free(temp_buffer);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307218out_free_cpumask:
Alexander Z Lamccfe9e42013-08-08 09:47:45 -07007219 free_cpumask_var(global_trace.tracing_cpumask);
Rusty Russell9e01c1b2009-01-01 10:12:22 +10307220out_free_buffer_mask:
7221 free_cpumask_var(tracing_buffer_mask);
7222out:
7223 return ret;
Steven Rostedtbc0c38d2008-05-12 21:20:42 +02007224}
Steven Rostedtb2821ae2009-02-02 21:38:32 -05007225
Steven Rostedt (Red Hat)5f893b22014-12-12 20:05:10 -05007226void __init trace_init(void)
7227{
Steven Rostedt (Red Hat)0daa23022014-12-12 22:27:10 -05007228 if (tracepoint_printk) {
7229 tracepoint_print_iter =
7230 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7231 if (WARN_ON(!tracepoint_print_iter))
7232 tracepoint_printk = 0;
7233 }
Steven Rostedt (Red Hat)5f893b22014-12-12 20:05:10 -05007234 tracer_alloc_buffers();
Steven Rostedt (Red Hat)0c564a52015-03-24 17:58:09 -04007235 trace_event_init();
Steven Rostedt (Red Hat)5f893b22014-12-12 20:05:10 -05007236}
7237
Steven Rostedtb2821ae2009-02-02 21:38:32 -05007238__init static int clear_boot_tracer(void)
7239{
7240 /*
7241 * The default tracer at boot buffer is an init section.
7242 * This function is called in lateinit. If we did not
7243 * find the boot tracer, then clear it out, to prevent
7244 * later registration from accessing the buffer that is
7245 * about to be freed.
7246 */
7247 if (!default_bootup_tracer)
7248 return 0;
7249
7250 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7251 default_bootup_tracer);
7252 default_bootup_tracer = NULL;
7253
7254 return 0;
7255}
7256
Steven Rostedt (Red Hat)8434dc92015-01-20 12:13:40 -05007257fs_initcall(tracer_init_tracefs);
Steven Rostedtb2821ae2009-02-02 21:38:32 -05007258late_initcall(clear_boot_tracer);